From b75dc806179585976371be2223460ab657521292 Mon Sep 17 00:00:00 2001 From: SYMSCAE Date: Tue, 17 Nov 2026 18:28:55 -0800 Subject: [PATCH 1/6] feat: accelerate PDF and Debye calculations on the CPU Group equivalent pairs and reuse neighbor lists across small structure changes. Automatically sum all pairs for supported calculators and retain pair-by-pair evaluation for custom models and partitioned runs. Choose between an incremental update and a full sum based on the estimated work. Use per-site occupancies in Debye sums, preserve valid updates across adapter classes, and isolate calculator copies and concurrent evaluations. Split large CPU pair batches across workers and merge their private sums in range order. Choose one worker per 8192 pairs in large batches, up to the hardware concurrency reported by the system. Keep distinct floating-point distances and widths separate at cutoffs. Track site order across incremental updates and honor site-dependent scattering factors in derived Debye calculators. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 2 +- CMakePresets.json | 44 +- cmake/CalculationKernels.cmake | 11 +- cmake/libdiffpyConfig.cmake.in | 1 + docs/source/development.rst | 6 +- news/cuda.rst | 35 + src/diffpy/EventTicker.cpp | 37 +- src/diffpy/EventTicker.hpp | 9 +- src/diffpy/HasClassRegistry.ipp | 8 +- src/diffpy/srreal/AtomicStructureAdapter.cpp | 13 + src/diffpy/srreal/AtomicStructureAdapter.hpp | 3 + src/diffpy/srreal/BaseBondGenerator.cpp | 3 +- src/diffpy/srreal/BaseBondGenerator.hpp | 8 + src/diffpy/srreal/BaseDebyeSum.cpp | 240 +++- src/diffpy/srreal/BaseDebyeSum.hpp | 39 +- src/diffpy/srreal/ConstantPeakWidth.cpp | 4 + src/diffpy/srreal/ConstantPeakWidth.hpp | 3 + src/diffpy/srreal/CrystalStructureAdapter.cpp | 42 +- src/diffpy/srreal/DebyePDFCalculator.cpp | 6 + src/diffpy/srreal/DebyePDFCalculator.hpp | 1 + src/diffpy/srreal/DebyeWallerPeakWidth.cpp | 10 +- src/diffpy/srreal/DebyeWallerPeakWidth.hpp | 3 + src/diffpy/srreal/JeongPeakWidth.cpp | 25 +- src/diffpy/srreal/JeongPeakWidth.hpp | 3 + src/diffpy/srreal/PDFCalculator.cpp | 66 +- src/diffpy/srreal/PDFCalculator.hpp | 10 +- src/diffpy/srreal/PQEvaluator.cpp | 181 +-- src/diffpy/srreal/PQEvaluator.hpp | 15 +- src/diffpy/srreal/PairQuantity.cpp | 31 +- src/diffpy/srreal/PairQuantity.hpp | 64 +- .../srreal/PeriodicStructureAdapter.hpp | 2 + src/diffpy/srreal/ScatteringFactorTable.cpp | 14 +- src/diffpy/srreal/ScatteringFactorTable.hpp | 4 + .../srreal/kernels/CalculationKernelCPU.cpp | 154 +++ .../srreal/kernels/CalculationKernelImpl.hpp | 143 ++- .../srreal/kernels/CalculationKernelSIMD.ipp | 15 +- .../kernels/CalculationKernelScalar.cpp | 10 +- src/diffpy/srreal/kernels/PairGeometry.hpp | 146 +++ src/diffpy/srreal/kernels/PairTable.cpp | 336 ++++++ src/diffpy/srreal/kernels/PairTable.hpp | 231 ++++ src/diffpy/srreal/scatteringfactordata.cpp | 114 +- src/tests/TestBondGeneratorBatching.cpp | 68 +- src/tests/TestCalculationKernel.cpp | 46 +- src/tests/TestDebyePDFCalculator.cpp | 170 +++ src/tests/TestPQEvaluator.cpp | 205 +++- src/tests/TestPairTable.cpp | 1028 +++++++++++++++++ src/tests/TestPeakWidthModel.cpp | 3 + src/tests/TestThreads.cpp | 165 +++ src/tests/pair_test_helpers.hpp | 157 +++ src/tests/test_helpers.hpp | 41 +- 50 files changed, 3618 insertions(+), 357 deletions(-) create mode 100644 news/cuda.rst create mode 100644 src/diffpy/srreal/kernels/CalculationKernelCPU.cpp create mode 100644 src/diffpy/srreal/kernels/PairGeometry.hpp create mode 100644 src/diffpy/srreal/kernels/PairTable.cpp create mode 100644 src/diffpy/srreal/kernels/PairTable.hpp create mode 100644 src/tests/TestPairTable.cpp create mode 100644 src/tests/TestThreads.cpp create mode 100644 src/tests/pair_test_helpers.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a78169d..bffee3d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,7 +91,7 @@ if(MSVC AND (LIBDIFFPY_COVERAGE OR LIBDIFFPY_PROFILE)) endif() if(LIBDIFFPY_COVERAGE) - add_compile_options(--coverage -O0 -g) + add_compile_options(--coverage -fprofile-update=atomic -O0 -g) add_link_options(--coverage) endif() diff --git a/CMakePresets.json b/CMakePresets.json index 3dd54d3..28331e0 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -18,44 +18,42 @@ } }, { - "name": "release", + "name": "debug", "inherits": "ninja", - "displayName": "Optimized build", + "displayName": "Debug build with C assertions and unit tests", "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" + "CMAKE_BUILD_TYPE": "Debug", + "LIBDIFFPY_BUILD_TESTS": "ON" } }, { - "name": "debug", + "name": "release", "inherits": "ninja", - "displayName": "Debug build with C assertions and unit tests", + "displayName": "Optimized build", "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "LIBDIFFPY_BUILD_TESTS": "ON" + "CMAKE_BUILD_TYPE": "Release" } }, { "name": "coverage", - "inherits": "ninja", + "inherits": "debug", "displayName": "Debug build instrumented for coverage analysis", "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "LIBDIFFPY_BUILD_TESTS": "ON", "LIBDIFFPY_COVERAGE": "ON" } } ], "buildPresets": [ - { - "name": "release", - "configurePreset": "release", - "configuration": "Release" - }, { "name": "debug", "configurePreset": "debug", "configuration": "Debug" }, + { + "name": "release", + "configurePreset": "release", + "configuration": "Release" + }, { "name": "coverage", "configurePreset": "coverage", @@ -74,16 +72,16 @@ "outputOnFailure": true } }, + { + "name": "release", + "inherits": "debug", + "configurePreset": "release", + "configuration": "Release" + }, { "name": "coverage", - "configurePreset": "coverage", - "configuration": "Debug", - "execution": { - "noTestsAction": "error" - }, - "output": { - "outputOnFailure": true - } + "inherits": "debug", + "configurePreset": "coverage" } ] } diff --git a/cmake/CalculationKernels.cmake b/cmake/CalculationKernels.cmake index 2987911..a7d5a40 100644 --- a/cmake/CalculationKernels.cmake +++ b/cmake/CalculationKernels.cmake @@ -40,9 +40,13 @@ endif() # The main source glob is not recursive, add the kernels explicitly. set(_kernel_dir "${PROJECT_SOURCE_DIR}/src/diffpy/srreal/kernels") +find_package(Threads REQUIRED) +target_link_libraries(diffpy PRIVATE Threads::Threads) target_sources(diffpy PRIVATE + "${_kernel_dir}/CalculationKernelCPU.cpp" "${_kernel_dir}/CalculationKernelScalar.cpp" - "${_kernel_dir}/CalculationKernelDispatch.cpp") + "${_kernel_dir}/CalculationKernelDispatch.cpp" + "${_kernel_dir}/PairTable.cpp") set(_kernel_names scalar) # All kernels need IEEE arithmetic to give consistent results. @@ -51,7 +55,10 @@ if(MSVC) else() set(_kernel_math_options -fno-fast-math -ffp-contract=off) endif() -set_source_files_properties("${_kernel_dir}/CalculationKernelScalar.cpp" +set_source_files_properties( + "${_kernel_dir}/CalculationKernelCPU.cpp" + "${_kernel_dir}/CalculationKernelScalar.cpp" + "${_kernel_dir}/PairTable.cpp" PROPERTIES COMPILE_OPTIONS "${_kernel_math_options}") # Inline functions compiled for the SIMD instruction sets must not be shared diff --git a/cmake/libdiffpyConfig.cmake.in b/cmake/libdiffpyConfig.cmake.in index 11700d6..75330b4 100644 --- a/cmake/libdiffpyConfig.cmake.in +++ b/cmake/libdiffpyConfig.cmake.in @@ -20,6 +20,7 @@ set_and_check(libdiffpy_RUNTIME_DIR "@PACKAGE_LIBDIFFPY_RUNTIME_INSTALL_DIR@") find_dependency(Boost CONFIG COMPONENTS serialization) if(NOT "@_libdiffpy_library_type@" STREQUAL "SHARED_LIBRARY") find_dependency(GSL) + find_dependency(Threads) endif() if(libdiffpy_HAS_OBJCRYST) list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") diff --git a/docs/source/development.rst b/docs/source/development.rst index 606b214..f651b27 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -32,8 +32,8 @@ The CMake presets require CMake 3.22 or later and use Ninja. On Windows, run them in a command prompt with the conda build environment activated. The build presets are -* ``release`` - optimized build with C assertion checks disabled * ``debug`` - unoptimized build with debug information and C assertion checks +* ``release`` - optimized build with C assertion checks disabled * ``coverage`` - unoptimized build instrumented for coverage analysis The library is built in the ``build/`` directory, for example @@ -48,8 +48,8 @@ ObjCryst adapter, pass ``-DLIBDIFFPY_ENABLE_OBJCRYST=OFF``. For gprof instrumentation, use ``-DLIBDIFFPY_PROFILE=ON``. To build the optimized shared library without link-time optimization, pass ``-DLIBDIFFPY_LTO=OFF``. -Calculation kernels -------------------- +SIMD calculation kernels +------------------------ ``PDFCalculator`` and ``DebyePDFCalculator`` run their inner loops in a calculation kernel. The scalar kernel is always built. With the default diff --git a/news/cuda.rst b/news/cuda.rst new file mode 100644 index 0000000..01e345d --- /dev/null +++ b/news/cuda.rst @@ -0,0 +1,35 @@ +**Added:** + +* Derived calculators can implement ``addAllPairContributions`` or reuse + ``addAllPairsOfPDF`` and ``addAllPairsOfDebyeSum``. The constant, Debye-Waller + and Jeong peak width models now provide ``calculateFromMSD``. + +**Changed:** + +* ``PDFCalculator`` and ``DebyePDFCalculator`` accelerate full sums with + batched pairs and multiple CPU threads, up to the hardware concurrency + reported by the system. Changing the summation order can affect rounding. +* ``OPTIMIZED`` can choose a full calculation when it costs less than updating + changed sites. ``getEvaluatorTypeUsed`` then returns ``BASIC``. +* ``PDFCalculator::getQmax`` and ``getQstep`` now return ``double`` by value. + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* Separate calculators can run concurrently and share a read-only structure. +* ``DebyePDFCalculator`` uses each site's own occupancy when several sites + share an element. +* The ``OPTIMIZED`` evaluator recalculates when a change of adapter class + changes the structure, for example when a cluster becomes periodic. +* Copies of a calculator now have independent evaluators. + +**Security:** + +* diff --git a/src/diffpy/EventTicker.cpp b/src/diffpy/EventTicker.cpp index 68f3e70..cde56c8 100644 --- a/src/diffpy/EventTicker.cpp +++ b/src/diffpy/EventTicker.cpp @@ -20,6 +20,8 @@ * *****************************************************************************/ +#include + #include #include @@ -36,14 +38,7 @@ EventTicker::EventTicker() : mtick(0, 0) {} // Public Methods ------------------------------------------------------------ -void EventTicker::click() { - ++gtick.second; - if (0 >= gtick.second) { - ++gtick.first; - gtick.second = 0; - } - mtick = gtick; -} +void EventTicker::click() { mtick = advanceGlobalTick(); } void EventTicker::updateFrom(const EventTicker& other) { if (other > *this) mtick = other.mtick; @@ -75,9 +70,31 @@ bool EventTicker::operator!=(const EventTicker& other) const { return mtick != other.mtick; } -// Static Global Data -------------------------------------------------------- +// Private Static Methods ---------------------------------------------------- + +namespace { + +// Tickers click also while other files initialize their static objects, +// so the counter and its mutex must be initialized at compile time. +constinit std::mutex gtickmutex; +constinit EventTicker::value_type gtick(0, 0); -EventTicker::value_type EventTicker::gtick(0, 0); +} // namespace + +EventTicker::value_type EventTicker::advanceGlobalTick() { + const std::lock_guard lock(gtickmutex); + ++gtick.second; + if (0 >= gtick.second) { + ++gtick.first; + gtick.second = 0; + } + return gtick; +} + +EventTicker::value_type EventTicker::globalTick() { + const std::lock_guard lock(gtickmutex); + return gtick; +} } // namespace eventticker } // namespace diffpy diff --git a/src/diffpy/EventTicker.hpp b/src/diffpy/EventTicker.hpp index 67d7e15..82fadb3 100644 --- a/src/diffpy/EventTicker.hpp +++ b/src/diffpy/EventTicker.hpp @@ -41,6 +41,7 @@ class DLL_EXPORT EventTicker { // methods /// record an event and advance the global ticker. + /// Tickers of different threads may click at the same time. void click(); /// advance internal ticker if the other ticker is newer. void updateFrom(const EventTicker& other); @@ -56,8 +57,10 @@ class DLL_EXPORT EventTicker { bool operator!=(const EventTicker&) const; private: - // global counter - static value_type gtick; + /// Advance the global counter and return its new value. + static value_type advanceGlobalTick(); + /// Return the current value of the global counter. + static value_type globalTick(); // data value_type mtick; @@ -66,6 +69,7 @@ class DLL_EXPORT EventTicker { friend class boost::serialization::access; template void save(Archive& ar, const unsigned int version) const { + const value_type gtick = globalTick(); ar << mtick << gtick; } @@ -73,6 +77,7 @@ class DLL_EXPORT EventTicker { void load(Archive& ar, const unsigned int version) { value_type ga; ar >> mtick >> ga; + const value_type gtick = globalTick(); if (ga > gtick) { if (ga.first != gtick.first) mtick.first = mtick.second = 0; diff --git a/src/diffpy/HasClassRegistry.ipp b/src/diffpy/HasClassRegistry.ipp index f289873..47847f7 100644 --- a/src/diffpy/HasClassRegistry.ipp +++ b/src/diffpy/HasClassRegistry.ipp @@ -163,12 +163,8 @@ template typename HasClassRegistry::RegistryStorage& HasClassRegistry::getRegistry() { - static std::unique_ptr the_registry; - if (!the_registry.get()) - { - the_registry.reset(new RegistryStorage()); - } - return *the_registry; + static RegistryStorage the_registry; + return the_registry; } } // namespace diffpy diff --git a/src/diffpy/srreal/AtomicStructureAdapter.cpp b/src/diffpy/srreal/AtomicStructureAdapter.cpp index dec63d4..37134fd 100644 --- a/src/diffpy/srreal/AtomicStructureAdapter.cpp +++ b/src/diffpy/srreal/AtomicStructureAdapter.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -135,6 +136,18 @@ StructureDifference AtomicStructureAdapter::diff( typedef std::shared_ptr APtr; APtr pother = std::dynamic_pointer_cast(other); if (!pother) return sd; + // The same atoms can be another structure in a derived adapter, such as a + // crystal. Only its diff can tell, so it must allow the fast update too. + if (typeid(*this) != typeid(*pother)) { + thread_local bool reversed = false; + if (!reversed) { + struct Reversal { + Reversal() { reversed = true; } + ~Reversal() { reversed = false; } + } reversal; + if (!other->diff(this->shared_from_this()).allowsfastupdate()) return sd; + } + } // try fast side-by-side comparison sd.diffmethod = StructureDifference::Method::SIDEBYSIDE; const AtomicStructureAdapter& astru0 = *this; diff --git a/src/diffpy/srreal/AtomicStructureAdapter.hpp b/src/diffpy/srreal/AtomicStructureAdapter.hpp index 9402792..46719b0 100644 --- a/src/diffpy/srreal/AtomicStructureAdapter.hpp +++ b/src/diffpy/srreal/AtomicStructureAdapter.hpp @@ -20,6 +20,8 @@ #ifndef ATOMICSTRUCTUREADAPTER_HPP_INCLUDED #define ATOMICSTRUCTUREADAPTER_HPP_INCLUDED +#include + #include #include @@ -115,6 +117,7 @@ class DLL_EXPORT AtomicStructureAdapter : public StructureAdapter { void assign(Iter first, Iter last) { matoms.assign(first, last); } + void assign(AtomVector&& atoms) noexcept { matoms = std::move(atoms); } void assign(size_t n, const Atom& a) { matoms.assign(n, a); } // iterator forwarding iterator begin() { return matoms.begin(); } diff --git a/src/diffpy/srreal/BaseBondGenerator.cpp b/src/diffpy/srreal/BaseBondGenerator.cpp index 919dc64..cedd5f1 100644 --- a/src/diffpy/srreal/BaseBondGenerator.cpp +++ b/src/diffpy/srreal/BaseBondGenerator.cpp @@ -185,7 +185,8 @@ void BaseBondGenerator::updateDistance() { // Private Methods ----------------------------------------------------------- void BaseBondGenerator::advanceWhileInvalid() { - while (!this->finished() && (this->bondOutOfRange() || this->atSelfPair())) { + while (!this->finished() && + (this->bondOutOfRange() || (!mkeepcoincident && this->atSelfPair()))) { this->getNextBond(); } } diff --git a/src/diffpy/srreal/BaseBondGenerator.hpp b/src/diffpy/srreal/BaseBondGenerator.hpp index 62080af..7d20433 100644 --- a/src/diffpy/srreal/BaseBondGenerator.hpp +++ b/src/diffpy/srreal/BaseBondGenerator.hpp @@ -28,6 +28,10 @@ namespace diffpy { namespace srreal { +namespace kernels { +class PairTable; +} + /// Use zero default for rmax so any misconfiguration of r-limits is obvious. const double DEFAULT_BONDGENERATOR_RMAX = 0.0; @@ -94,6 +98,10 @@ class DLL_EXPORT BaseBondGenerator { void updateDistance(); private: + friend class kernels::PairTable; + // A cached candidate may move away from a coincident site on the next call. + bool mkeepcoincident = false; + // methods void advanceWhileInvalid(); bool bondOutOfRange() const; diff --git a/src/diffpy/srreal/BaseDebyeSum.cpp b/src/diffpy/srreal/BaseDebyeSum.cpp index ea32084..9d3c089 100644 --- a/src/diffpy/srreal/BaseDebyeSum.cpp +++ b/src/diffpy/srreal/BaseDebyeSum.cpp @@ -16,14 +16,19 @@ * *****************************************************************************/ +#include +#include +#include #include +#include +#include +#include #include -#include -#include -#include #include -#include +#include +#include +#include #include #include #include @@ -43,6 +48,11 @@ namespace { /// Default cutoff for the Q-decreasing scale of the sine contributions. const double DEFAULT_DEBYE_PRECISION = 1e-6; +std::span factorRow(const QuantityType& factors, int row, + int nq) { + return std::span(factors).subspan(std::size_t(row) * nq, nq); +} + } // namespace // Constructor --------------------------------------------------------------- @@ -139,17 +149,12 @@ void BaseDebyeSum::addPairContribution(const BaseBondGenerator& bnds, int summationscale) { const double dist = bnds.distance(); if (eps_eq(0.0, dist)) return; - // calculate sigma parameter for the Debye-Waller dampign Gaussian const double fwhm = this->getPeakWidthModel()->calculate(bnds); - const double fwhmtosigma = 1.0 / (2 * sqrt(2 * M_LN2)); const int nqpts = pdfutils_qmaxSteps(this); - const auto& sftypeatkq = mstructure_cache.sftypeatkq; - const auto& typeofsite = mstructure_cache.typeofsite; - const auto& sf0 = sftypeatkq[typeofsite[bnds.site0()]]; - const auto& sf1 = sftypeatkq[typeofsite[bnds.site1()]]; - assert(nqpts <= int(mvalue.size())); - assert(nqpts <= int(sf0.size())); - assert(nqpts <= int(sf1.size())); + const auto& cache = mstructure_cache; + const int t0 = cache.types.typeofsite[bnds.site0()]; + const int t1 = cache.types.typeofsite[bnds.site1()]; + if (!nqpts) return; kernels::activeCalculationKernel().debyeSum({ .out = mvalue.data(), @@ -157,14 +162,48 @@ void BaseDebyeSum::addPairContribution(const BaseBondGenerator& bnds, .kqhi = nqpts, .qstep = this->getQstep(), .dist = dist, - .dwsigma = fwhmtosigma * fwhm, + .dwsigma = kernels::fwhmToSigma(fwhm), .scale = double(summationscale * bnds.multiplicity()), .prec = this->getDebyePrecision(), - .sf0 = sf0.data(), - .sf1 = sf1.data(), + .sf0 = factorRow(cache.factors, cache.types.curve[t0], nqpts).data(), + .sf1 = factorRow(cache.factors, cache.types.curve[t1], nqpts).data(), + .occ0 = cache.types.occupancy[t0], + .occ1 = cache.types.occupancy[t1], }); } +bool BaseDebyeSum::addAllPairContributions() { + if (typeid(*this) != typeid(BaseDebyeSum)) return false; + return this->addAllPairsOfDebyeSum(); +} + +bool BaseDebyeSum::addAllPairsOfDebyeSum() { + // Other peak width models and structures need per-bond evaluation. + kernels::DebyePairSumArgs args = {}; + if (!kernels::assignPairWidth(args.w, *this->getPeakWidthModel()) || + !kernels::PairTable::supports(*mstructure)) + return false; + BaseBondGenerator range(mstructure); + this->configureBondGenerator(range); + const auto& cache = mstructure_cache; + const int nq = cache.nq; + args.typeofsite = cache.types.typeofsite.data(); + args.curve = cache.types.curve.data(); + args.occupancy = cache.types.occupancy.data(); + args.ntypes = int(cache.types.curve.size()); + args.ncurves = nq ? int(cache.factors.size() / nq) : 0; + args.nq = nq; + args.sf = cache.factors.data(); + args.kqlo = pdfutils_qminSteps(this); + args.kqhi = nq; + args.qstep = this->getQstep(); + args.prec = this->getDebyePrecision(); + const auto& kernel = + this->preparePairSum(args, range.getRmin(), range.getRmax()); + mallpairs.record(kernel.debyePairSum(args)); + return true; +} + void BaseDebyeSum::stashPartialValue() { mdbsumstash = this->value(); } void BaseDebyeSum::restorePartialValue() { @@ -185,58 +224,135 @@ double BaseDebyeSum::sfAverageAtkQ(int kq) const { } void BaseDebyeSum::cacheStructureData() { - using std::placeholders::_1; - int cntsites = this->countSites(); - const int nqpts = pdfutils_qmaxSteps(this); - QuantityType zeros(nqpts, 0.0); - unordered_map atomtypeidx; - // sftypeatkq - mstructure_cache.typeofsite.clear(); - mstructure_cache.typeofsite.reserve(cntsites); - mstructure_cache.sftypeatkq.clear(); - for (int siteidx = 0; siteidx < cntsites; ++siteidx) { - const string& smbl = mstructure->siteAtomType(siteidx); - if (!atomtypeidx.count(smbl)) { - atomtypeidx.insert(make_pair(smbl, int(atomtypeidx.size()))); + auto& c = mstructure_cache; + const int nq = pdfutils_qmaxSteps(this); + SiteTypes sites; + sites.typeofsite.resize(this->countSites()); + if (typeid(*this) == typeid(DebyePDFCalculator) || + typeid(*this) == typeid(BaseDebyeSum)) { + this->cacheElementFactors(sites, nq); + } else { + this->cacheCustomFactors(sites, nq); + } + c.types = std::move(sites); + c.nq = nq; + c.totaloccupancy = mstructure->totalOccupancy(); + c.sfaverageatkq.assign(c.nq, 0.0); + std::vector multiplicity(c.types.curve.size(), 0); + for (int i = 0; i < this->countSites(); ++i) { + multiplicity[c.types.typeofsite[i]] += mstructure->siteMultiplicity(i); + } + const int lo = pdfutils_qminSteps(this); + for (size_t t = 0; t < multiplicity.size(); ++t) { + const auto row = factorRow(c.factors, c.types.curve[t], c.nq); + for (int k = lo; k < c.nq; ++k) { + c.sfaverageatkq[k] += (row[k] * c.types.occupancy[t]) * multiplicity[t]; } - int tpidx = atomtypeidx[smbl]; - mstructure_cache.typeofsite.push_back(tpidx); - // do nothing if the type has been already cached - if (tpidx < int(mstructure_cache.sftypeatkq.size())) continue; - assert(tpidx == int(mstructure_cache.sftypeatkq.size())); - // here we need to build a new array - mstructure_cache.sftypeatkq.push_back(zeros); - QuantityType& sfarray = mstructure_cache.sftypeatkq.back(); - for (int kq = pdfutils_qminSteps(this); kq < nqpts; ++kq) { - double q = this->getQstep() * kq; - sfarray[kq] = this->sfSiteAtQ(siteidx, q); + } + if (eps_gt(c.totaloccupancy, 0.0)) { + const double scale = 1.0 / c.totaloccupancy; + for (double& f : c.sfaverageatkq) f *= scale; + } +} + +void BaseDebyeSum::cacheElementFactors(SiteTypes& sites, int nq) { + auto& c = mstructure_cache; + std::map types; + std::map elements; + const auto* pdf = dynamic_cast(this); + for (int i = 0; i < this->countSites(); ++i) { + const auto& symbol = mstructure->siteAtomType(i); + const double occ = pdf ? mstructure->siteOccupancy(i) : 1.0; + auto [t, added] = + types.try_emplace({symbol, bit_cast(occ)}, int(types.size())); + sites.typeofsite[i] = t->second; + if (added) { + auto el = elements.try_emplace(symbol, int(elements.size())).first; + sites.curve.push_back(el->second); + sites.occupancy.push_back(occ); } } - assert(cntsites == int(mstructure_cache.typeofsite.size())); - assert(atomtypeidx.size() == mstructure_cache.sftypeatkq.size()); - // totaloccupancy - mstructure_cache.totaloccupancy = mstructure->totalOccupancy(); - // sfaverageatkq - QuantityType& sfak = mstructure_cache.sfaverageatkq; - sfak = zeros; - int ntps = mstructure_cache.sftypeatkq.size(); - vector tpmultipl(ntps, 0); - for (int siteidx = 0; siteidx < cntsites; ++siteidx) { - int tpidx = mstructure_cache.typeofsite[siteidx]; - assert(tpidx < ntps); - tpmultipl[tpidx] += mstructure->siteMultiplicity(siteidx); + const auto tick = this->ticker(); + if (elements == melements && tick == msfticker && + c.factors.size() == elements.size() * nq) + return; + QuantityType factors(elements.size() * nq); + const int lo = pdfutils_qminSteps(this); + for (const auto& [symbol, row] : elements) { + for (int k = lo; k < nq; ++k) { + factors[size_t(row) * nq + k] = + pdf ? pdf->getScatteringFactorTable()->lookup(symbol, + k * this->getQstep()) + : 1.0; + } } - for (int tpidx = 0; tpidx < ntps; ++tpidx) { - QuantityType& sfarray = mstructure_cache.sftypeatkq[tpidx]; - const int multipl = tpmultipl[tpidx]; - for (int kq = pdfutils_qminSteps(this); kq < nqpts; ++kq) { - sfak[kq] += sfarray[kq] * multipl; + c.factors.swap(factors); + melements.swap(elements); + msfticker = tick; +} + +void BaseDebyeSum::cacheCustomFactors(SiteTypes& sites, int nq) { + auto& c = mstructure_cache; + std::map types; + QuantityType factors, row(nq); + const bool comparePrevious = nq == c.nq && this->ticker() == msfticker; + const int lo = pdfutils_qminSteps(this); + bool changed = mcustomAmbiguous; + bool ambiguous = false; + for (int i = 0; i < this->countSites(); ++i) { + SiteKey key{mstructure->siteAtomType(i), + bit_cast(mstructure->siteOccupancy(i))}; + for (int k = lo; k < nq; ++k) + row[k] = this->sfSiteAtQ(i, k * this->getQstep()); + auto old = mcustomTypes.find(key); + if (comparePrevious && old != mcustomTypes.end() && + !std::ranges::equal(row, factorRow(c.factors, old->second, nq))) + changed = true; + const int next = int(sites.curve.size()); + auto [t, added] = types.try_emplace(std::move(key), next); + int id = t->second; + if (!added && !std::ranges::equal(row, factorRow(factors, id, nq))) { + id = next; + ambiguous = true; + } + if (id == next) { + factors.insert(factors.end(), row.begin(), row.end()); + sites.curve.push_back(id); + sites.occupancy.push_back(1.0); } + sites.typeofsite[i] = id; } - const double& totocc = mstructure_cache.totaloccupancy; - const double tosc = eps_gt(totocc, 0.0) ? (1.0 / totocc) : 1.0; - transform(sfak.begin(), sfak.end(), sfak.begin(), - bind(multiplies(), tosc, _1)); + if (changed || ambiguous) mticker.click(); + msfticker = this->ticker(); + mcustomAmbiguous = ambiguous; + mcustomTypes.swap(types); + c.factors.swap(factors); +} + +std::vector BaseDebyeSum::archiveFactors() const { + const auto& c = mstructure_cache; + std::vector rows(c.types.curve.size(), QuantityType(c.nq)); + for (size_t t = 0; t < rows.size(); ++t) { + const auto row = factorRow(c.factors, c.types.curve[t], c.nq); + std::ranges::transform(row, rows[t].begin(), + [&](double f) { return f * c.types.occupancy[t]; }); + } + return rows; +} + +void BaseDebyeSum::loadFactors(const std::vector& rows) { + auto& c = mstructure_cache; + c.nq = rows.empty() ? 0 : int(rows.front().size()); + c.factors.clear(); + c.types.curve.resize(rows.size()); + c.types.occupancy.assign(rows.size(), 1.0); + for (size_t t = 0; t < rows.size(); ++t) { + c.types.curve[t] = int(t); + c.factors.insert(c.factors.end(), rows[t].begin(), rows[t].end()); + } + melements.clear(); + mcustomTypes.clear(); + mcustomAmbiguous = false; } } // namespace srreal diff --git a/src/diffpy/srreal/BaseDebyeSum.hpp b/src/diffpy/srreal/BaseDebyeSum.hpp index b5cd11a..97a4204 100644 --- a/src/diffpy/srreal/BaseDebyeSum.hpp +++ b/src/diffpy/srreal/BaseDebyeSum.hpp @@ -19,6 +19,11 @@ #ifndef BASEDEBYESUM_HPP_INCLUDED #define BASEDEBYESUM_HPP_INCLUDED +#include +#include +#include +#include + #include #include #include @@ -62,6 +67,10 @@ class DLL_EXPORT BaseDebyeSum : public PairQuantity, // PairQuantity overloads virtual void resetValue(); virtual void addPairContribution(const BaseBondGenerator&, int); + virtual bool addAllPairContributions(); + /// addAllPairContributions for derived classes that keep the + /// addPairContribution of this class. + bool addAllPairsOfDebyeSum(); // support for PQEvaluatorOptimized virtual void stashPartialValue(); virtual void restorePartialValue(); @@ -81,13 +90,31 @@ class DLL_EXPORT BaseDebyeSum : public PairQuantity, double mqmax; double mqstep; double mdebyeprecision; - struct { + struct SiteTypes { std::vector typeofsite; - std::vector sftypeatkq; + // A scattering type names an elemental curve and a site occupancy. + std::vector curve; + std::vector occupancy; + }; + struct { + SiteTypes types; + QuantityType factors; // consecutive curves on the Q grid + int nq = 0; QuantityType sfaverageatkq; double totaloccupancy; } mstructure_cache; QuantityType mdbsumstash; + std::map melements; + eventticker::EventTicker msfticker; + + using SiteKey = std::pair; + // Representative curves identify changes to virtual, site-specific factors. + std::map mcustomTypes; + bool mcustomAmbiguous = false; + void cacheCustomFactors(SiteTypes&, int nq); + void cacheElementFactors(SiteTypes&, int nq); + std::vector archiveFactors() const; + void loadFactors(const std::vector& curves); // serialization friend class boost::serialization::access; @@ -100,8 +127,12 @@ class DLL_EXPORT BaseDebyeSum : public PairQuantity, ar & mqmax; ar & mqstep; ar & mdebyeprecision; - ar & mstructure_cache.typeofsite; - ar & mstructure_cache.sftypeatkq; + ar & mstructure_cache.types.typeofsite; + // Preserve the archive representation used before elemental curves. + std::vector curves; + if constexpr (Archive::is_saving::value) curves = this->archiveFactors(); + ar & curves; + if constexpr (Archive::is_loading::value) this->loadFactors(curves); ar & mstructure_cache.sfaverageatkq; ar & mstructure_cache.totaloccupancy; } diff --git a/src/diffpy/srreal/ConstantPeakWidth.cpp b/src/diffpy/srreal/ConstantPeakWidth.cpp index 2526da8..1659a63 100644 --- a/src/diffpy/srreal/ConstantPeakWidth.cpp +++ b/src/diffpy/srreal/ConstantPeakWidth.cpp @@ -82,6 +82,10 @@ const string& ConstantPeakWidth::type() const { return rv; } +double ConstantPeakWidth::calculateFromMSD(double r, double msd) const { + return this->getWidth(); +} + double ConstantPeakWidth::calculate(const BaseBondGenerator& bnds) const { return this->getWidth(); } diff --git a/src/diffpy/srreal/ConstantPeakWidth.hpp b/src/diffpy/srreal/ConstantPeakWidth.hpp index a991033..da78765 100644 --- a/src/diffpy/srreal/ConstantPeakWidth.hpp +++ b/src/diffpy/srreal/ConstantPeakWidth.hpp @@ -36,6 +36,9 @@ class DLL_EXPORT ConstantPeakWidth : public PeakWidthModel { // methods virtual const std::string& type() const; virtual double calculate(const BaseBondGenerator&) const; + /// Width of this model for a bond of the distance r with the mean square + /// displacement msd. This model uses neither. + double calculateFromMSD(double r, double msd) const; virtual double maxWidth(StructureAdapterPtr, double rmin, double rmax) const; // data access diff --git a/src/diffpy/srreal/CrystalStructureAdapter.cpp b/src/diffpy/srreal/CrystalStructureAdapter.cpp index 38a5a79..9d9cdf0 100644 --- a/src/diffpy/srreal/CrystalStructureAdapter.cpp +++ b/src/diffpy/srreal/CrystalStructureAdapter.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -65,6 +66,13 @@ bool sameLatticeBits(const Lattice& L0, const Lattice& L1) { return sameBits(L0.base().data(), L1.base().data()); } +/// Guards the symmetry positions of all adapters, which const methods +/// expand on demand, also in threads that share an adapter. +std::mutex& symmetryCacheMutex() { + static std::mutex mutex; + return mutex; +} + } // namespace ////////////////////////////////////////////////////////////////////////////// @@ -81,6 +89,7 @@ CrystalStructureAdapter::CrystalStructureAdapter() // Public Methods ------------------------------------------------------------ StructureAdapterPtr CrystalStructureAdapter::clone() const { + const std::lock_guard lock(symmetryCacheMutex()); StructureAdapterPtr rv(new CrystalStructureAdapter(*this)); return rv; } @@ -151,6 +160,7 @@ const SymOpRotTrans& CrystalStructureAdapter::getSymOp(int i) const { const CrystalStructureAdapter::AtomVector& CrystalStructureAdapter::getEquivalentAtoms(int idx) const { assert(0 <= idx && idx < this->countSites()); + const std::lock_guard lock(symmetryCacheMutex()); if (!this->isSiteSymmetryUpToDate(idx)) this->updateSymmetryPositions(); return msymatoms[idx]; } @@ -204,21 +214,24 @@ CrystalStructureAdapter::AtomVector CrystalStructureAdapter::expandLatticeAtom( } void CrystalStructureAdapter::updateSymmetryPositions() const { - // build asymmetric unit in lattice coordinates - AtomVector lcatoms(this->begin(), this->end()); - AtomVector::iterator lcai = lcatoms.begin(); - for (; lcai != lcatoms.end(); ++lcai) this->toFractional(*lcai); - // build symmetry positions for all atoms in the asymmetric unit - msymatoms.resize(this->countSites()); - assert(lcatoms.size() == msymatoms.size()); - lcai = lcatoms.begin(); - std::vector::iterator saii = msymatoms.begin(); - for (; lcai != lcatoms.end(); ++lcai, ++saii) { - *saii = this->expandLatticeAtom(*lcai); - iterator ai = saii->begin(); - for (; ai != saii->end(); ++ai) this->toCartesian(*ai); + // Expand only the sites whose atoms changed, or all of them after a change + // of the lattice or a reset of msymmetry_cached. + const int cntsites = this->countSites(); + const bool keepsame = this->isSymmetryCached() && + msymsource.size() == msymatoms.size() && + sameLatticeBits(msymlattice, this->getLattice()); + msymatoms.resize(cntsites); + if (!keepsame) msymsource.assign(cntsites, Atom()); + for (int i = 0; i < cntsites; ++i) { + const Atom& a = (*this)[i]; + if (keepsame && sameAtomBits(msymsource[i], a)) continue; + // expand the atom in lattice coordinates + Atom lca = a; + this->toFractional(lca); + msymatoms[i] = this->expandLatticeAtom(lca); + for (Atom& ai : msymatoms[i]) this->toCartesian(ai); + msymsource[i] = a; } - msymsource.assign(this->begin(), this->end()); msymlattice = this->getLattice(); msymmetry_cached = true; } @@ -272,6 +285,7 @@ bool CrystalStructureAdapter::isSymmetryUpToDate() const { const std::vector& CrystalStructureAdapter::symmetryAtoms() const { + const std::lock_guard lock(symmetryCacheMutex()); if (!this->isSymmetryUpToDate()) this->updateSymmetryPositions(); return msymatoms; } diff --git a/src/diffpy/srreal/DebyePDFCalculator.cpp b/src/diffpy/srreal/DebyePDFCalculator.cpp index b2f6300..364c04f 100644 --- a/src/diffpy/srreal/DebyePDFCalculator.cpp +++ b/src/diffpy/srreal/DebyePDFCalculator.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -231,6 +232,11 @@ void DebyePDFCalculator::configureBondGenerator(BaseBondGenerator& bnds) const { bnds.setRmax(this->rcalchi()); } +bool DebyePDFCalculator::addAllPairContributions() { + if (typeid(*this) != typeid(DebyePDFCalculator)) return false; + return this->addAllPairsOfDebyeSum(); +} + double DebyePDFCalculator::sfSiteAtQ(int siteidx, const double& Q) const { const ScatteringFactorTablePtr& sftable = this->getScatteringFactorTable(); const string& smbl = mstructure->siteAtomType(siteidx); diff --git a/src/diffpy/srreal/DebyePDFCalculator.hpp b/src/diffpy/srreal/DebyePDFCalculator.hpp index 0162fd6..2e3771d 100644 --- a/src/diffpy/srreal/DebyePDFCalculator.hpp +++ b/src/diffpy/srreal/DebyePDFCalculator.hpp @@ -104,6 +104,7 @@ class DLL_EXPORT DebyePDFCalculator : public BaseDebyeSum, virtual void resetValue(); virtual void configureBondGenerator(BaseBondGenerator&) const; virtual double sfSiteAtQ(int, const double& Q) const; + virtual bool addAllPairContributions(); private: // methods diff --git a/src/diffpy/srreal/DebyeWallerPeakWidth.cpp b/src/diffpy/srreal/DebyeWallerPeakWidth.cpp index e38db3a..b9d4fa2 100644 --- a/src/diffpy/srreal/DebyeWallerPeakWidth.cpp +++ b/src/diffpy/srreal/DebyeWallerPeakWidth.cpp @@ -19,6 +19,7 @@ #include #include +#include #include namespace diffpy { @@ -46,10 +47,11 @@ const string& DebyeWallerPeakWidth::type() const { } double DebyeWallerPeakWidth::calculate(const BaseBondGenerator& bnds) const { - using diffpy::mathutils::GAUSS_SIGMA_TO_FWHM; - double msdval = bnds.msd(); - double rv = (msdval < 0.0) ? 0.0 : GAUSS_SIGMA_TO_FWHM * sqrt(msdval); - return rv; + return this->calculateFromMSD(bnds.distance(), bnds.msd()); +} + +double DebyeWallerPeakWidth::calculateFromMSD(double r, double msdval) const { + return kernels::debyeWallerWidth(msdval); } double DebyeWallerPeakWidth::maxWidth(StructureAdapterPtr stru, double rmin, diff --git a/src/diffpy/srreal/DebyeWallerPeakWidth.hpp b/src/diffpy/srreal/DebyeWallerPeakWidth.hpp index ec98b3d..3e8c7d5 100644 --- a/src/diffpy/srreal/DebyeWallerPeakWidth.hpp +++ b/src/diffpy/srreal/DebyeWallerPeakWidth.hpp @@ -36,6 +36,9 @@ class DLL_EXPORT DebyeWallerPeakWidth : public PeakWidthModel { // methods virtual const std::string& type() const; virtual double calculate(const BaseBondGenerator&) const; + /// Width of this model for a bond of the distance r with the mean square + /// displacement msd. This model does not use r. + virtual double calculateFromMSD(double r, double msd) const; virtual double maxWidth(StructureAdapterPtr, double rmin, double rmax) const; private: diff --git a/src/diffpy/srreal/JeongPeakWidth.cpp b/src/diffpy/srreal/JeongPeakWidth.cpp index 0d4466d..cf79f62 100644 --- a/src/diffpy/srreal/JeongPeakWidth.cpp +++ b/src/diffpy/srreal/JeongPeakWidth.cpp @@ -19,7 +19,7 @@ *****************************************************************************/ #include -#include +#include #include namespace diffpy { @@ -63,11 +63,17 @@ double JeongPeakWidth::calculate(const BaseBondGenerator& bnds) const { double r = bnds.distance(); double corr = this->msdSharpeningRatio(r); // avoid calculating square root of negative value - double fwhm = (corr <= 0) - ? 0.0 - : (sqrt(corr) * this->DebyeWallerPeakWidth::calculate(bnds) + - pow(this->getQbroad_seperable() * r, 2)); - return fwhm; + // The bond calculates its displacement only when it is needed. + return (corr <= 0) + ? 0.0 + : kernels::sharpenedWidth(r, corr, bnds.msd(), mqbroad_seperable); +} + +double JeongPeakWidth::calculateFromMSD(double r, double msdval) const { + double corr = this->msdSharpeningRatio(r); + return (corr <= 0) + ? 0.0 + : kernels::sharpenedWidth(r, corr, msdval, mqbroad_seperable); } double JeongPeakWidth::maxWidth(StructureAdapterPtr stru, double rmin, @@ -114,12 +120,7 @@ void JeongPeakWidth::setQbroad(double qbroad) { // Private Methods ----------------------------------------------------------- double JeongPeakWidth::msdSharpeningRatio(const double& r) const { - using diffpy::mathutils::DOUBLE_EPS; - // avoid division by zero - if (r < DOUBLE_EPS) return 0.0; - double rv = 1.0 - this->getDelta1() / r - this->getDelta2() / pow(r, 2) + - pow(this->getQbroad() * r, 2); - return rv; + return kernels::sharpeningRatio(r, mdelta1, mdelta2, mqbroad); } // Registration -------------------------------------------------------------- diff --git a/src/diffpy/srreal/JeongPeakWidth.hpp b/src/diffpy/srreal/JeongPeakWidth.hpp index 021b76c..d1d88dd 100644 --- a/src/diffpy/srreal/JeongPeakWidth.hpp +++ b/src/diffpy/srreal/JeongPeakWidth.hpp @@ -38,6 +38,9 @@ class DLL_EXPORT JeongPeakWidth : public DebyeWallerPeakWidth { // methods virtual const std::string& type() const; virtual double calculate(const BaseBondGenerator&) const; + /// Width of this model for a bond of the distance r with the mean square + /// displacement msd. + double calculateFromMSD(double r, double msd) const override; virtual double maxWidth(StructureAdapterPtr, double rmin, double rmax) const; // data access diff --git a/src/diffpy/srreal/PDFCalculator.cpp b/src/diffpy/srreal/PDFCalculator.cpp index d464349..791f711 100644 --- a/src/diffpy/srreal/PDFCalculator.cpp +++ b/src/diffpy/srreal/PDFCalculator.cpp @@ -16,13 +16,19 @@ * *****************************************************************************/ +#include #include #include #include #include +#include #include #include +#include +#include +#include +#include #include #include #include @@ -245,19 +251,15 @@ void PDFCalculator::setQmax(double qmax) { if (_initialQstepUpdate(this)) this->resetValue(); } -const double& PDFCalculator::getQmax() const { - static double rv; - rv = min(mqmax, M_PI / this->getRstep()); - return rv; +double PDFCalculator::getQmax() const { + return min(mqmax, M_PI / this->getRstep()); } -const double& PDFCalculator::getQstep() const { - static double rv; +double PDFCalculator::getQstep() const { // replicate the zero padding as done in fftgtof - int Npad1 = this->extendedRmaxSteps(); - int Npad2 = (Npad1 > 0) ? (1 << int(ceil(log2(Npad1)))) : 0; - rv = (Npad2 > 0) ? M_PI / (Npad2 * this->getRstep()) : 0.0; - return rv; + const int n = this->extendedRmaxSteps(); + if (n <= 0) return 0.0; + return M_PI / (std::bit_ceil(std::size_t(n)) * this->getRstep()); } // R-range methods @@ -425,16 +427,35 @@ void PDFCalculator::addPairContribution(const BaseBondGenerator& bnds, double sfprod = this->sfSite(bnds.site0()) * this->sfSite(bnds.site1()); double peakscale = sfprod * bnds.multiplicity() * summationscale; double fwhm = this->getPeakWidthModel()->calculate(bnds); - const PeakProfile& pkf = *(this->getPeakProfile()); - double dist = bnds.distance(); - double xlo = dist + pkf.xboundlo(fwhm); - double xhi = dist + pkf.xboundhi(fwhm); - int i = max(0, this->calcIndex(xlo)); - int ilast = min(this->countCalcPoints(), this->calcIndex(xhi) + 1); - assert(ilast <= int(mvalue.size())); - assert(eps_gt(dist, 0.0)); - pkf.addRDFPeak(mvalue.data(), i, ilast, this->rcalcloSteps(), - this->getRstep(), dist, fwhm, peakscale); + kernels::addProfilePeak(*this->getPeakProfile(), mvalue.data(), + this->countCalcPoints(), this->rcalcloSteps(), + this->getRstep(), bnds.distance(), fwhm, peakscale); +} + +bool PDFCalculator::addAllPairContributions() { + if (typeid(*this) != typeid(PDFCalculator)) return false; + return this->addAllPairsOfPDF(); +} + +bool PDFCalculator::addAllPairsOfPDF() { + // Other peak width models and structures need per-bond evaluation. + kernels::PDFPairSumArgs args = {}; + if (!kernels::assignPairWidth(args.w, *this->getPeakWidthModel()) || + !kernels::PairTable::supports(*mstructure)) + return false; + const PeakProfile& pkf = *this->getPeakProfile(); + args.cropped = typeid(pkf) == typeid(CroppedGaussianProfile); + if (!args.cropped && typeid(pkf) != typeid(GaussianProfile)) return false; + args.height = pkf(0.0, 1.0); + args.halfboundrel = pkf.xboundhi(1.0); + args.sfsite = mstructure_cache.sfsite.data(); + args.npts = this->countCalcPoints(); + args.rlosteps = this->rcalcloSteps(); + args.rstep = this->getRstep(); + const auto& kernel = + this->preparePairSum(args, this->rcalclo(), this->rcalchi()); + mallpairs.record(kernel.pdfPairSum(args)); + return true; } void PDFCalculator::stashPartialValue() { @@ -519,11 +540,6 @@ int PDFCalculator::countCalcPoints() const { return rv; } -int PDFCalculator::calcIndex(double r) const { - int rv = int(floor(r / this->getRstep())) - this->rcalcloSteps(); - return rv; -} - void PDFCalculator::cutRipplePoints(QuantityType& y) const { if (y.empty()) return; assert(int(y.size()) == diff --git a/src/diffpy/srreal/PDFCalculator.hpp b/src/diffpy/srreal/PDFCalculator.hpp index ac89a8f..77828e9 100644 --- a/src/diffpy/srreal/PDFCalculator.hpp +++ b/src/diffpy/srreal/PDFCalculator.hpp @@ -65,8 +65,8 @@ class DLL_EXPORT PDFCalculator : public PairQuantity, void setQmin(double); const double& getQmin() const; void setQmax(double); - const double& getQmax() const; - const double& getQstep() const; + double getQmax() const; + double getQstep() const; // R-range methods QuantityType getRgrid() const; @@ -110,6 +110,10 @@ class DLL_EXPORT PDFCalculator : public PairQuantity, virtual void resetValue(); virtual void configureBondGenerator(BaseBondGenerator&) const; virtual void addPairContribution(const BaseBondGenerator&, int); + virtual bool addAllPairContributions(); + /// addAllPairContributions for derived classes that keep the + /// addPairContribution of this class. + bool addAllPairsOfPDF(); // support for PQEvaluatorOptimized virtual void stashPartialValue(); virtual void restorePartialValue(); @@ -136,8 +140,6 @@ class DLL_EXPORT PDFCalculator : public PairQuantity, int countExtendedPoints() const; /// number of points in the complete calculated r-grid int countCalcPoints() const; - /// index of a nearby point in the complete calculated r-grid - int calcIndex(double r) const; /// reduce extended grid to user-requested results grid /// by cutting away the points for termination ripples void cutRipplePoints(QuantityType& y) const; diff --git a/src/diffpy/srreal/PQEvaluator.cpp b/src/diffpy/srreal/PQEvaluator.cpp index aaa567b..4d676d0 100644 --- a/src/diffpy/srreal/PQEvaluator.cpp +++ b/src/diffpy/srreal/PQEvaluator.cpp @@ -28,6 +28,7 @@ #include #include #include +#include using namespace std; @@ -41,6 +42,15 @@ namespace { // tolerated load variance for splitting outer loop for parallel evaluation const double CPU_LOAD_VARIANCE = 0.1; +// Work of a sum of all pairs for each of its pairs and for each of the +// contributions that it merges them to, in the units of the work for a +// pair that is added by itself. +const double ALLPAIRS_PAIR_WORK = 0.2; +const double ALLPAIRS_PEAK_WORK = 0.85; +// The estimate is within about 15 %, so the sum of all pairs must take at +// most this share of the work of the fast update. +const double ALLPAIRS_WORK_MARGIN = 0.85; + SiteIndices complementary_indices(const int sz, const SiteIndices& indices0) { SiteIndices rv; rv.reserve(sz); @@ -73,9 +83,17 @@ void PQEvaluatorBasic::validate(PairQuantity& pq) const { return; } void PQEvaluatorBasic::updateValue(PairQuantity& pq, StructureAdapterPtr stru) { mtypeused = BASIC; + maddedallpairs = false; // The value is not valid until this update completes. mvalue_ticker = eventticker::EventTicker(); pq.setStructure(stru); + // A batch covers the complete half sum of an unpartitioned evaluation. + const bool halfsum = !this->getFlag(USEFULLSUM) && !this->isParallel(); + if (halfsum && pq.addAllPairContributions()) { + maddedallpairs = true; + mvalue_ticker.click(); + return; + } BaseBondGeneratorPtr bnds = pq.mstructure->createBondGenerator(); pq.configureBondGenerator(*bnds); int cntsites = pq.mstructure->countSites(); @@ -141,9 +159,8 @@ void PQEvaluatorOptimized::validate(PairQuantity& pq) const { void PQEvaluatorOptimized::updateValue(PairQuantity& pq, StructureAdapterPtr stru) { mtypeused = OPTIMIZED; - // revert to normal calculation if there is no structure or - // if PairQuantity uses mask - if (pq.ticker() >= mvalue_ticker || !mlast_structure) { + if (pq.ticker() >= mvalue_ticker || !mlast_structure || + mlastkernel != &kernels::activeCalculationKernel()) { return this->updateValueCompletely(pq, stru); } // do not do fast updates if they take more work @@ -155,47 +172,52 @@ void PQEvaluatorOptimized::updateValue(PairQuantity& pq, sd.diffmethod != StructureDifference::Method::SIDEBYSIDE) { return this->updateValueCompletely(pq, stru); } + if (this->allPairsTakeLessWork(pq, sd)) { + return this->updateValueCompletely(pq, stru); + } // Remove contributions from the extra sites in the old structure assert(sd.stru0 == mlast_structure); // The value belongs to no structure until this update completes. + const StructureAdapterPtr last_structure = mlast_structure; mlast_structure.reset(); - int cntsites0 = sd.stru0->countSites(); - BaseBondGeneratorPtr bnds0 = sd.stru0->createBondGenerator(); - pq.configureBondGenerator(*bnds0); // loop counter long n = mcpuindex; bool usefullsum = this->getFlag(USEFULLSUM); - // the loop is adjusted according to usefullsum and split within - // the outer loop in case of parallel evaluation. - SiteIndices anchors = sd.pop0; - SiteIndices unchanged; + const bool hasmask = pq.hasMask(); + // A structure of the same content has no sites to remove or to add. + // It needs no bond generators, which copy all positions of a crystal. if (!sd.pop0.empty()) { - unchanged = complementary_indices(cntsites0, sd.pop0); + int cntsites0 = sd.stru0->countSites(); + BaseBondGeneratorPtr bnds0 = sd.stru0->createBondGenerator(); + pq.configureBondGenerator(*bnds0); + // the loop is adjusted according to usefullsum and split within + // the outer loop in case of parallel evaluation. + SiteIndices anchors = sd.pop0; + SiteIndices unchanged = complementary_indices(cntsites0, sd.pop0); anchors.insert(anchors.end(), unchanged.begin(), unchanged.end()); - } - bnds0->selectSites(anchors.begin(), anchors.end()); - SiteIndices::const_iterator last_anchor = - usefullsum ? anchors.end() : (anchors.begin() + sd.pop0.size()); - SiteIndices::const_iterator ii0; - bool needsreselection = usefullsum; - const bool hasmask = pq.hasMask(); - for (ii0 = anchors.begin(); ii0 != last_anchor; ++ii0) { - if (n++ % mncpu) continue; - const int& i0 = *ii0; - bnds0->selectAnchorSite(i0); - // when using half sum, deselect visited popped sites - if (!usefullsum) bnds0->selectSites(ii0, anchors.end()); - // when using full sum, select only the popped sites when - // anchored at an unchanged atom - else if (needsreselection && ii0 >= (anchors.begin() + sd.pop0.size())) { - bnds0->selectSites(sd.pop0.begin(), sd.pop0.end()); - needsreselection = false; - } - for (bnds0->rewind(); !bnds0->finished(); bnds0->next()) { - int i1 = bnds0->site1(); - if (hasmask && !pq.getPairMask(i0, i1)) continue; - const int summationscale = (usefullsum || i0 == i1) ? -1 : -2; - pq.addPairContribution(*bnds0, summationscale); + bnds0->selectSites(anchors.begin(), anchors.end()); + SiteIndices::const_iterator last_anchor = + usefullsum ? anchors.end() : (anchors.begin() + sd.pop0.size()); + SiteIndices::const_iterator ii0; + bool needsreselection = usefullsum; + for (ii0 = anchors.begin(); ii0 != last_anchor; ++ii0) { + if (n++ % mncpu) continue; + const int& i0 = *ii0; + bnds0->selectAnchorSite(i0); + // when using half sum, deselect visited popped sites + if (!usefullsum) bnds0->selectSites(ii0, anchors.end()); + // when using full sum, select only the popped sites when + // anchored at an unchanged atom + else if (needsreselection && ii0 >= (anchors.begin() + sd.pop0.size())) { + bnds0->selectSites(sd.pop0.begin(), sd.pop0.end()); + needsreselection = false; + } + for (bnds0->rewind(); !bnds0->finished(); bnds0->next()) { + int i1 = bnds0->site1(); + if (hasmask && !pq.getPairMask(i0, i1)) continue; + const int summationscale = (usefullsum || i0 == i1) ? -1 : -2; + pq.addPairContribution(*bnds0, summationscale); + } } } // Add contributions from the new atoms in the updated structure @@ -210,47 +232,75 @@ void PQEvaluatorOptimized::updateValue(PairQuantity& pq, return this->updateValueCompletely(pq, stru); } pq.restorePartialValue(); - int cntsites1 = sd.stru1->countSites(); - BaseBondGeneratorPtr bnds1 = sd.stru1->createBondGenerator(); - pq.configureBondGenerator(*bnds1); - anchors = sd.add1; - unchanged.clear(); if (!sd.add1.empty()) { - unchanged = complementary_indices(cntsites1, sd.add1); + int cntsites1 = sd.stru1->countSites(); + BaseBondGeneratorPtr bnds1 = sd.stru1->createBondGenerator(); + pq.configureBondGenerator(*bnds1); + SiteIndices anchors = sd.add1; + SiteIndices unchanged = complementary_indices(cntsites1, sd.add1); anchors.insert(anchors.begin(), unchanged.begin(), unchanged.end()); - } - bnds1->selectSites(sd.add1.begin(), sd.add1.end()); - SiteIndices::const_iterator first_anchor = - usefullsum ? anchors.begin() : (anchors.end() - sd.add1.size()); - SiteIndices::const_iterator ii1; - needsreselection = usefullsum; - for (ii1 = first_anchor; ii1 != anchors.end(); ++ii1) { - if (n++ % mncpu) continue; - const int& i0 = *ii1; - bnds1->selectAnchorSite(i0); - // when using half sum, activate the added site - if (!usefullsum) bnds1->selectSites(anchors.begin(), ii1 + 1); - // when using full sum select unchanged atoms once anchored - // at an added atom. - else if (needsreselection && ii1 >= (anchors.end() - sd.add1.size())) { - bnds1->selectSites(anchors.begin(), anchors.end()); - needsreselection = false; - } - for (bnds1->rewind(); !bnds1->finished(); bnds1->next()) { - int i1 = bnds1->site1(); - if (hasmask && !pq.getPairMask(i0, i1)) continue; - const int summationscale = (usefullsum || i0 == i1) ? +1 : +2; - pq.addPairContribution(*bnds1, summationscale); + bnds1->selectSites(sd.add1.begin(), sd.add1.end()); + SiteIndices::const_iterator first_anchor = + usefullsum ? anchors.begin() : (anchors.end() - sd.add1.size()); + SiteIndices::const_iterator ii1; + bool needsreselection = usefullsum; + for (ii1 = first_anchor; ii1 != anchors.end(); ++ii1) { + if (n++ % mncpu) continue; + const int& i0 = *ii1; + bnds1->selectAnchorSite(i0); + // when using half sum, activate the added site + if (!usefullsum) bnds1->selectSites(anchors.begin(), ii1 + 1); + // when using full sum select unchanged atoms once anchored + // at an added atom. + else if (needsreselection && ii1 >= (anchors.end() - sd.add1.size())) { + bnds1->selectSites(anchors.begin(), anchors.end()); + needsreselection = false; + } + for (bnds1->rewind(); !bnds1->finished(); bnds1->next()) { + int i1 = bnds1->site1(); + if (hasmask && !pq.getPairMask(i0, i1)) continue; + const int summationscale = (usefullsum || i0 == i1) ? +1 : +2; + pq.addPairContribution(*bnds1, summationscale); + } } } - mlast_structure = pq.getStructure()->clone(); + // The saved geometry must use the site order of the new scattering arrays. + // A sorted comparison can report empty differences after a permutation. + const bool sameorder = + sd.diffmethod == StructureDifference::Method::SIDEBYSIDE && + sd.pop0.empty() && sd.add1.empty(); + mlast_structure = sameorder ? last_structure : pq.getStructure()->clone(); mvalue_ticker.click(); } void PQEvaluatorOptimized::updateValueCompletely(PairQuantity& pq, StructureAdapterPtr stru) { + mlast_structure.reset(); this->PQEvaluatorBasic::updateValue(pq, stru); mlast_structure = pq.getStructure()->clone(); + mlastkernel = &kernels::activeCalculationKernel(); +} + +// True when a sum of all pairs is expected to take less work than the fast +// update of the changed sites. + +bool PQEvaluatorOptimized::allPairsTakeLessWork( + const PairQuantity& pq, const StructureDifference& sd) const { + // A change of whether the calculator adds all pairs forces a full + // calculation anyway. + if (!maddedallpairs) return false; + // share of the pairs that have one of the sites + auto pairshare = [](const SiteIndices& sites, const StructureAdapter& s) { + if (sites.empty()) return 0.0; + const double others = 1.0 - double(sites.size()) / s.countSites(); + return 1.0 - others * others; + }; + const double pairs = pq.mallpairs.pairs; + const double fastwork = + (pairshare(sd.pop0, *sd.stru0) + pairshare(sd.add1, *sd.stru1)) * pairs; + const double fullwork = + ALLPAIRS_PAIR_WORK * pairs + ALLPAIRS_PEAK_WORK * pq.mallpairs.peaks; + return fullwork < ALLPAIRS_WORK_MARGIN * fastwork; } // Helper classes and functions for PQEvaluatorCheck ------------------------- @@ -373,6 +423,7 @@ PQEvaluatorPtr createPQEvaluator(PQEvaluatorType pqtp, PQEvaluatorPtr pqevsrc) { rv->mncpu = pqevsrc->mncpu; rv->mvalue_ticker = pqevsrc->mvalue_ticker; rv->mtypeused = pqevsrc->mtypeused; + rv->maddedallpairs = pqevsrc->maddedallpairs; } return rv; } diff --git a/src/diffpy/srreal/PQEvaluator.hpp b/src/diffpy/srreal/PQEvaluator.hpp index a4073b4..e1015e1 100644 --- a/src/diffpy/srreal/PQEvaluator.hpp +++ b/src/diffpy/srreal/PQEvaluator.hpp @@ -39,6 +39,10 @@ namespace srreal { class PairQuantity; +namespace kernels { +struct CalculationKernel; +} + /// shared pointer to PQEvaluatorBasic typedef std::shared_ptr PQEvaluatorPtr; @@ -82,6 +86,8 @@ class DLL_EXPORT PQEvaluatorBasic { eventticker::EventTicker mvalue_ticker; /// type of PQEvaluator that was actually used PQEvaluatorType mtypeused; + /// the calculator added all pairs at once in the last full calculation + bool maddedallpairs = false; private: // serialization @@ -92,6 +98,8 @@ class DLL_EXPORT PQEvaluatorBasic { } }; +class StructureDifference; + class DLL_EXPORT PQEvaluatorOptimized : public PQEvaluatorBasic { public: // methods @@ -102,9 +110,13 @@ class DLL_EXPORT PQEvaluatorOptimized : public PQEvaluatorBasic { private: // data StructureAdapterPtr mlast_structure; + // Process-local kernel identity. A restored value is recomputed on first use. + const kernels::CalculationKernel* mlastkernel = nullptr; - // helper method + // helper methods void updateValueCompletely(PairQuantity&, StructureAdapterPtr); + bool allPairsTakeLessWork(const PairQuantity&, + const StructureDifference&) const; // serialization friend class boost::serialization::access; @@ -113,6 +125,7 @@ class DLL_EXPORT PQEvaluatorOptimized : public PQEvaluatorBasic { using boost::serialization::base_object; ar& base_object(*this); ar & mlast_structure; + if constexpr (Archive::is_loading::value) mlastkernel = nullptr; } }; diff --git a/src/diffpy/srreal/PairQuantity.cpp b/src/diffpy/srreal/PairQuantity.cpp index 8efa799..97616a1 100644 --- a/src/diffpy/srreal/PairQuantity.cpp +++ b/src/diffpy/srreal/PairQuantity.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -106,7 +107,7 @@ const double& PairQuantity::getRmax() const { return mrmax; } void PairQuantity::setEvaluatorType(PQEvaluatorType evtp) { if (mevaluator.get() && mevaluator->typeint() == evtp) return; - PQEvaluatorPtr pqev = createPQEvaluator(evtp, mevaluator); + PQEvaluatorPtr pqev = createPQEvaluator(evtp, mevaluator.ptr()); // validate the new evaluator object try { pqev->validate(*this); @@ -206,11 +207,11 @@ bool PairQuantity::getPairMask(int i, int j) const { } void PairQuantity::setTypeMask(string smbli, string smblj, bool mask) { - static string upcaseall; - if (upcaseall.empty()) { - upcaseall = ALLATOMSSTR; - transform(upcaseall.begin(), upcaseall.end(), upcaseall.begin(), ::toupper); - } + static const string upcaseall = [] { + string rv = ALLATOMSSTR; + transform(rv.begin(), rv.end(), rv.begin(), ::toupper); + return rv; + }(); // accept "ALL" (upper ALLATOMSSTR) for smbli and smblj if (upcaseall == smbli) smbli = ALLATOMSSTR; if (upcaseall == smblj) smblj = ALLATOMSSTR; @@ -273,6 +274,24 @@ void PairQuantity::resetValue() { fill(mvalue.begin(), mvalue.end(), 0.0); } +const kernels::CalculationKernel& PairQuantity::preparePairSum( + kernels::PairSumArgs& args, double rmin, double rmax) { + if (!mallpairs.pairsum) + mallpairs.pairsum = std::make_shared(); + auto& state = *mallpairs.pairsum; + args.mask = this->hasMask() ? this : nullptr; + const auto& kernel = kernels::activeCalculationKernel(); + args.g = state.table.geometryArgs(*mstructure, rmin, rmax); + args.workspace = state.workspaceFor(kernel); + args.out = mvalue.data(); + return kernel; +} + +void PairQuantity::AllPairsState::record(kernels::PairSumStats stats) { + pairs = stats.pairs; + peaks = stats.peaks; +} + void PairQuantity::configureBondGenerator(BaseBondGenerator& bnds) const { bnds.setRmin(this->getRmin()); bnds.setRmax(this->getRmax()); diff --git a/src/diffpy/srreal/PairQuantity.hpp b/src/diffpy/srreal/PairQuantity.hpp index ee8fb51..87b32b9 100644 --- a/src/diffpy/srreal/PairQuantity.hpp +++ b/src/diffpy/srreal/PairQuantity.hpp @@ -39,6 +39,13 @@ namespace srreal { class BaseBondGenerator; +namespace kernels { +struct PairSumState; +struct PairSumArgs; +struct PairSumStats; +struct CalculationKernel; +} + class DLL_EXPORT PairQuantity : public diffpy::Attributes { public: // class constants @@ -93,6 +100,13 @@ class DLL_EXPORT PairQuantity : public diffpy::Attributes { virtual void resetValue(); virtual void configureBondGenerator(BaseBondGenerator&) const; virtual void addPairContribution(const BaseBondGenerator&, int) {} + /// For an unpartitioned half sum, add every unmasked pair as + /// addPairContribution would. Return false before changing the value + /// to use per-bond evaluation. Derived calculators can opt in when + /// their batch sum matches their per-bond contributions. + virtual bool addAllPairContributions() { return false; } + const kernels::CalculationKernel& preparePairSum(kernels::PairSumArgs&, + double rmin, double rmax); virtual void executeParallelMerge(const std::string& pdata); virtual void finishValue() {} int countSites() const; @@ -109,17 +123,63 @@ class DLL_EXPORT PairQuantity : public diffpy::Attributes { typedef std::unordered_map, bool, diffpy::PairHash> TypeMaskStorage; + /// Pairs of the last structures for calculators that add all pairs at + /// once. Neither copied nor serialized. + struct AllPairsState { + AllPairsState() = default; + void record(kernels::PairSumStats stats); + AllPairsState(const AllPairsState& other) + : pairs(other.pairs), peaks(other.peaks) {} + AllPairsState& operator=(const AllPairsState& other) { + pairsum.reset(); + pairs = other.pairs; + peaks = other.peaks; + return *this; + } + std::shared_ptr pairsum; + /// Pairs and merged contributions of the last sum of all pairs, from + /// which the OPTIMIZED evaluator estimates the work of the next one. + double pairs = 0.0; + double peaks = 0.0; + }; + QuantityType mvalue; StructureAdapterPtr mstructure; double mrmin; double mrmax; - PQEvaluatorPtr mevaluator; + /// The evaluator of the calculator. A copy gets a fresh evaluator, so + /// that it does not take the last structure of the original for its own. + class EvaluatorHolder { + public: + EvaluatorHolder() {} + EvaluatorHolder(const EvaluatorHolder& other) : mptr(other.copy()) {} + EvaluatorHolder& operator=(const EvaluatorHolder& other) { + if (this != &other) mptr = other.copy(); + return *this; + } + EvaluatorHolder& operator=(PQEvaluatorPtr pqev) { + mptr = pqev; + return *this; + } + PQEvaluatorBasic* operator->() const { return mptr.get(); } + PQEvaluatorBasic* get() const { return mptr.get(); } + PQEvaluatorPtr& ptr() { return mptr; } + + private: + PQEvaluatorPtr copy() const { + return mptr ? createPQEvaluator(mptr->typeint(), mptr) : mptr; + } + PQEvaluatorPtr mptr; + }; + + EvaluatorHolder mevaluator; bool mdefaultpairmask; PairMaskStorage minvertpairmask; std::unordered_map msiteallmask; TypeMaskStorage mtypemask; int mmergedvaluescount; mutable eventticker::EventTicker mticker; + AllPairsState mallpairs; private: // methods @@ -134,7 +194,7 @@ class DLL_EXPORT PairQuantity : public diffpy::Attributes { ar & mstructure; ar & mrmin; ar & mrmax; - ar & mevaluator; + ar & mevaluator.ptr(); ar & mdefaultpairmask; ar & minvertpairmask; ar & msiteallmask; diff --git a/src/diffpy/srreal/PeriodicStructureAdapter.hpp b/src/diffpy/srreal/PeriodicStructureAdapter.hpp index 6e3bcff..443959c 100644 --- a/src/diffpy/srreal/PeriodicStructureAdapter.hpp +++ b/src/diffpy/srreal/PeriodicStructureAdapter.hpp @@ -164,6 +164,8 @@ class DLL_EXPORT PeriodicStructureBondGenerator : public BaseBondGenerator { int nextpoint = 0; }; + friend class kernels::PairTable; + /// Create generator without site1 positions, add them with addPosition. PeriodicStructureBondGenerator(StructureAdapterConstPtr, BondOrder); diff --git a/src/diffpy/srreal/ScatteringFactorTable.cpp b/src/diffpy/srreal/ScatteringFactorTable.cpp index b759d7e..2aa4497 100644 --- a/src/diffpy/srreal/ScatteringFactorTable.cpp +++ b/src/diffpy/srreal/ScatteringFactorTable.cpp @@ -100,14 +100,16 @@ unordered_set ScatteringFactorTable::getCustomSymbols() const { void ScatteringFactorTableOwner::setScatteringFactorTable( ScatteringFactorTablePtr sft) { ensureNonNull("ScatteringFactorTable", sft); - if (msftable != sft) mprivateticker.click(); - msftable = sft; + if (msftable != sft) { + msftable = sft; + mtickedtable = msftable; + mprivateticker.click(); + } } void ScatteringFactorTableOwner::setScatteringFactorTableByType( const string& tp) { - msftable = ScatteringFactorTable::createByType(tp); - mprivateticker.click(); + this->setScatteringFactorTable(ScatteringFactorTable::createByType(tp)); } ScatteringFactorTablePtr& @@ -127,6 +129,10 @@ const string& ScatteringFactorTableOwner::getRadiationType() const { } eventticker::EventTicker& ScatteringFactorTableOwner::ticker() const { + if (mtickedtable.lock() != msftable) { + mtickedtable = msftable; + mprivateticker.click(); + } if (msftable) mprivateticker.updateFrom(msftable->ticker()); return mprivateticker; } diff --git a/src/diffpy/srreal/ScatteringFactorTable.hpp b/src/diffpy/srreal/ScatteringFactorTable.hpp index c059cc2..4308b89 100644 --- a/src/diffpy/srreal/ScatteringFactorTable.hpp +++ b/src/diffpy/srreal/ScatteringFactorTable.hpp @@ -22,6 +22,7 @@ #ifndef SCATTERINGFACTORTABLE_HPP_INCLUDED #define SCATTERINGFACTORTABLE_HPP_INCLUDED +#include #include #include @@ -88,6 +89,8 @@ class DLL_EXPORT ScatteringFactorTableOwner { private: // data ScatteringFactorTablePtr msftable; + // Detect replacements made through the mutable table reference. + mutable std::weak_ptr mtickedtable; mutable eventticker::EventTicker mprivateticker; // serialization @@ -95,6 +98,7 @@ class DLL_EXPORT ScatteringFactorTableOwner { template void serialize(Archive& ar, const unsigned int version) { ar & msftable & mprivateticker; + if constexpr (Archive::is_loading::value) mtickedtable = msftable; } }; diff --git a/src/diffpy/srreal/kernels/CalculationKernelCPU.cpp b/src/diffpy/srreal/kernels/CalculationKernelCPU.cpp new file mode 100644 index 0000000..4b40e98 --- /dev/null +++ b/src/diffpy/srreal/kernels/CalculationKernelCPU.cpp @@ -0,0 +1,154 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + *****************************************************************************/ + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace diffpy { +namespace srreal { +namespace kernels { + +namespace { + +// Each worker groups a bounded number of pairs before running the SIMD +// windows. Symmetric structures often need only a few distinct windows. +constexpr long long GROUP_BATCH = 65536; +constexpr long long PAIRS_PER_WORKER = 8192; + +int workerCount(long long pairs) { + if (pairs < 2 * PAIRS_PER_WORKER) return 1; + const long long hardware = std::max(1u, std::thread::hardware_concurrency()); + return int(std::min(pairs / PAIRS_PER_WORKER, hardware)); +} + +struct Worker { + PairGroups groups; + std::vector sum; + PairSumStats stats; +}; + +struct CPUState : PairSumWorkspace { + std::vector workers; +}; + +// Only immutable geometry, scattering arrays and mask data are read by +// the workers. Publish the result in range order after all workers succeed. +template +PairSumStats sumPairs(const PairSumArgs& a, int npts, + const GroupPair& groupPair, const AddPeak& addPeak) { + const long long count = pairCount(a.g); + if (!count || !npts) return {}; + const int nworkers = workerCount(count); + auto& workers = static_cast(*a.workspace).workers; + workers.resize(nworkers); + auto run = [&](int i) { + auto& w = workers[i]; + w.sum.assign(npts, 0.0); + // Count locally while neighboring workers append to their group vectors. + PairSumStats stats; + const auto geometry = a.g; + // Divide before multiplying so even a large implicit triangle fits. + const long long stride = count / nworkers; + const long long rest = count % nworkers; + const long long first = i * stride + std::min(i, rest); + const long long last = first + stride + (i < rest); + for (long long begin = first; begin < last; begin += GROUP_BATCH) { + w.groups.clear(); + const long long end = std::min(begin + GROUP_BATCH, last); + ClusterPair cluster = + geometry.translations ? ClusterPair{} : ClusterPair::at(begin); + for (long long k = begin; k < end; ++k) { + const Pair p = pairAt(geometry, k, cluster); + if (!geometry.translations) cluster.next(); + if (!p.ok) continue; + ++stats.pairs; + if (a.mask && !a.mask->getPairMask(p.site0, p.site1)) continue; + groupPair(w.groups, p, widthOf(a.w, p.dist, p.msd)); + } + for (const auto& g : w.groups.groups()) addPeak(w.sum.data(), g); + stats.peaks += w.groups.groups().size(); + } + w.stats = stats; + }; + std::vector> pending; + for (int i = 1; i < nworkers; ++i) { + pending.push_back(std::async(std::launch::async, run, i)); + } + run(0); + for (auto& task : pending) task.get(); + + PairSumStats stats; + for (const Worker& w : workers) { + stats.pairs += w.stats.pairs; + stats.peaks += w.stats.peaks; + for (int i = 0; i < npts; ++i) a.out[i] += w.sum[i]; + } + return stats; +} + +} // namespace + +void addProfilePeak(const PeakProfile& profile, double* out, int npts, + int rlosteps, double rstep, double dist, double fwhm, + double scale) { + const int ilo = + gridIndex(dist + profile.xboundlo(fwhm), rstep, rlosteps, 0, npts); + const int ihi = + gridIndex(dist + profile.xboundhi(fwhm), rstep, rlosteps, 1, npts); + profile.addRDFPeak(out, ilo, ihi, rlosteps, rstep, dist, fwhm, scale); +} + +PairSumStats cpuPDFPairSum(const PDFPairSumArgs& a, GaussianWindow gaussian) { + auto group = [&](PairGroups& groups, const Pair& p, double width) { + groups.add(p.dist, width, 0, + a.sfsite[p.site0] * a.sfsite[p.site1] * p.weight); + }; + auto peak = [&](double* out, const PairGroups::Group& g) { + if (g.width <= 0) return; + const double half = a.halfboundrel * g.width; + const int ilo = gridIndex(g.dist - half, a.rstep, a.rlosteps, 0, a.npts); + const int ihi = gridIndex(g.dist + half, a.rstep, a.rlosteps, 1, a.npts); + gaussian({out, ilo, ihi, a.rlosteps, a.rstep, g.dist, g.width, g.scale, + a.height / g.width, bool(a.cropped), a.halfboundrel}); + }; + return sumPairs(a, a.npts, group, peak); +} + +PairSumStats cpuDebyePairSum(const DebyePairSumArgs& a, DebyeSum debye) { + if (a.kqlo >= a.kqhi) return {}; + auto group = [&](PairGroups& groups, const Pair& p, double width) { + const int t0 = a.typeofsite[p.site0], t1 = a.typeofsite[p.site1]; + const auto typepair = std::int64_t(t0) * a.ntypes + t1; + groups.add(p.dist, fwhmToSigma(width), typepair, 1.0, p.weight); + }; + auto peak = [&](double* out, const PairGroups::Group& g) { + // Test the cutoff at the scale of one pair before multiplying by count. + const int t0 = int(g.tag / a.ntypes), t1 = int(g.tag % a.ntypes); + debye({out, a.kqlo, a.kqhi, a.qstep, g.dist, g.width, double(g.weight), + a.prec, a.sf + size_t(a.curve[t0]) * a.nq, + a.sf + size_t(a.curve[t1]) * a.nq, g.scale, a.occupancy[t0], + a.occupancy[t1]}); + }; + return sumPairs(a, a.nq, group, peak); +} + +PairSumWorkspace* makeCPUWorkspace() { return new CPUState; } + +} // namespace kernels +} // namespace srreal +} // namespace diffpy diff --git a/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp b/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp index 8e1a3d5..dcc47ed 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp +++ b/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp @@ -22,6 +22,7 @@ namespace diffpy { namespace srreal { +class PairQuantity; namespace kernels { /// Add one Gaussian peak to the PDFCalculator RDF array. @@ -46,13 +47,15 @@ struct GaussianWindowArgs { double halfboundrel; }; -/// Add one pair to the BaseDebyeSum F(Q) array. +/// Add count equal pairs to the BaseDebyeSum F(Q) array. /// /// For kq = kqlo, kqlo + 1, ... while kq < kqhi: /// q = kq * qstep /// t = scale * exp(-0.5 * (dwsigma * q)^2) * sf0[kq] * sf1[kq] / dist /// stop at the first kq where |t| <= prec -/// out[kq] += t * sin(q * dist) +/// out[kq] += count * t * sin(q * dist) +/// The sum stops where it does for one pair, so that count pairs add +/// count times what one pair adds. struct DebyeSumArgs { double* out; int kqlo; @@ -64,6 +67,8 @@ struct DebyeSumArgs { double prec; const double* sf0; const double* sf1; + double count = 1.0; + double occ0 = 1.0, occ1 = 1.0; }; /// Select candidate bonds from n points by their squared distance. @@ -86,17 +91,134 @@ struct BondCandidatesArgs { int* idx; }; +/// Positions and pairs of a structure for a sum over all its pairs, as +/// PairTable gives them. The arrays stay with the caller. +struct PairGeometryArgs { + int cntsites; + int cntpositions; + int cnttrans; + /// number of entries in the list of pairs, unused without translations + long long cntpairs; + /// The pairs are those of the list below, of positions with lattice + /// translations. Otherwise they are all pairs of two sites. + int translations; + /// Positions of site i are [sitebegin[i], sitebegin[i + 1]). A pair + /// starts at the first position of its site0. + const int* sitebegin; + /// site of each position + const int* possite; + /// Cartesian coordinates x, y, z of each position + const double* pos; + /// Displacement tensors of the positions and of the sites, each as its + /// elements xx, yy, zz, xy, xz, yz. + const double* posuij; + const double* siteuij; + const unsigned char* anisotropy; + const int* multiplicity; + /// Cartesian coordinates of the lattice translations + const double* trans; + /// Site of the first position, second position and translation of each + /// pair in the list. + const int* pairsite0; + const int* pairpos; + const int* pairtrans; + /// changes when the list of pairs does + int pairversion; + /// The pairs are those of rmin <= distance <= rmax and of a distance + /// above zeroeps. + double rmin; + double rmax; + double zeroeps; +}; + +/// Built-in peak width model and its parameters. +struct PairWidthArgs { + enum Kind { Jeong, DebyeWaller, Constant } kind; + double delta1; + double delta2; + double qbroad; + double qbroadsep; + double width; +}; + +/// Counts used to choose between a CPU batch and an incremental update. +struct PairSumStats { + long long pairs = 0; + long long peaks = 0; +}; + +/// Owned by the calculator and reused by kernels with the same backend. +struct PairSumWorkspace { + virtual ~PairSumWorkspace() = default; +}; +using MakeWorkspace = PairSumWorkspace* (*)(); + +struct PairSumArgs { + PairGeometryArgs g; + PairWidthArgs w; + double* out; + PairSumWorkspace* workspace; + /// A null mask includes every pair. + const PairQuantity* mask; +}; + +/// Add the peaks of PDFCalculator for all pairs of a structure, what +/// gaussianWindow adds for each of them. +struct PDFPairSumArgs : PairSumArgs { + /// scattering factor of each site with its occupancy + const double* sfsite; + /// The peaks are added to the npts values at out, whose first one is at + /// r = rlosteps * rstep. + int npts; + int rlosteps; + double rstep; + /// height of the peak profile for the width of one + double height; + int cropped; + double halfboundrel; +}; + +/// Add the terms of BaseDebyeSum for all pairs of a structure, what +/// debyeSum adds for each of them. +struct DebyePairSumArgs : PairSumArgs { + /// Each site type maps to one scattering curve and an occupancy. + /// sf contains ncurves consecutive rows of nq unweighted factors. + const int* typeofsite; + const int* curve; + const double* occupancy; + int ntypes, ncurves; + int nq; + const double* sf; + /// The terms are added to the values of kqlo <= kq < kqhi at out. + int kqlo; + int kqhi; + double qstep; + double prec; +}; + +using GaussianWindow = void (*)(const GaussianWindowArgs&); +using DebyeSum = void (*)(const DebyeSumArgs&); + struct CalculationKernel { /// Name for setCalculationKernel, a string literal. const char* name; - void (*gaussianWindow)(const GaussianWindowArgs&); - void (*debyeSum)(const DebyeSumArgs&); + GaussianWindow gaussianWindow; + DebyeSum debyeSum; int (*bondCandidates)(const BondCandidatesArgs&); + // Sums over all pairs of a structure. + // The output is updated only after the sum succeeds. + PairSumStats (*pdfPairSum)(const PDFPairSumArgs&); + PairSumStats (*debyePairSum)(const DebyePairSumArgs&); + MakeWorkspace makeWorkspace; }; /// The active kernel, exported for the unit tests. DLL_EXPORT const CalculationKernel& activeCalculationKernel(); +PairSumStats cpuPDFPairSum(const PDFPairSumArgs&, GaussianWindow); +PairSumStats cpuDebyePairSum(const DebyePairSumArgs&, DebyeSum); +PairSumWorkspace* makeCPUWorkspace(); + // Kernel tables per instruction set. The SIMD ones exist only when CMake // compiles them, see cmake/CalculationKernels.cmake. const CalculationKernel& scalarCalculationKernel(); @@ -105,6 +227,19 @@ const CalculationKernel& avx2CalculationKernel(); const CalculationKernel& avx512CalculationKernel(); const CalculationKernel& neonCalculationKernel(); +/// Bind both the per-pair and batch entry points to the same instruction set. +template +constexpr CalculationKernel makeCPUKernel( + const char* name, int (*candidates)(const BondCandidatesArgs&)) { + return {name, + Gaussian, + Debye, + candidates, + [](const PDFPairSumArgs& a) { return cpuPDFPairSum(a, Gaussian); }, + [](const DebyePairSumArgs& a) { return cpuDebyePairSum(a, Debye); }, + makeCPUWorkspace}; +} + } // namespace kernels } // namespace srreal } // namespace diffpy diff --git a/src/diffpy/srreal/kernels/CalculationKernelSIMD.ipp b/src/diffpy/srreal/kernels/CalculationKernelSIMD.ipp index 620e1fd..3ec5cf6 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelSIMD.ipp +++ b/src/diffpy/srreal/kernels/CalculationKernelSIMD.ipp @@ -225,6 +225,7 @@ struct SIMDKernel { const B vdwsigma(a.dwsigma); const B vsinescale(a.scale / a.dist); const B vprec(a.prec); + const B vcount(a.count); const B neghalf(-0.5); const B lanes = laneIndex(); for (int kq = a.kqlo; kq < a.kqhi;) { @@ -237,9 +238,14 @@ struct SIMDKernel { B g = vsinescale * xsimd::exp(neghalf * dwq * dwq); B ratio = xsimd::exp(negalpha * (twow * k + ww)); auto [s, c] = xsimd::sincos(q * vdist); + // The pairs are counted in the sine, which leaves the terms for + // their comparison with the precision. The rotation keeps the + // scale of (sin, cos). + s *= vcount; + c *= vcount; for (; kq < kqvec; kq += W) { - const B t = - g * B::load_unaligned(a.sf0 + kq) * B::load_unaligned(a.sf1 + kq); + const B t = g * (B::load_unaligned(a.sf0 + kq) * B(a.occ0)) * + (B::load_unaligned(a.sf1 + kq) * B(a.occ1)); const BB small = xsimd::abs(t) <= vprec; if (xsimd::any(small)) { // Keep the lanes before the first small term. @@ -256,7 +262,8 @@ struct SIMDKernel { if (kq < kqend) { const int n = kqend - kq; // Unused lanes have zero t and do not reach the output. - B t = g * loadPartial(a.sf0 + kq, n) * loadPartial(a.sf1 + kq, n); + B t = g * (loadPartial(a.sf0 + kq, n) * B(a.occ0)) * + (loadPartial(a.sf1 + kq, n) * B(a.occ1)); const BB small = xsimd::abs(t) <= vprec; if (xsimd::any(small)) t = truncated(t, small); addProductPartial(a.out + kq, t, s, n); @@ -304,7 +311,7 @@ struct SIMDKernel { template constexpr CalculationKernel makeSIMDKernel(const char* name) { using K = SIMDKernel; - return {name, K::gaussianWindow, K::debyeSum, K::bondCandidates}; + return makeCPUKernel(name, K::bondCandidates); } } // namespace diff --git a/src/diffpy/srreal/kernels/CalculationKernelScalar.cpp b/src/diffpy/srreal/kernels/CalculationKernelScalar.cpp index b0a1f80..cc874eb 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelScalar.cpp +++ b/src/diffpy/srreal/kernels/CalculationKernelScalar.cpp @@ -43,9 +43,10 @@ void debyeSumScalar(const DebyeSumArgs& a) { const double q = kq * a.qstep; const double dwq = a.dwsigma * q; const double dwscale = exp(-0.5 * dwq * dwq); - const double sinescale = a.scale * dwscale * a.sf0[kq] * a.sf1[kq] / a.dist; + const double sinescale = (a.scale / a.dist) * dwscale * + (a.sf0[kq] * a.occ0) * (a.sf1[kq] * a.occ1); if (fabs(sinescale) <= a.prec) break; - a.out[kq] += sinescale * sin(q * a.dist); + a.out[kq] += a.count * sinescale * sin(q * a.dist); } } @@ -66,8 +67,9 @@ int bondCandidatesScalar(const BondCandidatesArgs& a) { } // namespace const CalculationKernel& scalarCalculationKernel() { - static constexpr CalculationKernel kernel = { - "scalar", gaussianWindowScalar, debyeSumScalar, bondCandidatesScalar}; + static constexpr auto kernel = + makeCPUKernel("scalar", + bondCandidatesScalar); return kernel; } diff --git a/src/diffpy/srreal/kernels/PairGeometry.hpp b/src/diffpy/srreal/kernels/PairGeometry.hpp new file mode 100644 index 0000000..8921e80 --- /dev/null +++ b/src/diffpy/srreal/kernels/PairGeometry.hpp @@ -0,0 +1,146 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + *****************************************************************************/ + +#ifndef DIFFPY_KERNELS_PAIRGEOMETRY_HPP +#define DIFFPY_KERNELS_PAIRGEOMETRY_HPP + +#include +#include + +#include + +namespace diffpy { +namespace srreal { +namespace kernels { + +constexpr double PAIR_SIGMA_TO_FWHM = 2.3548200450309493; + +/// Use the per-bond rounding order for the Debye precision cutoff. +inline double fwhmToSigma(double fwhm) { + constexpr double scale = 1.0 / PAIR_SIGMA_TO_FWHM; + return fwhm * scale; +} + +inline long long pairCount(const PairGeometryArgs& g) { + return g.translations ? g.cntpairs + : (long long)g.cntsites * (g.cntsites - 1) / 2; +} + +/// Clip before converting to an integer, including for very wide peaks. +inline int gridIndex(double r, double step, int lo, int offset, int npts) { + const double i = floor(r / step) - lo + offset; + return i > 0 ? (i < npts ? int(i) : npts) : 0; +} + +struct Pair { + bool ok; + int site0; + int site1; + double dist; + double msd; + unsigned weight; +}; + +/// As meanSquareDisplacement for the tensor xx, yy, zz, xy, xz, yz. +inline double siteMSD(const double* U, double rx, double ry, double rz, + int anisotropy) { + if (!anisotropy) return U[0]; + const double sUs = U[0] * rx * rx + U[1] * ry * ry + U[2] * rz * rz + + 2 * (U[3] * rx * ry + U[4] * rx * rz + U[5] * ry * rz); + return sUs / (rx * rx + ry * ry + rz * rz); +} + +/// Coordinates of a pair in the strict lower triangle of a cluster. +struct ClusterPair { + int site0, site1; + + static ClusterPair at(long long k) { + // k = i0 * (i0 - 1) / 2 + i1 with i1 < i0. + long long row = (long long)((1.0 + sqrt(1.0 + 8.0 * (double)k)) / 2); + while (row * (row - 1) / 2 > k) --row; + while ((row + 1) * row / 2 <= k) ++row; + return {int(row), int(k - row * (row - 1) / 2)}; + } + + void next() { + if (++site1 == site0) { + ++site0; + site1 = 0; + } + } +}; + +inline Pair pairAt(const PairGeometryArgs& g, long long k, + ClusterPair cluster) { + Pair rv{}; + int p1 = cluster.site1; + rv.site0 = cluster.site0; + double tx = 0.0, ty = 0.0, tz = 0.0; + if (g.translations) { + rv.site0 = g.pairsite0[k]; + p1 = g.pairpos[k]; + const double* t = g.trans + 3LL * g.pairtrans[k]; + tx = t[0]; + ty = t[1]; + tz = t[2]; + } + rv.site1 = g.possite[p1]; + const double* r0 = g.pos + 3LL * g.sitebegin[rv.site0]; + const double* r1 = g.pos + 3LL * p1; + const double rx = r1[0] + tx - r0[0]; + const double ry = r1[1] + ty - r0[1]; + const double rz = r1[2] + tz - r0[2]; + rv.dist = sqrt(rx * rx + ry * ry + rz * rz); + rv.ok = !(rv.dist < g.rmin || rv.dist > g.rmax || rv.dist <= g.zeroeps); + if (!rv.ok) return rv; + rv.msd = + siteMSD(g.siteuij + 6LL * rv.site0, rx, ry, rz, g.anisotropy[rv.site0]) + + siteMSD(g.posuij + 6LL * p1, rx, ry, rz, g.anisotropy[rv.site1]); + const unsigned m = g.multiplicity[rv.site0]; + rv.weight = (g.translations && rv.site0 == rv.site1) ? m : 2 * m; + return rv; +} + +/// Look up a pair directly from its index. +inline Pair pairAt(const PairGeometryArgs& g, long long k) { + return pairAt(g, k, g.translations ? ClusterPair{} : ClusterPair::at(k)); +} + +inline double debyeWallerWidth(double msd) { + return msd < 0.0 ? 0.0 : PAIR_SIGMA_TO_FWHM * sqrt(msd); +} + +inline double sharpeningRatio(double r, double delta1, double delta2, + double qbroad) { + if (r < DBL_EPSILON) return 0.0; + const double qr = qbroad * r; + return 1.0 - delta1 / r - delta2 / (r * r) + qr * qr; +} + +inline double sharpenedWidth(double r, double corr, double msd, + double qbroadsep) { + const double qr = qbroadsep * r; + return sqrt(corr) * debyeWallerWidth(msd) + qr * qr; +} + +/// FWHM of the built-in peak width models for one pair. +inline double widthOf(const PairWidthArgs& w, double dist, double msd) { + if (w.kind == PairWidthArgs::Constant) return w.width; + if (w.kind == PairWidthArgs::DebyeWaller) return debyeWallerWidth(msd); + const double corr = sharpeningRatio(dist, w.delta1, w.delta2, w.qbroad); + return corr <= 0.0 ? 0.0 : sharpenedWidth(dist, corr, msd, w.qbroadsep); +} + +} // namespace kernels +} // namespace srreal +} // namespace diffpy + +#endif diff --git a/src/diffpy/srreal/kernels/PairTable.cpp b/src/diffpy/srreal/kernels/PairTable.cpp new file mode 100644 index 0000000..9ac5614 --- /dev/null +++ b/src/diffpy/srreal/kernels/PairTable.cpp @@ -0,0 +1,336 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + ****************************************************************************** + * + * class PairTable -- all pairs of a structure for a sum over them + * + *****************************************************************************/ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace diffpy { +namespace srreal { +namespace kernels { + +namespace { + +/// The pairs of a periodic structure are searched for distances up to this +/// much beyond the r-range. They stay valid while the positions and the +/// lattice change the distances by less. +const double SKIN = 0.25; + +bool isPeriodicClass(const StructureAdapter& s) { + return typeid(s) == typeid(PeriodicStructureAdapter) || + typeid(s) == typeid(CrystalStructureAdapter); +} + +} // namespace + +// Public Methods ------------------------------------------------------------ + +PairTable::View PairTable::view(const StructureAdapter& stru) { + View rv = {&stru, nullptr, nullptr}; + while (true) { + const StructureAdapter& s = *rv.source; + if (typeid(s) == typeid(NoMetaStructureAdapter)) { + const auto& nm = static_cast(s); + rv.source = nm.getSourceStructure().get(); + } else if (typeid(s) == typeid(NoSymmetryStructureAdapter)) { + const auto& ns = static_cast(s); + if (!rv.cluster) rv.cluster = &s; + rv.source = ns.getSourceStructure().get(); + } else { + break; + } + } + if (isPeriodicClass(*rv.source)) { + rv.periodic = static_cast(rv.source); + } + return rv; +} + +bool PairTable::supports(const StructureAdapter& stru) { + const View v = view(stru); + // The empty structure has no pairs whatever its class. + if (stru.countSites() == 0) return true; + if (typeid(*v.source) == typeid(AtomicStructureAdapter)) return true; + return v.periodic; +} + +// Positions, translations and pairs of the structure for the range. + +void PairTable::preparePeriodic(const PeriodicStructureAdapter& stru, + double rmin, double rmax) { + Cell& cell = mcell; + fillCell(stru, cell); + if (!this->hasPairsFor(stru, cell, rmin, rmax)) { + this->searchPairs(stru, cell, rmin, rmax); + } + // site of each position + const int cntsites = stru.countSites(); + mpossite.resize(cell.pos.size()); + for (int i = 0; i < cntsites; ++i) { + for (int p = cell.sitebegin[i]; p < cell.sitebegin[i + 1]; ++p) { + mpossite[p] = i; + } + } +} + +PairGeometryArgs PairTable::geometryArgs(const StructureAdapter& stru, + double rmin, double rmax) { + PairGeometryArgs rv = {}; + Geometry& g = mgeometry; + const View v = view(stru); + const StructureAdapter& sites = v.sites(); + const int cntsites = sites.countSites(); + auto tensor = [](std::vector& out, const R3::Matrix& U) { + for (double u : {U(0, 0), U(1, 1), U(2, 2), U(0, 1), U(0, 2), U(1, 2)}) { + out.push_back(u); + } + }; + rv.translations = v.hasTranslations(); + g.pos.clear(); + g.posuij.clear(); + g.trans.clear(); + if (rv.translations) { + this->preparePeriodic(*v.periodic, rmin, rmax); + for (std::size_t p = 0; p < mcell.pos.size(); ++p) { + g.pos.insert(g.pos.end(), mcell.pos[p].begin(), mcell.pos[p].end()); + tensor(g.posuij, mcell.uij[p]); + } + for (const auto& mno : mmno) { + const auto t = + v.periodic->getLattice().cartesian(R3::Vector(mno[0], mno[1], mno[2])); + g.trans.insert(g.trans.end(), t.begin(), t.end()); + } + rv.cntpairs = mpairsite0.size(); + rv.pairsite0 = mpairsite0.data(); + rv.pairpos = mpairpos.data(); + rv.pairtrans = mpairtrans.data(); + rv.pairversion = msearches; + } else { + mcell.sitebegin.resize(cntsites + 1); + mpossite.resize(cntsites); + for (int i = 0; i < cntsites; ++i) { + mcell.sitebegin[i] = mpossite[i] = i; + const R3::Vector r = sites.siteCartesianPosition(i); + g.pos.insert(g.pos.end(), r.begin(), r.end()); + } + mcell.sitebegin[cntsites] = cntsites; + } + g.siteuij.clear(); + g.anisotropy.resize(cntsites); + g.multiplicity.resize(cntsites); + for (int i = 0; i < cntsites; ++i) { + tensor(g.siteuij, sites.siteCartesianUij(i)); + g.anisotropy[i] = sites.siteAnisotropy(i); + g.multiplicity[i] = sites.siteMultiplicity(i); + } + rv.cntsites = cntsites; + rv.cntpositions = int(mpossite.size()); + rv.cnttrans = int(g.trans.size() / 3); + rv.sitebegin = mcell.sitebegin.data(); + rv.possite = mpossite.data(); + rv.pos = g.pos.data(); + rv.posuij = rv.translations ? g.posuij.data() : g.siteuij.data(); + rv.siteuij = g.siteuij.data(); + rv.anisotropy = g.anisotropy.data(); + rv.multiplicity = g.multiplicity.data(); + rv.trans = g.trans.data(); + rv.rmin = rmin; + rv.rmax = rmax; + // as eps_eq(distance, 0.0) + rv.zeroeps = mathutils::SQRT_DOUBLE_EPS; + return rv; +} + +// The positions of PeriodicStructureBondGenerator and of +// CrystalStructureBondGenerator. + +void PairTable::fillCell(const PeriodicStructureAdapter& stru, Cell& cell) { + const int cntsites = stru.countSites(); + cell.sitebegin.assign(1, 0); + cell.pos.clear(); + cell.uij.clear(); + if (typeid(stru) == typeid(CrystalStructureAdapter)) { + const auto& cstru = static_cast(stru); + for (int i = 0; i < cntsites; ++i) { + for (const Atom& a : cstru.getEquivalentAtoms(i)) { + cell.pos.push_back(a.xyz_cartn); + cell.uij.push_back(a.uij_cartn); + } + cell.sitebegin.push_back(int(cell.pos.size())); + } + return; + } + const Lattice& L = stru.getLattice(); + for (int i = 0; i < cntsites; ++i) { + const Atom& a = stru[i]; + cell.pos.push_back(L.ucvCartesian(a.xyz_cartn)); + cell.uij.push_back(a.uij_cartn); + cell.sitebegin.push_back(int(cell.pos.size())); + } +} + +// Whether the remembered pairs include all pairs of the cell in the range. +// Keep the same cell images as a fresh bond generator at cutoff boundaries. + +bool PairTable::hasPairsFor(const PeriodicStructureAdapter& stru, + const Cell& cell, double rmin, double rmax) const { + if (cell.sitebegin != msitebegin0) return false; + const Lattice& L = stru.getLattice(); + // largest change of a position and of a lattice translation + double dmax = 0.0; + for (std::size_t i = 0; i < mpos0.size(); ++i) { + const double d = R3::distance(cell.pos[i], mpos0[i]); + // NaN makes the pairs invalid + if (!(d <= SKIN)) return false; + if (d > dmax) dmax = d; + } + double tmax = 0.0; + if (L.base() != mbase0) { + // An in-range bond has translation t = r01 - (p1 - p0), even if the + // previous search missed it. Twice the maximum distance from the + // first position bounds |p1 - p0|. + double radius = 0.0; + for (const auto& p : cell.pos) { + radius = std::max(radius, R3::distance(p, cell.pos.front())); + } + // t * strain is the change back to its old translation. The Frobenius + // norm bounds this change for every direction, including in skew cells. + const R3::Matrix strain = R3::prod(L.recbase(), mbase0 - L.base()); + double norm2 = 0.0; + for (double x : strain.data()) norm2 += x * x; + tmax = (rmax + 2 * radius) * std::sqrt(norm2); + } + // No pair that was out of the searched range can be in the range now. + const double change = 2 * dmax + tmax; + const bool lowok = (mrlo0 <= 0.0) || (rmin - change >= mrlo0); + const bool highok = (rmax + change <= mrhi0); + return lowok && highok; +} + +// Search the pairs with the bond generator of the structure. + +void PairTable::searchPairs(const PeriodicStructureAdapter& stru, + const Cell& cell, double rmin, double rmax) { + // An empty site-offset array never matches a cell (even an empty cell has + // the end offset). A failed search must be retried before reusing its lists. + msitebegin0.clear(); + const Lattice& L = stru.getLattice(); + const int cntsites = stru.countSites(); + mrlo0 = rmin - SKIN; + mrhi0 = rmax + SKIN; + mpos0 = cell.pos; + mbase0 = L.base(); + mmno.clear(); + mpairsite0.clear(); + mpairpos.clear(); + mpairtrans.clear(); + // index of each translation by its lattice coordinates + std::map, int> transindex; + auto indexOf = [&](const std::array& mno) { + const auto [it, added] = transindex.try_emplace(mno, int(mmno.size())); + if (added) mmno.push_back(mno); + return it->second; + }; + auto latticeCoordinates = [&](const R3::Vector& t) { + const R3::Vector f = L.fractional(t); + return std::array{int(std::lround(f[0])), int(std::lround(f[1])), + int(std::lround(f[2]))}; + }; + // Keep distinct coincident positions in the candidate list. pairAt tests + // their current distance each time the table is used. + auto owner = stru.createBondGenerator(); + auto& bnds = static_cast(*owner); + bnds.mkeepcoincident = true; + bnds.setRmin(mrlo0); + bnds.setRmax(mrhi0); + std::vector> coincident; + for (int i0 = 0; i0 < cntsites; ++i0) { + auto append = [&](int position, int translation) { + mpairsite0.push_back(i0); + mpairpos.push_back(position); + mpairtrans.push_back(translation); + }; + bnds.selectAnchorSite(i0); + bnds.selectSiteRange(0, i0 + 1); + for (bnds.rewind(); !bnds.finished(); bnds.next()) { + const int p = bnds.mcellindex; + const auto mno = latticeCoordinates(bnds.r1() - cell.pos[p]); + if (p == cell.sitebegin[i0] && mno == std::array{}) continue; + const int t = indexOf(mno); + if (mathutils::eps_eq(bnds.distance(), 0.0)) + coincident.emplace_back(p, t); + else + append(p, t); + } + // Reserve the trailing slots of each anchor for pairs that may separate. + // Interleaving them shifts the regular bonds between CUDA reduction rows. + for (auto [p, t] : coincident) append(p, t); + coincident.clear(); + } + msitebegin0 = cell.sitebegin; + ++msearches; +} + +////////////////////////////////////////////////////////////////////////////// +// Built-in peak width models +////////////////////////////////////////////////////////////////////////////// + +bool assignPairWidth(PairWidthArgs& rv, const PeakWidthModel& model) { + rv = {}; + if (typeid(model) == typeid(JeongPeakWidth)) { + const auto& jeong = static_cast(model); + rv = {PairWidthArgs::Jeong, jeong.getDelta1(), jeong.getDelta2(), + jeong.getQbroad(), jeong.getQbroad_seperable(), 0.0}; + } else if (typeid(model) == typeid(DebyeWallerPeakWidth)) { + rv.kind = PairWidthArgs::DebyeWaller; + } else if (typeid(model) == typeid(ConstantPeakWidth)) { + rv.kind = PairWidthArgs::Constant; + rv.width = static_cast(model).getWidth(); + } else { + return false; + } + return true; +} + +////////////////////////////////////////////////////////////////////////////// +// struct PairSumState +////////////////////////////////////////////////////////////////////////////// + +PairSumWorkspace* PairSumState::workspaceFor(const CalculationKernel& kernel) { + if (mfactory != kernel.makeWorkspace) { + mworkspace.reset(kernel.makeWorkspace()); + mfactory = kernel.makeWorkspace; + } + return mworkspace.get(); +} + +} // namespace kernels +} // namespace srreal +} // namespace diffpy + +// End of file diff --git a/src/diffpy/srreal/kernels/PairTable.hpp b/src/diffpy/srreal/kernels/PairTable.hpp new file mode 100644 index 0000000..3d9c38b --- /dev/null +++ b/src/diffpy/srreal/kernels/PairTable.hpp @@ -0,0 +1,231 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + ****************************************************************************** + * + * class PairTable -- all pairs of a structure for a sum over them + * class PairGroups -- pairs that add the same peak + * + * Internal header, not installed. + * + * PairTable remembers the pairs of a periodic structure and only updates + * their distances while the structure stays close to that of the search, + * as in a refinement. PairGroups merges pairs of the same distance and + * width, so that each kind of peak is added once. + * + *****************************************************************************/ + +#ifndef KERNELS_PAIRTABLE_HPP_INCLUDED +#define KERNELS_PAIRTABLE_HPP_INCLUDED + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace diffpy { +namespace srreal { + +class PeriodicStructureAdapter; +class PeakWidthModel; +class PeakProfile; + +namespace kernels { + +// Exported for the unit tests. +class DLL_EXPORT PairTable { + public: + /// True when the table can give the pairs of the structure. These are + /// the structures of the classes AtomicStructureAdapter, + /// PeriodicStructureAdapter and CrystalStructureAdapter, not of classes + /// derived from them, also within the nometa and nosymmetry wrappers. + static bool supports(const StructureAdapter& stru); + + /// Number of searches for the pairs of periodic structures so far. + int countSearches() const { return msearches; } + + /// The geometry for a kernel that adds all pairs of the structure in + /// the range. Its arrays are those of this table, they are valid until + /// the next call. + PairGeometryArgs geometryArgs(const StructureAdapter& stru, double rmin, + double rmax); + + private: + /// Sources of the sites and lattice translations. + struct View { + /// The structure within any nometa and nosymmetry wrappers. + const StructureAdapter* source; + /// The outermost nosymmetry wrapper or null. Its pairs are those of + /// the sites alone, without symmetry positions and lattice + /// translations. + const StructureAdapter* cluster; + /// The source when it has a lattice, also for a cluster of its sites. + const PeriodicStructureAdapter* periodic; + /// True for the pairs of symmetry positions and lattice translations. + bool hasTranslations() const { return periodic && !cluster; } + /// The structure whose sites are those of the pairs. + const StructureAdapter& sites() const { + return cluster ? *cluster : *source; + } + }; + static View view(const StructureAdapter& stru); + + /// Contiguous storage for the arrays read by the pair-sum kernels. + struct Geometry { + std::vector pos, posuij, siteuij, trans; + std::vector anisotropy; + std::vector multiplicity; + }; + + /// Positions and displacement tensors of a periodic structure as its + /// bond generator has them. + struct Cell { + /// Positions of site i are [sitebegin[i], sitebegin[i + 1]), the anchor + /// of the site is the first one. + std::vector sitebegin; + std::vector pos; + std::vector uij; + }; + + // methods + void preparePeriodic(const PeriodicStructureAdapter& stru, double rmin, + double rmax); + static void fillCell(const PeriodicStructureAdapter& stru, Cell& cell); + bool hasPairsFor(const PeriodicStructureAdapter& stru, const Cell& cell, + double rmin, double rmax) const; + void searchPairs(const PeriodicStructureAdapter& stru, const Cell& cell, + double rmin, double rmax); + + // data + Cell mcell; + // Pairs of a periodic structure and what they were searched for. + /// Positions and lattice basis at the search. + std::vector msitebegin0; + std::vector mpos0; + R3::Matrix mbase0; + /// Range of the distances at the search. + double mrlo0 = 0.0; + double mrhi0 = -1.0; + /// Lattice translations of the pairs in lattice coordinates. + std::vector> mmno; + /// Anchor site, position of the other site and lattice translation of + /// each pair, ordered by the anchor site. + std::vector mpairsite0; + std::vector mpairpos; + std::vector mpairtrans; + int msearches = 0; + // work arrays + std::vector mpossite; + Geometry mgeometry; +}; + +/// Sums of the scales of the pairs that have the same distance and width. +/// +/// Even adjacent floating-point values can straddle a peak or Debye cutoff. +class PairGroups { + public: + struct Group { + double dist; + double width; + /// Tells apart pairs that the distance and the width do not, for + /// example by their scattering factors. + std::int64_t tag; + /// Sum of the scales of the pairs. + double scale; + /// Multiplicity scale of a single Debye pair, before its precision cutoff. + unsigned weight; + }; + + void clear() { + mgroups.clear(); + mslots.assign(mslots.size(), -1); + } + + void add(double dist, double width, std::int64_t tag, double scale, + unsigned weight = 1) { + const std::uint64_t kd = key(dist); + const std::uint64_t kw = key(width); + if (2 * (mgroups.size() + 1) > mslots.size()) this->grow(); + const std::size_t mask = mslots.size() - 1; + std::size_t i = hash(kd, kw, tag, weight) & mask; + for (;; i = (i + 1) & mask) { + const int g = mslots[i]; + if (g < 0) break; + Group& group = mgroups[g]; + if (key(group.dist) == kd && key(group.width) == kw && group.tag == tag && + group.weight == weight) { + group.scale += scale; + return; + } + } + mslots[i] = int(mgroups.size()); + mgroups.push_back({dist, width, tag, scale, weight}); + } + + const std::vector& groups() const { return mgroups; } + + private: + static std::uint64_t key(double x) { return std::bit_cast(x); } + + static std::size_t hash(std::uint64_t kd, std::uint64_t kw, std::int64_t tag, + unsigned weight) { + std::uint64_t h = kd * 0x9e3779b97f4a7c15ull; + h ^= (h >> 29) ^ (kw * 0xbf58476d1ce4e5b9ull); + h ^= (h >> 31) ^ (std::uint64_t(tag) * 0x94d049bb133111ebull); + return std::size_t((h ^ (h >> 32)) + weight); + } + + void grow() { + const std::size_t n = std::max(1024, 2 * mslots.size()); + mslots.assign(n, -1); + const std::size_t mask = n - 1; + for (std::size_t g = 0; g < mgroups.size(); ++g) { + const Group& group = mgroups[g]; + std::size_t i = + hash(key(group.dist), key(group.width), group.tag, group.weight) & mask; + while (mslots[i] >= 0) i = (i + 1) & mask; + mslots[i] = int(g); + } + } + + std::vector mgroups; + /// Open addressing table of the group indices, -1 for a free slot. + std::vector mslots; +}; + +/// Peak width of the built-in models from the values of a bond. +bool assignPairWidth(PairWidthArgs&, const PeakWidthModel&); + +/// Add a peak with the bounds and values supplied by the profile. +void addProfilePeak(const PeakProfile&, double* out, int npts, int rlosteps, + double rstep, double dist, double fwhm, double scale); + +/// What a calculator keeps for its sums over all pairs. +struct PairSumState { + PairTable table; + PairSumWorkspace* workspaceFor(const CalculationKernel& kernel); + + private: + MakeWorkspace mfactory = nullptr; + std::unique_ptr mworkspace; +}; + +} // namespace kernels +} // namespace srreal +} // namespace diffpy + +#endif // KERNELS_PAIRTABLE_HPP_INCLUDED diff --git a/src/diffpy/srreal/scatteringfactordata.cpp b/src/diffpy/srreal/scatteringfactordata.cpp index 6d38fc9..d1191dc 100644 --- a/src/diffpy/srreal/scatteringfactordata.cpp +++ b/src/diffpy/srreal/scatteringfactordata.cpp @@ -109,12 +109,10 @@ class wksmbl_hash { typedef unordered_set SetOfWKFormulas; -const SetOfWKFormulas& getWKFormulasSet() { +SetOfWKFormulas loadWKFormulasSet() { using namespace diffpy::runtimepath; using diffpy::validators::ensureFileOK; - static unique_ptr the_set; - if (the_set) return *the_set; - the_set.reset(new SetOfWKFormulas); + SetOfWKFormulas the_set; string wkfile = datapath("f0_WaasKirf.dat"); ifstream fp(wkfile.c_str()); ensureFileOK(wkfile, fp); @@ -144,16 +142,23 @@ const SetOfWKFormulas& getWKFormulasSet() { throw line.format_error( wkfile, "Line should contain 11 floating point values."); } - if (the_set->count(wk)) { + if (the_set.count(wk)) { string emsg = "Duplicate atom symbol \""; emsg += wk.symbol + "\"."; throw line.format_error(wkfile, emsg); } - the_set->insert(wk); + the_set.insert(wk); wk.symbol.clear(); } } - return getWKFormulasSet(); + return the_set; +} + +/// The table is loaded on the first call, also when the first calls come +/// from several threads. A failed load is repeated on the next call. +const SetOfWKFormulas& getWKFormulasSet() { + static const SetOfWKFormulas the_set = loadWKFormulasSet(); + return the_set; } SetOfWKFormulas::const_iterator findWKFormula(const string& smbl) { @@ -186,56 +191,57 @@ SetOfWKFormulas::const_iterator findWKFormula(const string& smbl) { typedef unordered_map ElectronNumberStorage; -ElectronNumberStorage& getElectronNumberTable() { +ElectronNumberStorage loadElectronNumberTable() { using namespace diffpy::runtimepath; using diffpy::validators::ensureFileOK; - static unique_ptr entable; + ElectronNumberStorage entable; typedef ElectronNumberStorage::value_type ENPair; - if (!entable) { - entable.reset(new ElectronNumberStorage); - string ionfile = datapath("ionlist.dat"); - ifstream fp0(ionfile.c_str()); - ensureFileOK(ionfile, fp0); - LineReader line; - while (fp0 >> line) { - if (line.isignored()) continue; - istringstream fpline(line.line); - string element; - int z = 0; - fpline >> element >> z; - if (!fpline) { - throw line.format_error(ionfile, - "Expected at least 2 columns for (symbol, Z)."); - } - entable->insert(ENPair(element, z)); - for (int v; fpline >> v;) { - ostringstream smbl; - smbl << element << abs(v) << ((v > 0) ? '+' : '-'); - entable->insert(ENPair(smbl.str(), z - v)); - } + string ionfile = datapath("ionlist.dat"); + ifstream fp0(ionfile.c_str()); + ensureFileOK(ionfile, fp0); + LineReader line; + while (fp0 >> line) { + if (line.isignored()) continue; + istringstream fpline(line.line); + string element; + int z = 0; + fpline >> element >> z; + if (!fpline) { + throw line.format_error(ionfile, + "Expected at least 2 columns for (symbol, Z)."); } - const size_t mintablesize = 447; - if (entable->size() < mintablesize) { - ostringstream emsg; - emsg << "Incomplete file. Expected " << mintablesize << " items loaded " - << entable->size() << "."; - throw line.format_error(ionfile, emsg.str()); + entable.insert(ENPair(element, z)); + for (int v; fpline >> v;) { + ostringstream smbl; + smbl << element << abs(v) << ((v > 0) ? '+' : '-'); + entable.insert(ENPair(smbl.str(), z - v)); } } - return *entable; + const size_t mintablesize = 447; + if (entable.size() < mintablesize) { + ostringstream emsg; + emsg << "Incomplete file. Expected " << mintablesize << " items loaded " + << entable.size() << "."; + throw line.format_error(ionfile, emsg.str()); + } + return entable; +} + +/// Loaded on the first call as the Waasmaier-Kirfel formulas. +const ElectronNumberStorage& getElectronNumberTable() { + static const ElectronNumberStorage entable = loadElectronNumberTable(); + return entable; } // Neutron scattering lengths typedef unordered_map NeutronBCStorage; -const NeutronBCStorage& getNeutronBCTable() { +NeutronBCStorage loadNeutronBCTable() { using namespace diffpy::runtimepath; using diffpy::validators::ensureFileOK; typedef NeutronBCStorage::value_type BCPair; - static unique_ptr bctable; - if (bctable) return *bctable; - bctable.reset(new NeutronBCStorage); + NeutronBCStorage bctable; string nsffile = datapath("nsftable.dat"); ifstream fp(nsffile.c_str()); ensureFileOK(nsffile, fp); @@ -264,12 +270,12 @@ const NeutronBCStorage& getNeutronBCTable() { if (!fpbc) { throw line.format_error(nsffile, "Invalid b_c value."); } - if (bctable->count(smbl)) { + if (bctable.count(smbl)) { string emsg = "Duplicate atom symbol \""; emsg += smbl + "\"."; throw line.format_error(nsffile, emsg); } - bctable->insert(BCPair(smbl, bc)); + bctable.insert(BCPair(smbl, bc)); // elements are not explicitly included if there is just one isotope // or if all isotopes are unstable size_t p2 = smbl.find_first_of('-'); @@ -278,23 +284,29 @@ const NeutronBCStorage& getNeutronBCTable() { // there is just one isotope const string& chlf = line.words[1]; bool addel = - (chlf == "100") || (!bctable->count(el) && *chlf.rbegin() == 'Y'); + (chlf == "100") || (!bctable.count(el) && *chlf.rbegin() == 'Y'); if (addel) { - if (bctable->count(el)) { + if (bctable.count(el)) { string emsg = "Duplicate element entry for \""; emsg += el + "\"."; throw line.format_error(nsffile, emsg); ; } - bctable->insert(BCPair(el, bc)); + bctable.insert(BCPair(el, bc)); } } } // define aliases for neutron, deuterium and tritium - bctable->insert(BCPair("n", bctable->at("1-n"))); - bctable->insert(BCPair("D", bctable->at("2-H"))); - bctable->insert(BCPair("T", bctable->at("3-H"))); - return getNeutronBCTable(); + bctable.insert(BCPair("n", bctable.at("1-n"))); + bctable.insert(BCPair("D", bctable.at("2-H"))); + bctable.insert(BCPair("T", bctable.at("3-H"))); + return bctable; +} + +/// Loaded on the first call as the Waasmaier-Kirfel formulas. +const NeutronBCStorage& getNeutronBCTable() { + static const NeutronBCStorage bctable = loadNeutronBCTable(); + return bctable; } } // namespace diff --git a/src/tests/TestBondGeneratorBatching.cpp b/src/tests/TestBondGeneratorBatching.cpp index f422e24..3cd2676 100644 --- a/src/tests/TestBondGeneratorBatching.cpp +++ b/src/tests/TestBondGeneratorBatching.cpp @@ -196,6 +196,17 @@ CrystalStructureAdapterPtr loadTestCrystalStructure(const std::string& f) { return rv; } +/// The same crystal built anew, without any cached symmetry positions. +CrystalStructureAdapterPtr rebuiltCrystal(const CrystalStructureAdapter& s) { + const Lattice& L = s.getLattice(); + CrystalStructureAdapterPtr rv(new CrystalStructureAdapter); + rv->setLatPar(L.a(), L.b(), L.c(), L.alpha(), L.beta(), L.gamma()); + rv->assign(s.begin(), s.end()); + rv->setSymmetryPrecision(s.getSymmetryPrecision()); + for (int i = 0; i < s.countSymOps(); ++i) rv->addSymOp(s.getSymOp(i)); + return rv; +} + /// Select bonds up to 6 from the site 1, used with CaTiO3. void nearSite1(BaseBondGenerator& bnds) { bnds.setRmax(6.0); @@ -277,23 +288,13 @@ TEST_CASE("TestBondGeneratorBatching::test_crystal_changes", bnds->selectAnchorSite(0); return generated(*bnds); }; - // the same structure rebuilt without cached symmetry positions - auto rebuilt = [](const CrystalStructureAdapter& s) { - const Lattice& L = s.getLattice(); - CrystalStructureAdapterPtr rv(new CrystalStructureAdapter); - rv->setLatPar(L.a(), L.b(), L.c(), L.alpha(), L.beta(), L.gamma()); - rv->assign(s.begin(), s.end()); - rv->setSymmetryPrecision(s.getSymmetryPrecision()); - for (int i = 0; i < s.countSymOps(); ++i) rv->addSymOp(s.getSymOp(i)); - return rv; - }; auto same = [](const std::vector& a, const std::vector& b) { return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin(), sameBond); }; std::vector last = bonds(*stru); REQUIRE(!last.empty()); - checkSameBonds(last, bonds(*rebuilt(*stru))); + checkSameBonds(last, bonds(*rebuiltCrystal(*stru))); R3::Matrix R = R3::identity(); R(2, 2) = -1.0; const std::pair> changes[] = { @@ -310,7 +311,7 @@ TEST_CASE("TestBondGeneratorBatching::test_crystal_changes", change(); const std::vector now = bonds(*stru); CHECK(!same(last, now)); - checkSameBonds(now, bonds(*rebuilt(*stru))); + checkSameBonds(now, bonds(*rebuiltCrystal(*stru))); last = now; } }); @@ -346,6 +347,49 @@ TEST_CASE("TestBondGeneratorBatching::test_crystal_refresh", }); } +TEST_CASE("TestBondGeneratorBatching::test_crystal_site_expansion", + "[BondGeneratorBatching]") { + // A changed atom expands only its own site again. All sites then have + // the positions of a new expansion. + CrystalStructureAdapterPtr stru = loadTestCrystalStructure("CaTiO3.stru"); + const int cntsites = stru->countSites(); + auto arrays = [&] { + std::vector rv; + for (int i = 0; i < cntsites; ++i) { + rv.push_back(stru->getEquivalentAtoms(i).data()); + } + return rv; + }; + auto checkAsRebuilt = [&] { + const CrystalStructureAdapterPtr fresh = rebuiltCrystal(*stru); + for (int i = 0; i < cntsites; ++i) { + INFO("site " << i); + CHECK(fresh->getEquivalentAtoms(i) == stru->getEquivalentAtoms(i)); + } + }; + const std::vector before = arrays(); + stru->at(3).xyz_cartn[1] += 0.2; + stru->at(5).uij_cartn(0, 0) += 0.01; + const R3::Vector moved = stru->getEquivalentAtoms(3)[0].xyz_cartn; + const std::vector after = arrays(); + for (int i = 0; i < cntsites; ++i) { + if (i == 3 || i == 5) continue; + INFO("site " << i); + CHECK(before[i] == after[i]); + } + checkAsRebuilt(); + // an atom that moves back gets its first positions again + stru->at(3).xyz_cartn[1] -= 0.2; + CHECK(moved != stru->getEquivalentAtoms(3)[0].xyz_cartn); + checkAsRebuilt(); + // a new lattice moves the positions of every site + stru->setLatPar(5.5, 7.7, 5.4, 90.0, 95.0, 90.0); + checkAsRebuilt(); + // sites that take the atoms of their neighbors + std::swap(stru->at(1), stru->at(2)); + checkAsRebuilt(); +} + TEST_CASE("TestBondGeneratorBatching::test_crystal_constructor", "[BondGeneratorBatching]") { // The public constructor finds the symmetry positions when needed. diff --git a/src/tests/TestCalculationKernel.cpp b/src/tests/TestCalculationKernel.cpp index 7f3c7a0..accbce8 100644 --- a/src/tests/TestCalculationKernel.cpp +++ b/src/tests/TestCalculationKernel.cpp @@ -355,29 +355,33 @@ TEST_CASE("TestCalculationKernel::test_window_lengths", "[CalculationKernel]") { // The scale is negative when a calculator removes a pair. std::vector sf(hi); for (int kq = 0; kq < hi; ++kq) sf[kq] = std::exp(-0.01 * kq); + // Several equal pairs add what one pair adds that many times. for (double scale : {2.0, -2.0}) { for (double prec : {0.0, 1e-3, 0.1}) { - INFO("lo " << lo << ", n " << n << ", scale " << scale << ", prec " - << prec); - QuantityType reference(hi + padding, fill); - QuantityType actual(reference); - DebyeSumArgs args = { - .out = reference.data(), - .kqlo = lo, - .kqhi = hi, - .qstep = 0.05, - .dist = 1.7, - .dwsigma = 0.1, - .scale = scale, - .prec = prec, - .sf0 = sf.data(), - .sf1 = sf.data(), - }; - scalar.debyeSum(args); - args.out = actual.data(); - kernel.debyeSum(args); - CHECK(maxRelativeError(actual, reference) <= KERNEL_TOLERANCE); - CHECK(sameUntouched(actual, reference, fill)); + for (int count : {1, 3}) { + INFO("lo " << lo << ", n " << n << ", scale " << scale + << ", prec " << prec << ", count " << count); + QuantityType reference(hi + padding, fill); + QuantityType actual(reference); + DebyeSumArgs args = { + .out = reference.data(), + .kqlo = lo, + .kqhi = hi, + .qstep = 0.05, + .dist = 1.7, + .dwsigma = 0.1, + .scale = scale, + .prec = prec, + .sf0 = sf.data(), + .sf1 = sf.data(), + }; + for (int k = 0; k < count; ++k) scalar.debyeSum(args); + args.out = actual.data(); + args.count = count; + kernel.debyeSum(args); + CHECK(maxRelativeError(actual, reference) <= KERNEL_TOLERANCE); + CHECK(sameUntouched(actual, reference, fill)); + } } } } diff --git a/src/tests/TestDebyePDFCalculator.cpp b/src/tests/TestDebyePDFCalculator.cpp index bda39e7..0fcf5d6 100644 --- a/src/tests/TestDebyePDFCalculator.cpp +++ b/src/tests/TestDebyePDFCalculator.cpp @@ -19,7 +19,10 @@ #include #include +#include #include +#include +#include #include #include @@ -352,4 +355,171 @@ TEST_CASE_METHOD(TestDebyePDFCalculatorFixture, CHECK(BASIC == mpdfc->getEvaluatorTypeUsed()); } +TEST_CASE_METHOD(TestDebyePDFCalculatorFixture, + "TestDebyePDFCalculator::test_DBPDF_site_occupancies", + "[DebyePDFCalculator]") { + // Sites of one element with different occupancies scatter as sites of + // different atom types with those occupancies. + AtomicStructureAdapterPtr mixed = + std::make_shared(*mstru10); + AtomicStructureAdapterPtr named = + std::make_shared(*mstru10); + for (int i : {0, 3, 4, 8}) { + mixed->at(i).occupancy = 0.4; + named->at(i).occupancy = 0.4; + named->at(i).atomtype = "C2"; + } + mpdfc->getScatteringFactorTable()->setCustomAs("C2", "C"); + for (PQEvaluatorType tp : {BASIC, OPTIMIZED}) { + INFO("evaluator " << tp); + mpdfc->setEvaluatorType(tp); + mpdfc->eval(named); + const QuantityType gnamed = mpdfc->getPDF(); + const QuantityType fnamed = mpdfc->getF(); + mpdfc->eval(mstru10); + CHECK(!allclose(gnamed, mpdfc->getPDF())); + mpdfc->eval(mixed); + CHECK(allclose(gnamed, mpdfc->getPDF())); + CHECK(allclose(fnamed, mpdfc->getF())); + } + // A new occupancy of the first site of an element is a change of that + // site only, which the fast update must get right. + mpdfc->setEvaluatorType(OPTIMIZED); + mpdfc->eval(mixed); + mixed->at(0).occupancy = 0.7; + named->at(0).occupancy = 0.7; + mpdfc->eval(mixed); + CHECK(OPTIMIZED == mpdfc->getEvaluatorTypeUsed()); + const QuantityType gfast = mpdfc->getPDF(); + mpdfc->setEvaluatorType(BASIC); + mpdfc->eval(named); + CHECK(allclose(gfast, mpdfc->getPDF())); +} + +TEST_CASE_METHOD(TestDebyePDFCalculatorFixture, + "TestDebyePDFCalculator::test_retry_unknown_atom", + "[DebyePDFCalculator]") { + const QuantityType expected = mpdfc->eval(mstru10); + mstru10->at(1).atomtype = "Bogus"; + CHECK_THROWS_AS(mpdfc->eval(mstru10), invalid_argument); + CHECK_THROWS_AS(mpdfc->eval(mstru10), invalid_argument); + mstru10->at(1).atomtype = "C"; + CHECK(mpdfc->eval(mstru10) == expected); +} + +TEST_CASE_METHOD(TestDebyePDFCalculatorFixture, + "TestDebyePDFCalculator::test_replace_table_by_reference", + "[DebyePDFCalculator]") { + DebyePDFCalculator reference; + reference.setScatteringFactorTableByType("neutron"); + const auto neutron = reference.eval(mstru10); + for (const auto evaluator : {BASIC, OPTIMIZED}) { + DebyePDFCalculator calc; + calc.setEvaluatorType(evaluator); + auto& table = calc.getScatteringFactorTable(); + REQUIRE(calc.eval(mstru10) != neutron); + table = reference.getScatteringFactorTable(); + CHECK(calc.eval(mstru10) == neutron); + CHECK(calc.getEvaluatorTypeUsed() == BASIC); + } +} + +TEST_CASE_METHOD(TestDebyePDFCalculatorFixture, + "TestDebyePDFCalculator::test_DBPDF_kept_scattering_factors", + "[DebyePDFCalculator]") { + // The kept scattering factors must follow every change of the calculator + // and of the occupancies. + typedef std::function Config; + std::vector history; + auto configure = [&](Config cfg) { + cfg(*mpdfc); + history.push_back(cfg); + }; + // the calculator that is checked + DebyePDFCalculator* calc = mpdfc.get(); + auto check = [&](StructureAdapterPtr stru) { + DebyePDFCalculator fresh; + for (const Config& cfg : history) cfg(fresh); + fresh.setEvaluatorType(BASIC); + fresh.eval(stru); + calc->eval(stru); + CHECK(allclose(fresh.getPDF(), calc->getPDF())); + CHECK(allclose(fresh.getF(), calc->getF())); + }; + check(mstru10); + check(mstru10d1); + configure( + [](DebyePDFCalculator& c) { c.setScatteringFactorTableByType("neutron"); }); + check(mstru10d1); + configure([](DebyePDFCalculator& c) { + c.getScatteringFactorTable()->setCustomAs("C", "Au"); + }); + check(mstru10d1); + configure([](DebyePDFCalculator& c) { c.setQmax(15.0); }); + check(mstru10d1); + configure([](DebyePDFCalculator& c) { c.setRstep(0.02); }); + check(mstru10d1); + // Many occupancies in turn, which leave arrays that no site uses. + AtomicStructureAdapterPtr stru = + std::make_shared(*mstru10d1); + // a calculator of a derived class adds its pairs one by one + struct DerivedDebyePDFCalculator : DebyePDFCalculator {}; + DerivedDebyePDFCalculator onebyone; + for (const Config& cfg : history) cfg(onebyone); + for (DebyePDFCalculator* c : {(DebyePDFCalculator*)&onebyone, mpdfc.get()}) { + const bool pairbypair = (c == &onebyone); + calc = c; + check(stru); + for (int k = 0; k < 40; ++k) { + INFO((pairbypair ? "pair by pair" : "all pairs") << " step " << k); + stru->at(3).occupancy = 0.5 + 0.01 * k; + stru->at(7).occupancy = 0.9 - 0.01 * k; + check(stru); + if (pairbypair) CHECK(OPTIMIZED == calc->getEvaluatorTypeUsed()); + } + } + calc = mpdfc.get(); + // a restored calculator builds its scattering factors again + stringstream storage(ios::in | ios::out | ios::binary); + diffpy::serialization::oarchive oa(storage, ios::binary); + oa << mpdfc; + diffpy::serialization::iarchive ia(storage, ios::binary); + std::shared_ptr pdfc1; + ia >> pdfc1; + const QuantityType g = mpdfc->getPDF(); + stru->at(3).occupancy = 0.3; + pdfc1->eval(stru); + CHECK(!allclose(g, pdfc1->getPDF())); + mpdfc.swap(pdfc1); + check(stru); +} + +TEST_CASE_METHOD(TestDebyePDFCalculatorFixture, + "TestDebyePDFCalculator::test_structure_dependent_scattering", + "[DebyePDFCalculator]") { + struct SiteFactor : DebyePDFCalculator { + double sfSiteAtQ(int site, const double&) const override { + // Both site-specific and structure-wide dependence. + return mstructure->siteCartesianUij(site)(0, 0) + + mstructure->siteCartesianUij(0)(1, 1); + } + }; + for (auto evaluator : {BASIC, OPTIMIZED}) { + SiteFactor calc; + calc.setEvaluatorType(evaluator); + auto stru = std::make_shared(*mstru10); + for (double u : {0.004, 0.05, 0.02, 0.004}) { + (*stru)[0].uij_cartn = u * R3::identity(); + // Give the reference sites unique keys. It must always evaluate each + // site's curve, including sites whose own parameters stayed unchanged. + auto named = std::make_shared(*stru); + for (int i = 0; i < named->countSites(); ++i) + (*named)[i].atomtype = std::to_string(i); + SiteFactor fresh; + fresh.setEvaluatorType(BASIC); + CHECK(allclose(calc.eval(stru), fresh.eval(named))); + } + } +} + // End of file diff --git a/src/tests/TestPQEvaluator.cpp b/src/tests/TestPQEvaluator.cpp index 08cb8c5..9ca7019 100644 --- a/src/tests/TestPQEvaluator.cpp +++ b/src/tests/TestPQEvaluator.cpp @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -92,12 +94,14 @@ CrystalStructureAdapterPtr inversionCrystal(double x) { // TestPQEvaluator ////////////////////////////////////////////////////////////////////////////// -class TestPQEvaluatorFixture { +/// Calculators of the class PDF with the BASIC and OPTIMIZED evaluators. +template +class TestPQEvaluatorFixtureOf { protected: // data EpsilonEqual allclose; - PDFCalculator mpdfcb; - PDFCalculator mpdfco; + PDF mpdfcb; + PDF mpdfco; QuantityType mzeros; AtomicStructureAdapterPtr mstru10; AtomicStructureAdapterPtr mstru10d1; @@ -117,7 +121,7 @@ class TestPQEvaluatorFixture { } public: - TestPQEvaluatorFixture() { + TestPQEvaluatorFixtureOf() { // setup PDFCalculator instances and the zero vector for comparison mpdfcb.setEvaluatorType(BASIC); mpdfco.setEvaluatorType(OPTIMIZED); @@ -140,7 +144,12 @@ class TestPQEvaluatorFixture { mstru9 = std::make_shared(*mstru10); mstru9->erase(9); } -}; // class TestPQEvaluatorFixture +}; // class TestPQEvaluatorFixtureOf + +typedef TestPQEvaluatorFixtureOf TestPQEvaluatorFixture; +// Calculators that add their pairs one by one also in a full calculation. +typedef TestPQEvaluatorFixtureOf> + TestPQEvaluatorPairByPairFixture; TEST_CASE_METHOD(TestPQEvaluatorFixture, "TestPQEvaluator::test_initial_evaluator_type_used", @@ -187,6 +196,40 @@ TEST_CASE_METHOD(TestPQEvaluatorFixture, CHECK(!allclose(gb1, gb2)); } +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_PDF_copies_of_calculator", + "[PQEvaluator]") { + // A copy of a calculator has an evaluator of its own. + for (PQEvaluatorType tp : {OPTIMIZED, PQEvaluatorType::CHECK, BASIC}) { + INFO("evaluator " << tp); + PDFCalculator pdfc; + pdfc.setEvaluatorType(tp); + PDFCalculator early = pdfc; + const QuantityType v10 = pdfc.eval(mstru10); + CHECK(tp == early.getEvaluatorType()); + CHECK(NONE == early.getEvaluatorTypeUsed()); + CHECK(v10 == early.eval(mstru10)); + CHECK(BASIC == early.getEvaluatorTypeUsed()); + // a copy with a value and its assignment + PDFCalculator late = pdfc; + PDFCalculator assigned; + assigned = pdfc; + const QuantityType v9 = pdfc.eval(mstru9); + CHECK(!allclose(v9, v10)); + for (PDFCalculator* c : {&late, &assigned}) { + CHECK(tp == c->getEvaluatorType()); + CHECK(v10 == c->value()); + CHECK(v10 == c->eval(mstru10)); + CHECK(BASIC == c->getEvaluatorTypeUsed()); + CHECK(allclose(v9, c->eval(mstru9))); + if (tp != BASIC) CHECK(tp == c->getEvaluatorTypeUsed()); + } + // the calculator is as it was + CHECK(v9 == pdfc.eval(mstru9)); + if (tp != BASIC) CHECK(tp == pdfc.getEvaluatorTypeUsed()); + } +} + TEST_CASE_METHOD(TestPQEvaluatorFixture, "TestPQEvaluator::test_PDF_reverse_atoms", "[PQEvaluator]") { mpdfco.eval(mstru10); @@ -228,8 +271,10 @@ TEST_CASE_METHOD(TestPQEvaluatorFixture, "TestPQEvaluator::test_PDF_type_mask", CHECK(!allclose(gb0, gb2)); } -TEST_CASE_METHOD(TestPQEvaluatorFixture, "TestPQEvaluator::test_PDF_pair_mask", - "[PQEvaluator]") { +TEST_CASE_METHOD(TestPQEvaluatorPairByPairFixture, + "TestPQEvaluator::test_PDF_pair_mask", "[PQEvaluator]") { + // The fast updates below are of pairs that are added one by one. A sum + // of all pairs takes less work for some changes of this small structure. mpdfcb.setPairMask(0, 3, false); mpdfco.setPairMask(0, 3, false); CHECK(mzeros == this->pdfcdiff(mstru10)); @@ -365,6 +410,152 @@ TEST_CASE_METHOD(TestPQEvaluatorFixture, "TestPQEvaluator::test_failed_update", } } +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_PDF_unchanged_structure", + "[PQEvaluator]") { + // New structures of the same content keep the value, and later changes + // are still found. + mpdfco.eval(mstru10); + const QuantityType v10 = mpdfco.value(); + for (int i = 0; i < 3; ++i) { + StructureAdapterPtr copy = mstru10->clone(); + mpdfco.eval(copy); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + CHECK(copy == mpdfco.getStructure()); + CHECK(v10 == mpdfco.value()); + } + CHECK(allclose(mzeros, this->pdfcdiff(mstru10d1))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + CHECK(allclose(mzeros, this->pdfcdiff(mstru10d1->clone()))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + CHECK(allclose(mzeros, this->pdfcdiff(mstru10))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); +} + +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_PDF_unchanged_structure_new_baseline", + "[PQEvaluator]") { + // The slope of a linear baseline comes from the structure at every + // evaluation, also when the structure did not change. + PeriodicStructureAdapterPtr stru(new PeriodicStructureAdapter); + stru->setLatPar(11.0, 5.0, 6.0, 90.0, 90.0, 90.0); + stru->assign(mstru10->begin(), mstru10->end()); + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + const QuantityType g0 = mpdfco.getPDF(); + mpdfcb.setBaselineByType("linear"); + mpdfco.setBaselineByType("linear"); + CHECK(allclose(mzeros, this->pdfcdiff(stru->clone()))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + CHECK(allclose(g0, mpdfco.getPDF())); +} + +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_PDF_structure_kind", "[PQEvaluator]") { + // A periodic structure differs from a cluster of the same atoms. + PeriodicStructureAdapterPtr periodic(new PeriodicStructureAdapter); + periodic->setLatPar(11.0, 5.0, 6.0, 90.0, 90.0, 90.0); + periodic->assign(mstru10->begin(), mstru10->end()); + CrystalStructureAdapterPtr crystal(new CrystalStructureAdapter); + crystal->setLatPar(11.0, 5.0, 6.0, 90.0, 90.0, 90.0); + crystal->assign(mstru10->begin(), mstru10->end()); + crystal->addSymOp(R3::identity(), R3::zerovector); + crystal->addSymOp(R3::identity(), R3::Vector(0.0, 0.5, 0.5)); + const StructureAdapterPtr strus[] = {mstru10, periodic, crystal, mstru10, + crystal, periodic, mstru10}; + QuantityType glast; + for (const StructureAdapterPtr& stru : strus) { + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + const QuantityType g = mpdfcb.getPDF(); + CHECK(!allclose(glast, g)); + glast = g; + } +} + +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_PDF_derived_structure_class", + "[PQEvaluator]") { + // A class that adds nothing to a built-in adapter, as one defined in + // Python, has copies of the built-in class and keeps its fast updates. + struct DerivedPeriodic : PeriodicStructureAdapter {}; + struct DerivedCrystal : CrystalStructureAdapter {}; + std::shared_ptr periodic(new DerivedPeriodic); + periodic->setLatPar(11.0, 5.0, 6.0, 90.0, 90.0, 90.0); + periodic->assign(mstru10->begin(), mstru10->end()); + std::shared_ptr crystal(new DerivedCrystal); + crystal->setLatPar(11.0, 5.0, 6.0, 90.0, 90.0, 90.0); + crystal->assign(mstru10->begin(), mstru10->end()); + crystal->addSymOp(R3::identity(), R3::zerovector); + crystal->addSymOp(R3::identity(), R3::Vector(0.0, 0.5, 0.5)); + const std::shared_ptr strus[] = {periodic, crystal}; + for (const auto& stru : strus) { + REQUIRE(typeid(*stru) != typeid(*stru->clone())); + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + CHECK(BASIC == mpdfco.getEvaluatorTypeUsed()); + const QuantityType g0 = mpdfcb.getPDF(); + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + stru->at(3).xyz_cartn[1] += 0.2; + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + CHECK(!allclose(g0, mpdfcb.getPDF())); + } +} + +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_PDF_structure_kind_in_wrappers", + "[PQEvaluator]") { + // Wrappers compare their source structures. + PeriodicStructureAdapterPtr periodic(new PeriodicStructureAdapter); + periodic->setLatPar(11.0, 5.0, 6.0, 90.0, 90.0, 90.0); + periodic->assign(mstru10->begin(), mstru10->end()); + const StructureAdapterPtr strus[] = {nometa(mstru10), nometa(periodic), + nometa(mstru10), nosymmetry(periodic), + nosymmetry(mstru10)}; + QuantityType glast; + for (const StructureAdapterPtr& stru : strus) { + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + const QuantityType g = mpdfcb.getPDF(); + glast = g; + } + CHECK(!glast.empty()); +} + +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_crystal_site_changes", + "[PQEvaluator]") { + // Changes of single sites of a crystal, which expand only their own + // symmetry positions again. + mpdfcb.setRmax(7.0); + mpdfco.setRmax(7.0); + mzeros.assign(mpdfcb.getRgrid().size(), 0.0); + CrystalStructureAdapterPtr stru = inversionCrystal(0.3); + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + QuantityType glast = mpdfcb.getPDF(); + const std::function changes[] = { + [](CrystalStructureAdapter& s) { s.at(2).xyz_cartn[1] += 0.1; }, + [](CrystalStructureAdapter& s) { s.at(3).uij_cartn *= 1.5; }, + [](CrystalStructureAdapter& s) { s.at(4).occupancy = 0.5; }, + // to a special position and away from it + [](CrystalStructureAdapter& s) { s.at(0).xyz_cartn[0] = 0.0; }, + [](CrystalStructureAdapter& s) { s.at(0).xyz_cartn[0] = 0.2; }, + [](CrystalStructureAdapter& s) { s.at(1).xyz_cartn[2] -= 0.1; }, + }; + bool inplace = false; + for (const auto& change : changes) { + // every second change is made in place + if (!inplace) { + stru = std::dynamic_pointer_cast(stru->clone()); + } + inplace = !inplace; + change(*stru); + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + const QuantityType g = mpdfcb.getPDF(); + CHECK(!allclose(glast, g)); + glast = g; + } +} + TEST_CASE_METHOD(TestPQEvaluatorFixture, "TestPQEvaluator::test_structure_changed_in_place", "[PQEvaluator]") { diff --git a/src/tests/TestPairTable.cpp b/src/tests/TestPairTable.cpp new file mode 100644 index 0000000..16ee71d --- /dev/null +++ b/src/tests/TestPairTable.cpp @@ -0,0 +1,1028 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + ****************************************************************************** + * + * TestPairTable -- calculators that add all pairs of a structure at once + * + * The pairs must be those of the bond generators and the results those of + * the pairs added one by one, within rounding errors. + * + *****************************************************************************/ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if __has_include() +#define LIBDIFFPY_TEST_PAIR_TABLE +#include +#include +#endif + +#include "pair_test_helpers.hpp" + +using namespace diffpy::srreal; +using namespace pairtests; +#ifdef LIBDIFFPY_TEST_PAIR_TABLE +using kernels::PairGroups; +using kernels::PairTable; +#endif + +namespace { + +const double TOLERANCE = 1e-12; + +// Pairs --------------------------------------------------------------------- + +#ifdef LIBDIFFPY_TEST_PAIR_TABLE + +struct Pair { + int site0; + int site1; + double dist; + double msd; + double weight; +}; + +/// Pairs from the bond generator, as the evaluator adds them one by one. +std::vector generatedPairs(const StructureAdapter& stru, double rmin, + double rmax) { + std::vector rv; + BaseBondGeneratorPtr bnds = stru.createBondGenerator(); + bnds->setRmin(rmin); + bnds->setRmax(rmax); + for (int i0 = 0; i0 < stru.countSites(); ++i0) { + bnds->selectAnchorSite(i0); + bnds->selectSiteRange(0, i0 + 1); + for (bnds->rewind(); !bnds->finished(); bnds->next()) { + const int scale = (i0 == bnds->site1()) ? 1 : 2; + rv.push_back({i0, bnds->site1(), bnds->distance(), bnds->msd(), + double(scale * bnds->multiplicity())}); + } + } + return rv; +} + +std::vector tablePairs(PairTable& table, const StructureAdapter& stru, + double rmin, double rmax) { + std::vector rv; + const auto g = table.geometryArgs(stru, rmin, rmax); + for (long long i = 0; i < kernels::pairCount(g); ++i) { + const auto p = kernels::pairAt(g, i); + if (p.ok) rv.push_back({p.site0, p.site1, p.dist, p.msd, double(p.weight)}); + } + return rv; +} + +/// The pairs are the same within rounding errors. Distances that differ +/// in their last bits may sort differently, so the distances and the +/// displacements of every pair of sites are compared in their own orders. +void checkSamePairs(const std::vector& expected, + const std::vector& pairs) { + REQUIRE(expected.size() == pairs.size()); + auto sorted = [](std::vector v, double Pair::* member) { + std::sort(v.begin(), v.end(), [=](const Pair& a, const Pair& b) { + return std::tie(a.site0, a.site1, a.weight, a.*member) < + std::tie(b.site0, b.site1, b.weight, b.*member); + }); + return v; + }; + int mismatches = 0; + double maxerror = 0.0; + for (double Pair::* member : {&Pair::dist, &Pair::msd}) { + const std::vector e = sorted(expected, member); + const std::vector p = sorted(pairs, member); + for (size_t i = 0; i < p.size(); ++i) { + mismatches += (e[i].site0 != p[i].site0) || (e[i].site1 != p[i].site1) || + (e[i].weight != p[i].weight); + const double error = std::fabs(e[i].*member - p[i].*member); + maxerror = std::max(maxerror, error / std::fabs(e[i].*member)); + } + } + CHECK(0 == mismatches); + CHECK(maxerror <= TOLERANCE); +} + +#endif + +// Calculators --------------------------------------------------------------- + +/// Value and results of the calculator after its setup. +template +std::vector resultsOf(C&& calc, const Setup& setup, + StructureAdapterPtr s) { + calc.setEvaluatorType(BASIC); + setup(calc); + calc.eval(s); + return {calc.value(), calc.getPDF(), calc.getF()}; +} + +/// Check that a calculator of the class T adds all pairs at once and gives +/// what it does with the pairs one by one. +template +void checkAllPairs(const Setup& setup, StructureAdapterPtr s) { + const auto expected = resultsOf(PairByPair(), setup, s); + AllPairs calc; + const auto results = resultsOf(calc, setup, s); + CHECK(calc.addedAllPairs()); + REQUIRE(expected.size() == results.size()); + // the structure has some pairs + REQUIRE(*std::ranges::max_element(expected[0]) > 0.0); + for (size_t i = 0; i < results.size(); ++i) { + CHECK(maxRelativeError(results[i], expected[i]) <= TOLERANCE); + } + const auto builtin = resultsOf(T(), setup, s); + if constexpr (std::is_same_v) { + CHECK(results[0] == builtin[0]); + // Normalization may round the occupied scattering factors differently. + for (size_t i = 1; i < results.size(); ++i) { + CHECK(maxRelativeError(results[i], builtin[i]) <= TOLERANCE); + } + } else { + CHECK(results == builtin); + } +} + +/// Check that a calculator of the class T adds the pairs one by one. +template +void checkPairByPair(const Setup& setup, StructureAdapterPtr s) { + const auto expected = resultsOf(PairByPair(), setup, s); + REQUIRE(*std::ranges::max_element(expected[0]) > 0.0); + AllPairs calc; + CHECK(expected == resultsOf(calc, setup, s)); + CHECK(!calc.addedAllPairs()); + CHECK(expected == resultsOf(T(), setup, s)); +} + +double maxAbsolute(const QuantityType& v) { + double rv = 0.0; + for (double x : v) rv = std::max(rv, std::fabs(x)); + return rv; +} + +} // namespace + +// Tests --------------------------------------------------------------------- + +#ifdef LIBDIFFPY_TEST_PAIR_TABLE +TEST_CASE("TestPairTable::test_pair_groups", "[PairTable]") { + PairGroups groups; + CHECK(groups.groups().empty()); + const double d = 2.5; + const double dnext = std::nextafter(d, 3.0); + groups.add(d, 0.1, 0, 1.0); + groups.add(d, 0.1, 0, 2.0); + // Adjacent representable inputs can fall on opposite sides of a cutoff. + groups.add(dnext, 0.1, 0, 4.0); + groups.add(d, std::nextafter(0.1, 1.0), 0, 8.0); + REQUIRE(3u == groups.groups().size()); + CHECK(d == groups.groups()[0].dist); + CHECK(0.1 == groups.groups()[0].width); + CHECK(3.0 == groups.groups()[0].scale); + CHECK(4.0 == groups.groups()[1].scale); + CHECK(8.0 == groups.groups()[2].scale); + groups.add(d * (1 + 1e-12), 0.1, 0, 1.0); + groups.add(d, 0.1 * (1 + 1e-12), 0, 1.0); + groups.add(d, 0.1, 1, 1.0); + groups.add(-d, 0.1, 0, 1.0); + groups.add(d, 0.0, 0, 1.0); + CHECK(8u == groups.groups().size()); + CHECK(3.0 == groups.groups()[0].scale); + // many groups keep their sums + groups.clear(); + CHECK(groups.groups().empty()); + const int n = 5000; + for (int repeat = 0; repeat < 3; ++repeat) { + for (int i = 0; i < n; ++i) groups.add(1.0 + i, 0.1, i % 7, i); + } + REQUIRE(size_t(n) == groups.groups().size()); + for (int i = 0; i < n; ++i) { + const PairGroups::Group& g = groups.groups()[i]; + CHECK((1.0 + i == g.dist && i % 7 == g.tag && 3.0 * i == g.scale)); + } +} + +#endif + +TEST_CASE("TestPairTable::test_debye_cutoff_of_adjacent_distances", + "[PairTable]") { + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "H"; + atom.occupancy = std::nextafter(0.001, 1.0); + stru->append(atom); + atom.xyz_cartn[0] = 2.0; + stru->append(atom); + atom.xyz_cartn = R3::Vector(0, 30, 0); + stru->append(atom); + atom.xyz_cartn[0] = 2.0 + 4 * (std::nextafter(2.0, 3.0) - 2.0); + stru->append(atom); + DebyePDFCalculator grouped; + PairByPair individual; + for (DebyePDFCalculator* calc : + {&grouped, static_cast(&individual)}) { + calc->setRmax(5); + calc->setMaxExtension(0); + calc->setQstep(0.1); + calc->setScatteringFactorTableByType("neutron"); + calc->getScatteringFactorTable()->setCustomAs("H", "H", 1.0); + } + testAllKernels([&] { + const QuantityType reference = individual.eval(stru); + REQUIRE(maxAbsolute(reference) > 0); + CHECK(maxRelativeError(grouped.eval(stru), reference) <= TOLERANCE); + }); +} + +TEST_CASE("TestPairTable::test_ordered_scattering_at_cutoff", "[PairTable]") { + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "Ni"; + atom.occupancy = 0.0002; + stru->append(atom); + atom.xyz_cartn[0] = 3.0; + atom.occupancy = 7.069469318503158e-05; + stru->append(atom); + DebyePDFCalculator batch; + PairByPair individual; + for (DebyePDFCalculator* c : + {&batch, static_cast(&individual)}) { + c->setEvaluatorType(BASIC); + c->setScatteringFactorTableByType("neutron"); + c->setRmax(8); + } + // Reversing the site factors rounds the first term above the cutoff. + testAllKernels([&] { + CHECK(batch.eval(stru) == individual.eval(stru)); + CHECK(maxAbsolute(batch.value()) == 0.0); + }); +} + +TEST_CASE("TestPairTable::test_debye_width_at_cutoff", "[PairTable]") { + auto stru = std::make_shared(); + Atom atom; + stru->append(atom); + atom.xyz_cartn[0] = 2; + stru->append(atom); + BaseDebyeSum batch; + PairByPair individual; + for (BaseDebyeSum* c : {&batch, static_cast(&individual)}) { + c->setEvaluatorType(BASIC); + c->setRmax(3); + c->setQmin(18); + c->setQmax(19); + c->setQstep(1); + c->setPeakWidthModelByType("constant"); + c->getPeakWidthModel()->setDoubleAttr("width", 0.63); + // Midway between the cutoffs from multiplying and dividing by the + // Gaussian conversion constants. + c->setDebyePrecision(9.20959530020065e-6); + } + KernelGuard guard; + for (const auto& name : availableCalculationKernels()) { + INFO(name); + setCalculationKernel(name); + CHECK(individual.eval(stru).at(18) == 0.0); + CHECK(batch.eval(stru).at(18) == 0.0); + } +} + +TEST_CASE("TestPairTable::test_wrapped_cropped_peak", "[PairTable]") { + auto stru = std::make_shared(); + stru->setLatPar(8, 20, 20, 90, 90, 90); + Atom atom; + atom.atomtype = "Ni"; + atom.xyz_cartn[0] = 7.99; + stru->append(atom); + atom.xyz_cartn[0] = 2.02; + stru->append(atom); + PDFCalculator cached; + cached.setEvaluatorType(BASIC); + cached.setRmax(4); + cached.setMaxExtension(0); + cached.setPeakWidthModelByType("constant"); + cached.setDoubleAttr("width", 0.2); + cached.setPeakProfileByType("croppedgaussian"); + cached.setDoubleAttr("peakprecision", 0.5); + PDFCalculator fresh = cached; + cached.eval(stru); + (*stru)[0].xyz_cartn[0] = 0.02; + CHECK(cached.eval(stru) == fresh.eval(stru)); +} + +TEST_CASE("TestPairTable::test_incremental_update_after_reordering", + "[PairTable]") { + const auto original = asCluster(*loadPeriodic("CaTiO3.stru")); + auto reversed = std::make_shared(); + reversed->assign(original->rbegin(), original->rend()); + auto moved = std::make_shared(*reversed); + (*moved)[2].xyz_cartn[2] += 0.09; + auto check = [&](auto& calc) { + calc.setRmax(8); + calc.eval(original); + calc.eval(reversed); + REQUIRE(calc.getEvaluatorTypeUsed() == OPTIMIZED); + const QuantityType incremental = calc.eval(moved); + REQUIRE(calc.getEvaluatorTypeUsed() == OPTIMIZED); + calc.setEvaluatorType(BASIC); + CHECK(maxRelativeError(incremental, calc.eval(moved)) <= TOLERANCE); + }; + PDFCalculator pdf; + DebyePDFCalculator debye; + check(pdf); + check(debye); +} + +#ifdef LIBDIFFPY_TEST_PAIR_TABLE +TEST_CASE("TestPairTable::test_pairs_of_bond_generator", "[PairTable]") { + const std::pair ranges[] = { + {0.0, 9.7}, {5.0, 4.0}, {0.0, 0.0}, {6.1, 6.9}, {2.0, 12.0}}; + for (const auto& [name, stru] : allStructures()) { + INFO(name); + CHECK(PairTable::supports(*stru)); + PairTable table; + for (const auto& [rmin, rmax] : ranges) { + INFO("range " << rmin << " to " << rmax); + const std::vector expected = generatedPairs(*stru, rmin, rmax); + checkSamePairs(expected, tablePairs(table, *stru, rmin, rmax)); + // and again from the remembered pairs + checkSamePairs(expected, tablePairs(table, *stru, rmin, rmax)); + } + } +} + +TEST_CASE("TestPairTable::test_unsupported_structures", "[PairTable]") { + struct DerivedPeriodic : PeriodicStructureAdapter {}; + struct DerivedAtomic : AtomicStructureAdapter {}; + const auto periodic = loadPeriodic("Ni.stru"); + auto derived = std::make_shared(); + derived->setLatPar(3.52, 3.52, 3.52, 90, 90, 90); + derived->assign(periodic->begin(), periodic->end()); + CHECK(!PairTable::supports(*derived)); + CHECK(!PairTable::supports(*nometa(derived))); + CHECK(!PairTable::supports(*nosymmetry(derived))); + auto cluster = std::make_shared(); + cluster->assign(periodic->begin(), periodic->end()); + CHECK(!PairTable::supports(*cluster)); + // structures without sites have no pairs + CHECK(PairTable::supports(*emptyStructureAdapter())); + CHECK(PairTable::supports(*std::make_shared())); + PairTable table; + CHECK(tablePairs(table, *emptyStructureAdapter(), 0.0, 10.0).empty()); + // The calculators add the pairs of such structures one by one. + checkPairByPair(setupDefault, derived); + checkPairByPair(setupDefault, cluster); +} + +TEST_CASE("TestPairTable::test_remembered_pairs", "[PairTable]") { + // Small changes of a periodic structure keep its pairs, large ones or + // a longer range search them again. + for (const char* kind : {"periodic", "crystal"}) { + INFO(kind); + const auto periodic = loadPeriodic("CaTiO3.stru"); + std::shared_ptr stru = periodic; + // The first sites of the crystal are at special positions, where a + // site that moves gets more symmetry positions and other pairs. + int i0 = 0; + if (std::string(kind) == "crystal") { + stru = asCrystal(*periodic); + i0 = 4; + } + PairTable table; + const double rmin = 0.0, rmax = 8.0; + auto check = [&](int added = -1) { + const int previous = table.countSearches(); + checkSamePairs(generatedPairs(*stru, rmin, rmax), + tablePairs(table, *stru, rmin, rmax)); + if (added >= 0) CHECK(previous + added == table.countSearches()); + }; + check(1); + check(0); + // changes within the margin of the search + stru->at(i0 + 1).xyz_cartn[0] += 0.05; + // An equivalent site of the crystal crosses the unit cell here. + check(stru == periodic ? 0 : 1); + stru->at(i0).xyz_cartn[2] += 0.03; + stru->at(i0 + 1).uij_cartn *= 1.3; + stru->at(i0 + 2).occupancy = 0.5; + check(0); + const Lattice L = stru->getLattice(); + for (Atom& a : *stru) stru->toFractional(a); + stru->setLatPar(1.0005 * L.a(), 0.9995 * L.b(), L.c(), L.alpha(), 90.02, + L.gamma()); + for (Atom& a : *stru) stru->toCartesian(a); + check(); + // an atom that crosses the unit cell appears at its other side + const R3::Vector xyz0 = stru->at(i0).xyz_cartn; + stru->at(i0).xyz_cartn = + xyz0 - stru->getLattice().cartesian(R3::Vector(0.0, 0.0, 1e-5)); + check(); + stru->at(i0).xyz_cartn -= R3::Vector(0.0, 0.0, 0.06); + check(); + stru->at(i0).xyz_cartn = xyz0; + check(); + // a shorter range needs no search, a longer one does + const int beforeRangeChange = table.countSearches(); + checkSamePairs(generatedPairs(*stru, 1.0, 6.5), + tablePairs(table, *stru, 1.0, 6.5)); + CHECK(beforeRangeChange == table.countSearches()); + checkSamePairs(generatedPairs(*stru, 0.0, 9.0), + tablePairs(table, *stru, 0.0, 9.0)); + CHECK(beforeRangeChange + 1 == table.countSearches()); + check(0); + // changes beyond the margin + stru->at(i0 + 1).xyz_cartn[1] += 0.4; + check(1); + for (Atom& a : *stru) stru->toFractional(a); + stru->setLatPar(1.03 * L.a(), L.b(), L.c(), L.alpha(), L.beta(), L.gamma()); + for (Atom& a : *stru) stru->toCartesian(a); + check(1); + // A site that moves by less than the margin can still bring pairs + // from beyond the margin into the range, with the site that moves + // the other way. + stru->at(i0 + 1).xyz_cartn[2] += 0.2; + check(1); + stru->at(i0 + 2).xyz_cartn[2] -= 0.1; + check(0); + stru->at(i0 + 2).xyz_cartn[2] -= 0.1; + check(1); + // other sites + stru->append(stru->at(i0)); + stru->at(stru->countSites() - 1).xyz_cartn[1] += 1.0; + check(1); + stru->erase(1); + check(1); + // The pairs below the range of a search are not remembered. + PairTable other; + checkSamePairs(generatedPairs(*stru, 4.0, 7.0), + tablePairs(other, *stru, 4.0, 7.0)); + checkSamePairs(generatedPairs(*stru, 3.9, 7.1), + tablePairs(other, *stru, 3.9, 7.1)); + CHECK(1 == other.countSearches()); + checkSamePairs(generatedPairs(*stru, 3.0, 7.0), + tablePairs(other, *stru, 3.0, 7.0)); + CHECK(2 == other.countSearches()); + stru->at(i0).xyz_cartn[0] += 0.07; + checkSamePairs(generatedPairs(*stru, 2.9, 7.0), + tablePairs(other, *stru, 2.9, 7.0)); + CHECK(2 == other.countSearches()); + checkSamePairs(generatedPairs(*stru, 2.85, 7.0), + tablePairs(other, *stru, 2.85, 7.0)); + CHECK(3 == other.countSearches()); + } +} + +TEST_CASE("TestPairTable::test_strained_lattice", "[PairTable]") { + // The pairs are searched again when the changes of the positions and of + // the translations together exceed the margin, even if neither does. + auto stru = std::make_shared(); + stru->setLatPar(3.0, 3.0, 3.0, 90.0, 90.0, 90.0); + Atom a; + a.atomtype = "Ni"; + a.uij_cartn = 0.005 * R3::identity(); + stru->append(a); + a.xyz_cartn = R3::Vector(1.5, 0.0, 0.0); + stru->append(a); + PairTable table; + auto check = [&](double alat, int searches) { + INFO("lattice parameter " << alat); + for (Atom& ai : *stru) stru->toFractional(ai); + stru->setLatPar(alat, 3.0, 3.0, 90.0, 90.0, 90.0); + for (Atom& ai : *stru) stru->toCartesian(ai); + checkSamePairs(generatedPairs(*stru, 0.0, 8.0), + tablePairs(table, *stru, 0.0, 8.0)); + CHECK(searches == table.countSearches()); + }; + check(3.0, 1); + // The second atom is up to three cells from the first one within the + // range: the translation changes by 0.15 and the position by 0.025. + check(3.05, 1); + // by 0.21 and by 0.035, which is 0.28 for the distance of two atoms + check(3.07, 2); + check(3.05, 2); +} + +TEST_CASE("TestPairTable::test_new_neighbors_after_contraction", + "[PairTable]") { + auto stru = std::make_shared(); + stru->append(Atom()); + // The first search finds either no translations or only translations + // along b. Shortening a brings a new pair of neighbors into range. + for (double b : {6.0, 2.0}) { + PairTable table; + stru->setLatPar(6, b, 6, 90, 90, 90); + const auto before = tablePairs(table, *stru, 0, 5); + stru->setLatPar(4.9, b, 6, 90, 90, 90); + const auto expected = generatedPairs(*stru, 0, 5); + REQUIRE(expected.size() > before.size()); + checkSamePairs(expected, tablePairs(table, *stru, 0, 5)); + } +} + +TEST_CASE("TestPairTable::test_retry_interrupted_search", "[PairTable]") { + // A stack adapter cannot supply the shared owner needed by its generator. + PeriodicStructureAdapter stack; + stack.setLatPar(3, 3, 3, 90, 90, 90); + stack.append(Atom{}); + PairTable table; + REQUIRE_THROWS_AS(table.geometryArgs(stack, 0, 3.1), std::bad_weak_ptr); + auto owned = std::make_shared(stack); + const auto expected = generatedPairs(*owned, 0, 3.1); + REQUIRE(expected.size() == 6); + checkSamePairs(expected, tablePairs(table, *owned, 0, 3.1)); +} + +TEST_CASE("TestPairTable::test_near_coincidence_on_a_short_axis", + "[PairTable]") { + auto stru = std::make_shared(); + stru->setLatPar(0.005, 1, 1, 90, 90, 90); + Atom site; + site.xyz_cartn = R3::Vector(0.00125, 0.2, 0.2); + stru->append(site); + site.xyz_cartn[0] += 1.3e-8; + stru->append(site); + PairTable table; + REQUIRE(generatedPairs(*stru, 0, 0.001).empty()); + REQUIRE(tablePairs(table, *stru, 0, 0.001).empty()); + // The dormant pair follows the regular neighbors of the last anchor, + // preserving their assignment to reduction rows. + const auto g = table.geometryArgs(*stru, 0, 0.001); + REQUIRE(g.cntpairs > 0); + const auto last = kernels::pairAt(g, g.cntpairs - 1); + CHECK(last.site0 == 1); + CHECK(last.site1 == 0); + CHECK(last.dist <= g.zeroeps); + stru->at(1).xyz_cartn[0] += 0.0001; + const auto expected = generatedPairs(*stru, 0, 0.001); + REQUIRE(expected.size() == 1); + checkSamePairs(expected, tablePairs(table, *stru, 0, 0.001)); + CHECK(table.countSearches() == 1); +} + +TEST_CASE("TestPairTable::test_coinciding_sites", "[PairTable]") { + // Sites at the same position make no bond until they move apart. + for (const char* kind : {"periodic", "crystal", "cluster"}) { + INFO(kind); + const auto periodic = loadPeriodic("CaTiO3.stru"); + // a second atom on the first site and one a lattice vector away + periodic->append(periodic->at(0)); + periodic->at(periodic->countSites() - 1).atomtype = "Sr"; + Atom away = periodic->at(3); + away.xyz_cartn += periodic->getLattice().cartesian(R3::Vector(1, -2, 0)); + periodic->append(away); + std::shared_ptr stru = periodic; + if (std::string(kind) == "crystal") { + auto crystal = asCrystal(*periodic); + crystal->append(crystal->at(0)); + stru = crystal; + } + if (std::string(kind) == "cluster") stru = asCluster(*periodic); + PairTable table; + auto check = [&] { + checkSamePairs(generatedPairs(*stru, 0.0, 7.0), + tablePairs(table, *stru, 0.0, 7.0)); + }; + check(); + const size_t cntpairs = tablePairs(table, *stru, 0.0, 7.0).size(); + const int last = stru->countSites() - 1; + stru->at(last).xyz_cartn[0] += 0.01; + stru->at(last - 1).xyz_cartn[1] += 0.02; + check(); + CHECK(cntpairs < tablePairs(table, *stru, 0.0, 7.0).size()); + // The sites of the crystal get other symmetry positions and their + // pairs are searched again. + if (std::string(kind) != "crystal") CHECK(1 >= table.countSearches()); + stru->at(last).xyz_cartn[0] -= 0.01; + stru->at(last - 1).xyz_cartn[1] -= 0.02; + check(); + CHECK(cntpairs == tablePairs(table, *stru, 0.0, 7.0).size()); + } +} + +TEST_CASE("TestPairTable::test_special_positions", "[PairTable]") { + // A site that moves to a special position has fewer positions. + auto stru = std::make_shared(); + stru->setLatPar(4.0, 4.5, 5.0, 90.0, 90.0, 90.0); + stru->addSymOp(R3::identity(), R3::zerovector); + stru->addSymOp(-1.0 * R3::identity(), R3::zerovector); + Atom a; + a.atomtype = "Ni"; + a.uij_cartn = 0.005 * R3::identity(); + for (const R3::Vector& xyz : + {R3::Vector(0.3, 0.0, 0.0), R3::Vector(0.9, 0.6, 1.3), + R3::Vector(1.7, 1.2, 0.5)}) { + a.xyz_cartn = xyz; + stru->append(a); + } + PairTable table; + auto check = [&] { + checkSamePairs(generatedPairs(*stru, 0.0, 7.0), + tablePairs(table, *stru, 0.0, 7.0)); + }; + check(); + CHECK(2 == stru->siteMultiplicity(0)); + stru->at(0).xyz_cartn[0] = 0.0; + check(); + CHECK(1 == stru->siteMultiplicity(0)); + stru->at(0).xyz_cartn[0] = 0.01; + check(); + stru->addSymOp(R3::identity(), R3::Vector(0.5, 0.5, 0.5)); + check(); + stru->setSymmetryPrecision(0.5); + check(); +} + +#endif + +TEST_CASE("TestPairTable::test_results_of_calculators", "[PairTable]") { + testAllKernels([] { + forEachCalculatorCase( + [](const Setup& setup, StructureAdapterPtr structure) { + checkAllPairs(setup, structure); + checkAllPairs(setup, structure); + }); + }); +} + +TEST_CASE("TestPairTable::test_peak_profiles", "[PairTable]") { + // Any peak profile is linear in the scale of its peaks. + struct WideGaussian : GaussianProfile { + double operator()(double x, double fwhm) const override { + return this->GaussianProfile::operator()(x, 1.5 * fwhm); + } + }; + const auto periodic = loadPeriodic("CaTiO3.stru"); + const Setup cropped = [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c).setPeakProfileByType("croppedgaussian"); + }; + const Setup wide = [](PairQuantity& c) { + setupDefault(c); + PeakProfilePtr pkf(new WideGaussian); + pkf->setPrecision(1e-5); + dynamic_cast(c).setPeakProfile(pkf); + }; + testAllKernels([&] { + for (StructureAdapterPtr stru : + {StructureAdapterPtr(periodic), + StructureAdapterPtr(asCrystal(*periodic)), + StructureAdapterPtr(asCluster(*periodic))}) { + checkAllPairs(cropped, stru); + checkPairByPair(wide, stru); + } + }); +} + +TEST_CASE("TestPairTable::test_pairs_added_one_by_one", "[PairTable]") { + // A derived class may add its pairs in another way, as these that add + // every pair twice. + struct TwicePDFCalculator : PDFCalculator { + void addPairContribution(const BaseBondGenerator& bnds, + int summationscale) override { + this->PDFCalculator::addPairContribution(bnds, 2 * summationscale); + } + }; + struct TwiceDebyePDFCalculator : DebyePDFCalculator { + void addPairContribution(const BaseBondGenerator& bnds, + int summationscale) override { + this->DebyePDFCalculator::addPairContribution(bnds, 2 * summationscale); + } + }; + const StructureAdapterPtr stru = loadPeriodic("CaTiO3.stru"); + auto twice = [](QuantityType v) { + for (double& x : v) x *= 2; + return v; + }; + const auto once = resultsOf(PDFCalculator(), setupDefault, stru); + const auto doubled = resultsOf(TwicePDFCalculator(), setupDefault, stru); + CHECK(maxRelativeError(doubled[0], twice(once[0])) <= TOLERANCE); + const auto donce = resultsOf(DebyePDFCalculator(), setupDefault, stru); + const auto ddoubled = + resultsOf(TwiceDebyePDFCalculator(), setupDefault, stru); + CHECK(maxRelativeError(ddoubled[0], twice(donce[0])) <= TOLERANCE); + // The other peak width models need the bond of each pair. + struct DerivedWidth : JeongPeakWidth {}; + const Setup derivedwidth = [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c).setPeakWidthModel( + PeakWidthModelPtr(new DerivedWidth)); + }; + checkPairByPair(derivedwidth, stru); + checkPairByPair(derivedwidth, stru); + // One part of a parallel run has a share of the pairs. + const Setup parallel = [](PairQuantity& c) { + setupDefault(c); + c.setupParallelRun(0, 2); + }; + checkPairByPair(parallel, stru); +} + +TEST_CASE("TestPairTable::test_masks", "[PairTable]") { + // The masks leave out the same pairs as for the pairs added one by one. + const std::pair setups[] = { + {"without the pairs of two types", + [](PairQuantity& c) { + setupDefault(c); + c.setTypeMask("Ti4+", "O2-", false); + }}, + {"only the pairs of one type with any other", + [](PairQuantity& c) { + setupDefault(c); + c.maskAllPairs(false); + c.setTypeMask("Ca2+", PairQuantity::ALLATOMSSTR, true); + }}, + {"without some pairs of sites", + [](PairQuantity& c) { + setupDefault(c); + c.setPairMask(0, 5, false); + c.setPairMask(2, 2, false); + c.setPairMask(9, 1, false); + }}, + {"without one site", + [](PairQuantity& c) { + setupDefault(c); + c.setPairMask(4, PairQuantity::ALLATOMSINT, false); + }}, + {"only one site with itself", + [](PairQuantity& c) { + setupDefault(c); + c.maskAllPairs(false); + c.setPairMask(5, 5, true); + }}, + }; + const auto periodic = loadPeriodic("CaTiO3.stru"); + const std::pair structures[] = { + {"periodic", periodic}, {"crystal", asCrystal(*periodic)}}; + for (const auto& [kind, stru] : structures) { + AllPairs pdfc; + AllPairs dpdfc; + setupDefault(pdfc); + setupDefault(dpdfc); + const QuantityType vall = pdfc.eval(stru); + const QuantityType dvall = dpdfc.eval(stru); + for (const auto& [name, setup] : setups) { + INFO(kind << " " << name); + checkAllPairs(setup, stru); + checkAllPairs(setup, stru); + // the mask leaves out some pairs + setup(pdfc); + setup(dpdfc); + CHECK(maxRelativeError(pdfc.eval(stru), vall) > 1e-3); + CHECK(maxRelativeError(dpdfc.eval(stru), dvall) > 1e-3); + pdfc.maskAllPairs(true); + dpdfc.maskAllPairs(true); + } + // A mask of all pairs changes nothing in the sum. + const int cntsites = stru->countSites(); + for (PairQuantity* c : {(PairQuantity*)&pdfc, (PairQuantity*)&dpdfc}) { + c->maskAllPairs(false); + CHECK(maxAbsolute(c->eval(stru)) == 0.0); + for (int i = 0; i < cntsites; ++i) { + for (int j = 0; j <= i; ++j) c->setPairMask(i, j, true); + } + } + CHECK(vall == pdfc.eval(stru)); + CHECK(dvall == dpdfc.eval(stru)); + CHECK(pdfc.addedAllPairs()); + CHECK(dpdfc.addedAllPairs()); + } +} + +TEST_CASE("TestPairTable::test_evaluators", "[PairTable]") { + const StructureAdapterPtr stru = loadPeriodic("CaTiO3.stru"); + const QuantityType expected = + resultsOf(PairByPair(), setupDefault, stru)[0]; + for (PQEvaluatorType tp : {BASIC, OPTIMIZED, PQEvaluatorType::CHECK}) { + INFO("evaluator " << tp); + AllPairs pdfc; + setupDefault(pdfc); + pdfc.setEvaluatorType(tp); + CHECK(!pdfc.addedAllPairs()); + pdfc.eval(stru); + CHECK(pdfc.addedAllPairs()); + CHECK(maxRelativeError(pdfc.value(), expected) <= TOLERANCE); + const QuantityType v = pdfc.value(); + pdfc.eval(stru); + if (tp != BASIC) CHECK(tp == pdfc.getEvaluatorTypeUsed()); + CHECK(v == pdfc.value()); + } +} + +TEST_CASE("TestPairTable::test_fast_updates", "[PairTable]") { + // The OPTIMIZED evaluator updates the sum of all pairs for the sites + // that change, pair by pair. + for (const auto& [name, stru0] : allStructures()) { + INFO(name); + PairByPair basic; + PairByPair dbasic; + AllPairs optimized; + AllPairs doptimized; + for (PairQuantity* c : {(PairQuantity*)&basic, (PairQuantity*)&dbasic}) { + c->setEvaluatorType(BASIC); + } + for (PairQuantity* c : + {(PairQuantity*)&basic, (PairQuantity*)&optimized, + (PairQuantity*)&dbasic, (PairQuantity*)&doptimized}) { + setupDefault(*c); + } + StructureAdapterPtr stru = stru0->clone(); + for (int step = 0; step < 4; ++step) { + INFO("step " << step); + if (step > 0) { + // one changed site of the source structure + StructureAdapter* s = stru.get(); + while (!dynamic_cast(s)) { + if (auto* nm = dynamic_cast(s)) { + s = nm->getSourceStructure().get(); + } else { + s = dynamic_cast(*s) + .getSourceStructure() + .get(); + } + } + auto& atoms = dynamic_cast(*s); + atoms.at(step % atoms.countSites()).xyz_cartn[step % 3] += 0.07; + } + basic.eval(stru); + optimized.eval(stru); + dbasic.eval(stru); + doptimized.eval(stru); + CHECK(maxRelativeError(optimized.getPDF(), basic.getPDF()) <= 1e-10); + CHECK(maxRelativeError(doptimized.getPDF(), dbasic.getPDF()) <= 1e-10); + // the first calculation is of all pairs + if (!step) CHECK(optimized.addedAllPairs()); + if (!step) CHECK(doptimized.addedAllPairs()); + } + } +} + +TEST_CASE("TestPairTable::test_fast_update_or_all_pairs", "[PairTable]") { + // The OPTIMIZED evaluator adds all pairs again when that takes less work + // than the pairs of the changed sites one by one. + struct Case { + const char* structure; + /// sites that move + std::vector sites; + /// evaluator for these sites when the calculator can add all pairs + PQEvaluatorType used; + }; + const Case cases[] = { + // Four sites, the pairs merge to few peaks. + {"Ni.stru", {1}, BASIC}, + // Twenty sites: one of them takes less work by itself, the four of + // one element do not. + {"CaTiO3.stru", {5}, OPTIMIZED}, + {"CaTiO3.stru", {4, 5, 6, 7}, BASIC}, + // No pairs merge without symmetry. + {"alpha_K2Bi8Se13.stru", {3}, OPTIMIZED}, + {"alpha_K2Bi8Se13.stru", {2, 3, 4, 5}, OPTIMIZED}, + }; + for (const Case& c : cases) { + INFO(c.structure << " with " << c.sites.size() << " moved sites"); + const auto stru = loadPeriodic(c.structure); + // calculators that add all pairs at once, then ones that do not + auto check = [&](PairQuantity& basic, PairQuantity& optimized, + PairQuantity& dbasic, PairQuantity& doptimized, + bool pairbypair) { + INFO((pairbypair ? "pair by pair" : "all pairs")); + for (PairQuantity* calc : {&basic, &optimized, &dbasic, &doptimized}) { + setupDefault(*calc); + } + basic.setEvaluatorType(BASIC); + dbasic.setEvaluatorType(BASIC); + optimized.eval(stru); + doptimized.eval(stru); + const PQEvaluatorType used = pairbypair ? OPTIMIZED : c.used; + for (int step = 0; step < 3; ++step) { + for (int i : c.sites) stru->at(i).xyz_cartn[step] += 0.02; + StructureAdapterPtr copy = stru->clone(); + CHECK(maxRelativeError(optimized.eval(copy), basic.eval(copy)) <= + 1e-10); + CHECK(maxRelativeError(doptimized.eval(copy), dbasic.eval(copy)) <= + 1e-10); + CHECK(used == optimized.getEvaluatorTypeUsed()); + CHECK(used == doptimized.getEvaluatorTypeUsed()); + } + // a structure of the same content takes no work in either way + optimized.eval(stru->clone()); + CHECK(OPTIMIZED == optimized.getEvaluatorTypeUsed()); + // a removed site has half the work of a moved one + stru->erase(c.sites[0]); + CHECK(maxRelativeError(optimized.eval(stru), basic.eval(stru)) <= 1e-10); + CHECK(OPTIMIZED == optimized.getEvaluatorTypeUsed()); + stru->insert(c.sites[0], Atom(stru->at(0))); + }; + PDFCalculator basic, optimized; + DebyePDFCalculator dbasic, doptimized; + check(basic, optimized, dbasic, doptimized, false); + PairByPair pbasic, poptimized; + PairByPair pdbasic, pdoptimized; + check(pbasic, poptimized, pdbasic, pdoptimized, true); + } +} + +TEST_CASE("TestPairTable::test_copies_and_archives", "[PairTable]") { + const StructureAdapterPtr stru = loadPeriodic("CaTiO3.stru"); + AllPairs pdfc; + setupDefault(pdfc); + pdfc.eval(stru); + const QuantityType v0 = pdfc.value(); + // a copy has pairs of its own + AllPairs copy = pdfc; + copy.setEvaluatorType(BASIC); + copy.eval(nosymmetry(stru)); + CHECK(copy.addedAllPairs()); + CHECK(maxRelativeError(copy.value(), v0) > 0.01); + pdfc.setEvaluatorType(BASIC); + pdfc.eval(stru); + CHECK(maxRelativeError(pdfc.value(), v0) <= TOLERANCE); + AllPairs assigned; + assigned = pdfc; + assigned.eval(stru); + CHECK(assigned.addedAllPairs()); + CHECK(maxRelativeError(assigned.value(), v0) <= TOLERANCE); + // and so has a calculator from an archive + auto saved = std::make_shared(); + setupDefault(*saved); + saved->eval(stru); + std::stringstream storage(std::ios::in | std::ios::out | std::ios::binary); + diffpy::serialization::oarchive oa(storage, std::ios::binary); + oa << saved; + diffpy::serialization::iarchive ia(storage, std::ios::binary); + std::shared_ptr loaded; + ia >> loaded; + CHECK(loaded->value() == v0); + loaded->setEvaluatorType(BASIC); + loaded->eval(stru); + CHECK(loaded->value() == v0); +} + +TEST_CASE("TestPairTable::test_many_evaluations", "[PairTable]") { + // A refinement of a crystal: positions, displacements and the lattice + // change a little at every evaluation. + const auto periodic = loadPeriodic("CaTiO3.stru"); + for (std::shared_ptr stru : + {periodic, + std::shared_ptr(asCrystal(*periodic))}) { + PairByPair onebyone; + PairByPair donebyone; + AllPairs atonce; + AllPairs datonce; + for (PairQuantity* c : + {(PairQuantity*)&onebyone, (PairQuantity*)&atonce, + (PairQuantity*)&donebyone, (PairQuantity*)&datonce}) { + setupDefault(*c); + c->setEvaluatorType(BASIC); + } + const Lattice L = stru->getLattice(); + for (int k = 0; k < 40; ++k) { + INFO("evaluation " << k); + const int n = stru->countSites(); + stru->at(k % n).xyz_cartn[k % 3] += 0.004 * ((k % 5) - 2); + stru->at((3 * k) % n).uij_cartn *= 1.0 + 0.01 * ((k % 3) - 1); + for (Atom& a : *stru) stru->toFractional(a); + const double f = 1.0 + 1e-4 * ((k % 7) - 3); + stru->setLatPar(f * L.a(), L.b(), L.c() / f, L.alpha(), L.beta(), + L.gamma()); + for (Atom& a : *stru) stru->toCartesian(a); + onebyone.setDoubleAttr("delta2", 0.5 + 0.05 * (k % 4)); + atonce.setDoubleAttr("delta2", 0.5 + 0.05 * (k % 4)); + StructureAdapterPtr copy = stru->clone(); + onebyone.eval(copy); + atonce.eval(copy); + donebyone.eval(copy); + datonce.eval(copy); + CHECK(atonce.addedAllPairs()); + CHECK(datonce.addedAllPairs()); + CHECK(maxRelativeError(atonce.value(), onebyone.value()) <= TOLERANCE); + CHECK(maxRelativeError(datonce.value(), donebyone.value()) <= TOLERANCE); + } + } +} diff --git a/src/tests/TestPeakWidthModel.cpp b/src/tests/TestPeakWidthModel.cpp index 3f3ee0e..4154846 100644 --- a/src/tests/TestPeakWidthModel.cpp +++ b/src/tests/TestPeakWidthModel.cpp @@ -104,6 +104,9 @@ TEST_CASE_METHOD(TestPeakWidthModelFixture, CHECK_THAT(w, WithinAbs(mjepw->calculate(*bnds), meps)); mcjepw->setDelta2(0.75); CHECK_THAT(0.5 * w, WithinAbs(mjepw->calculate(*bnds), meps)); + const DebyeWallerPeakWidth& model = *mcjepw; + CHECK_THAT(model.calculateFromMSD(bnds->distance(), bnds->msd()), + WithinAbs(0.5 * w, meps)); mcjepw->setDelta2(0); mcjepw->setQbroad(sqrt(3)); mcjepw->setQbroad_seperable(0); diff --git a/src/tests/TestThreads.cpp b/src/tests/TestThreads.cpp new file mode 100644 index 0000000..22f09ca --- /dev/null +++ b/src/tests/TestThreads.cpp @@ -0,0 +1,165 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + ****************************************************************************** + * + * TestThreads -- calculators that evaluate in several threads at once + * + * Catch2 assertions are not thread-safe, the threads only collect results + * and the main thread checks them. + * + *****************************************************************************/ + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "test_helpers.hpp" + +using namespace diffpy::srreal; + +namespace { + +const int NTHREADS = 4; + +/// Results of a short refinement-like sequence of evaluations. The job +/// number selects the calculator and makes the sequences differ. +std::vector evaluateSequence(int job, StructureAdapterPtr stru0) { + PDFCalculator pdfc; + DebyePDFCalculator dpdfc; + const bool usepdf = (job % 2 == 0); + PairQuantity& calc = usepdf ? static_cast(pdfc) + : static_cast(dpdfc); + calc.setDoubleAttr("rmax", 8.0 + job); + calc.setDoubleAttr("qmax", 20.0); + std::vector rv; + for (int k = 0; k < 6; ++k) { + // The first evaluation reads stru0 itself, which other threads may + // read too. The later ones get a new structure with a moved atom and + // changed calculator attributes. + StructureAdapterPtr stru = stru0; + if (k > 0) { + stru = stru0->clone(); + auto& atoms = dynamic_cast(*stru); + atoms[0].xyz_cartn[0] += 0.01 * k * (job + 1); + } + if (k == 3) calc.setDoubleAttr("delta2", 1.0 + 0.1 * job); + if (k == 4) calc.setDoubleAttr("scale", 1.5); + if (k == 5) calc.setDoubleAttr("qmax", 17.0 + job); + calc.eval(stru); + rv.push_back(usepdf ? pdfc.getPDF() : dpdfc.getPDF()); + rv.push_back(usepdf ? pdfc.getF() : dpdfc.getF()); + } + return rv; +} + +/// Cubic crystal with a face-centered Ni site and a body-centered Cu site. +std::shared_ptr makeCrystal() { + auto stru = std::make_shared(); + stru->setLatPar(3.52, 3.52, 3.52, 90, 90, 90); + const double centering[4][3] = { + {0.0, 0.0, 0.0}, {0.0, 0.5, 0.5}, {0.5, 0.0, 0.5}, {0.5, 0.5, 0.0}}; + for (const auto& t : centering) { + stru->addSymOp(R3::identity(), R3::Vector(t[0], t[1], t[2])); + } + Atom a; + a.atomtype = "Ni"; + a.uij_cartn(0, 0) = a.uij_cartn(1, 1) = a.uij_cartn(2, 2) = 0.004; + stru->append(a); + a.atomtype = "Cu"; + a.occupancy = 0.5; + a.xyz_cartn = R3::Vector(0.5, 0.5, 0.5); + stru->toCartesian(a); + stru->append(a); + return stru; +} + +/// Call fn(job) in NTHREADS threads that start together. +template +void runInThreads(Fn fn) { + std::latch start(NTHREADS); + std::vector threads; + try { + for (int job = 0; job < NTHREADS; ++job) { + threads.emplace_back([&, job] { + start.arrive_and_wait(); + fn(job); + }); + } + } catch (...) { + start.count_down(NTHREADS - threads.size()); + throw; + } +} + +} // namespace + +TEST_CASE("TestThreads::test_separate_calculators", "[Threads]") { + const StructureAdapterPtr stru = loadTestPeriodicStructure("CaTiO3.stru"); + std::vector> serial(NTHREADS); + for (int job = 0; job < NTHREADS; ++job) { + serial[job] = evaluateSequence(job, stru->clone()); + } + // Every thread has its own calculator and its own structure. + for (int repeat = 0; repeat < 5; ++repeat) { + std::vector> threaded(NTHREADS); + std::vector strus; + for (int job = 0; job < NTHREADS; ++job) strus.push_back(stru->clone()); + runInThreads( + [&](int job) { threaded[job] = evaluateSequence(job, strus[job]); }); + CHECK(serial == threaded); + } +} + +TEST_CASE("TestThreads::test_shared_structure", "[Threads]") { + std::vector> serial(NTHREADS); + for (int job = 0; job < NTHREADS; ++job) { + serial[job] = evaluateSequence(job, makeCrystal()); + } + // The threads read one crystal, whose symmetry positions are expanded on + // the first use by any of them. + for (int repeat = 0; repeat < 5; ++repeat) { + std::vector> threaded(NTHREADS); + const StructureAdapterPtr shared = makeCrystal(); + runInThreads( + [&](int job) { threaded[job] = evaluateSequence(job, shared); }); + CHECK(serial == threaded); + } +} + +TEST_CASE("TestThreads::test_event_ticker", "[Threads]") { + using diffpy::eventticker::EventTicker; + const int nclicks = 20000; + EventTicker before, after; + std::vector unordered(NTHREADS, 0); + before.click(); + runInThreads([&](int job) { + EventTicker last, current; + for (int i = 0; i < nclicks; ++i) { + current.click(); + if (!(last < current)) ++unordered[job]; + last = current; + } + }); + after.click(); + // No click was lost and every click gave a newer value. + CHECK(before.value().first == after.value().first); + CHECK(long(NTHREADS) * nclicks + 1 == + after.value().second - before.value().second); + CHECK(std::vector(NTHREADS, 0) == unordered); +} diff --git a/src/tests/pair_test_helpers.hpp b/src/tests/pair_test_helpers.hpp new file mode 100644 index 0000000..85294fd --- /dev/null +++ b/src/tests/pair_test_helpers.hpp @@ -0,0 +1,157 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + *****************************************************************************/ + +#ifndef PAIR_TEST_HELPERS_HPP_INCLUDED +#define PAIR_TEST_HELPERS_HPP_INCLUDED + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "test_helpers.hpp" + +namespace pairtests { +using namespace diffpy::srreal; + +using Setup = std::function; + +inline std::shared_ptr loadPeriodic( + const std::string& file, bool varyOccupancy = false) { + auto rv = std::dynamic_pointer_cast( + loadTestPeriodicStructure(file)); + REQUIRE(rv); + for (int i = 0; i < rv->countSites(); ++i) { + Atom& a = rv->at(i); + if (a.uij_cartn == R3::zeromatrix()) { + a.uij_cartn = (0.004 + 0.001 * (i % 3)) * R3::identity(); + } + if (varyOccupancy) a.occupancy = 1.0 - 0.1 * (i % 3); + } + return rv; +} + +inline std::shared_ptr asCluster( + const PeriodicStructureAdapter& stru) { + auto rv = std::make_shared(); + rv->assign(stru.begin(), stru.end()); + return rv; +} + +/// The first half of the atoms with the operations of a space group that +/// suits the lattice. +inline std::shared_ptr asCrystal( + const PeriodicStructureAdapter& stru) { + const Lattice& L = stru.getLattice(); + auto rv = std::make_shared(); + rv->setLatPar(L.a(), L.b(), L.c(), L.alpha(), L.beta(), L.gamma()); + const int n = std::max(1, stru.countSites() / 2); + for (int i = 0; i < n; ++i) rv->append(stru[i]); + rv->addSymOp(R3::identity(), R3::zerovector); + rv->addSymOp(-1.0 * R3::identity(), R3::zerovector); + if (L.alpha() == 90.0 && L.beta() == 90.0) { + R3::Matrix R = R3::identity(); + R(0, 0) = -1.0; + R(1, 1) = -1.0; + rv->addSymOp(R, R3::Vector(0.5, 0.5, 0.0)); + rv->addSymOp(-1.0 * R, R3::Vector(0.5, 0.5, 0.0)); + } + return rv; +} + +inline std::vector> allStructures( + bool varyOccupancy = false) { + std::vector> rv; + for (const char* f : {"ZnS_wurtzite.stru", "CaTiO3.stru", "Ni.stru", + "alpha_K2Bi8Se13.stru"}) { + const auto periodic = loadPeriodic(f, varyOccupancy); + const auto crystal = asCrystal(*periodic); + const std::string name = f; + rv.emplace_back(name + " periodic", periodic); + rv.emplace_back(name + " crystal", crystal); + rv.emplace_back(name + " cluster", asCluster(*periodic)); + rv.emplace_back(name + " nometa periodic", nometa(periodic)); + rv.emplace_back(name + " nosymmetry crystal", nosymmetry(crystal)); + rv.emplace_back(name + " nometa nosymmetry periodic", + nometa(nosymmetry(periodic))); + } + return rv; +} + +inline void setupDefault(PairQuantity& calc) { + calc.setRmax(9.5); + calc.setDoubleAttr("qmax", 22.0); +} + +template +void forEachCalculatorCase(Check check) { + for (const auto& [name, structure] : allStructures(true)) { + INFO(name); + check(setupDefault, structure); + } + const auto perovskite = loadPeriodic("CaTiO3.stru", true); + const std::pair setups[] = { + {"jeong", + [](PairQuantity& c) { + setupDefault(c); + c.setDoubleAttr("delta1", 0.3); + c.setDoubleAttr("delta2", 1.2); + c.setDoubleAttr("qbroad", 0.03); + c.setDoubleAttr("qbroad_seperable", 0.02); + }}, + {"debye-waller", + [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c).setPeakWidthModelByType( + "debye-waller"); + }}, + {"constant", + [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c).setPeakWidthModelByType( + "constant"); + c.setDoubleAttr("width", 0.21); + }}, + {"sharpened peaks", + [](PairQuantity& c) { + setupDefault(c); + c.setDoubleAttr("delta2", 5.5); + }}, + {"rmin", + [](PairQuantity& c) { + setupDefault(c); + c.setRmin(3.0); + c.setRmax(11.0); + }}, + {"neutrons", + [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c) + .setScatteringFactorTableByType("neutron"); + }}, + }; + for (const auto& [name, setup] : setups) { + INFO(name); + check(setup, perovskite); + } +} + +} // namespace pairtests +#endif diff --git a/src/tests/test_helpers.hpp b/src/tests/test_helpers.hpp index 16ab01f..e5054d4 100644 --- a/src/tests/test_helpers.hpp +++ b/src/tests/test_helpers.hpp @@ -50,9 +50,9 @@ class KernelGuard { std::string msaved; }; -/// Call fn() with each available calculation kernel active, then restore the -/// kernel that was active before. Failed checks report their kernel. -/// Do not use SECTION in fn, it would run only with the first kernel. +/// Call fn() with each available CPU kernel active, then restore the kernel +/// that was active before. Do not use SECTION in fn, it would run only with +/// the first kernel. template void testAllKernels(Fn&& fn) { KernelGuard guard; @@ -64,4 +64,39 @@ void testAllKernels(Fn&& fn) { } } +// Sums of all pairs --------------------------------------------------------- + +/// A calculator of the class T that adds its pairs one by one, as the +/// calculators of the library do in every class derived from them. +template +struct PairByPair : T {}; + +/// A calculator of the class T that adds all pairs at once where T does, +/// and tells whether it did. +template +class AllPairs : public T { + public: + /// True when the last full calculation added all pairs at once. + bool addedAllPairs() const { return maddedallpairs; } + + protected: + // every calculation starts with the value reset + void resetValue() override { + maddedallpairs = false; + this->T::resetValue(); + } + + bool addAllPairContributions() override { + if constexpr (requires { this->addAllPairsOfPDF(); }) { + maddedallpairs = this->addAllPairsOfPDF(); + } else { + maddedallpairs = this->addAllPairsOfDebyeSum(); + } + return maddedallpairs; + } + + private: + bool maddedallpairs = false; +}; + #endif // TEST_HELPERS_HPP_INCLUDED From 84df4a1f0c48f1b3146f7e5c31235e6e728e19e1 Mon Sep 17 00:00:00 2001 From: SYMSCAE Date: Wed, 18 Nov 2026 13:13:55 -0800 Subject: [PATCH 2/6] feat: add CUDA Debye and PDF calculation kernels --- .pre-commit-config.yaml | 2 +- CMakeLists.txt | 25 +- CMakePresets.json | 36 ++ cmake/Cuda.cmake | 37 ++ cmake/libdiffpyConfig.cmake.in | 5 + docs/source/development.rst | 22 + news/cuda.rst | 5 + src/diffpy/features.tpl | 4 + src/diffpy/srreal/BaseDebyeSum.cpp | 2 +- src/diffpy/srreal/CalculationKernel.hpp | 8 +- src/diffpy/srreal/PDFCalculator.cpp | 2 +- src/diffpy/srreal/PQEvaluator.cpp | 4 + src/diffpy/srreal/PairQuantity.cpp | 6 +- src/diffpy/srreal/PairQuantity.hpp | 7 +- .../srreal/kernels/CalculationKernelCuda.cu | 586 ++++++++++++++++++ .../kernels/CalculationKernelDispatch.cpp | 82 ++- .../srreal/kernels/CalculationKernelImpl.hpp | 20 +- src/diffpy/srreal/kernels/PairGeometry.hpp | 39 +- src/diffpy/srreal/kernels/PairTable.cpp | 2 +- src/diffpy/srreal/kernels/PairTable.hpp | 2 +- src/tests/CMakeLists.txt | 8 + src/tests/TestCalculationKernel.cpp | 31 +- src/tests/TestCalculationKernelCuda.cpp | 527 ++++++++++++++++ src/tests/test_helpers.hpp | 5 + 24 files changed, 1424 insertions(+), 43 deletions(-) create mode 100644 cmake/Cuda.cmake create mode 100644 src/diffpy/srreal/kernels/CalculationKernelCuda.cu create mode 100644 src/tests/TestCalculationKernelCuda.cpp diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8857347..5ff9500 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,7 +43,7 @@ repos: hooks: - id: clang-format name: clang-format (C/C++) - files: \.(c|cpp|cc|cxx|h|hpp|hh)$ + files: \.(c|cpp|cc|cxx|cu|h|hpp|hh)$ - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: diff --git a/CMakeLists.txt b/CMakeLists.txt index bffee3d..d470998 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ option(LIBDIFFPY_TEST_INSTALLED "Build only the unit tests and link them with the installed libdiffpy" OFF) set(LIBDIFFPY_TESTS "" CACHE STRING "Comma or space separated patterns selecting the Test*.cpp unit test files") +option(LIBDIFFPY_CUDA "Build the CUDA calculation kernels" OFF) option(LIBDIFFPY_COVERAGE "Instrument the build for coverage analysis" OFF) option(LIBDIFFPY_PROFILE "Build with gprof profiling information" OFF) @@ -78,10 +79,13 @@ if(PROJECT_IS_TOP_LEVEL) if(MSVC) # /W3 rather than /Wall, which floods the log with warnings from the # standard library and boost headers. - add_compile_options(/W3) + add_compile_options($<$:/W3>) add_link_options(/OPT:NOREF /OPT:NOICF) else() - add_compile_options(-Wall $<$:-ffast-math>) + # for the C++ sources, nvcc does not take these options + add_compile_options( + $<$:-Wall> + $<$,$>:-ffast-math>) endif() endif() @@ -91,12 +95,12 @@ if(MSVC AND (LIBDIFFPY_COVERAGE OR LIBDIFFPY_PROFILE)) endif() if(LIBDIFFPY_COVERAGE) - add_compile_options(--coverage -fprofile-update=atomic -O0 -g) + add_compile_options("$<$:--coverage;-fprofile-update=atomic;-O0;-g>") add_link_options(--coverage) endif() if(LIBDIFFPY_PROFILE) - add_compile_options(-pg) + add_compile_options($<$:-pg>) add_link_options(-pg) endif() @@ -247,6 +251,11 @@ if(LIBDIFFPY_HAS_OBJCRYST) else() set(DIFFPY_HAS_OBJCRYST 0) endif() +if(LIBDIFFPY_CUDA) + set(DIFFPY_HAS_CUDA 1) +else() + set(DIFFPY_HAS_CUDA 0) +endif() set(_features_header "${_generated_include_dir}/diffpy/features.hpp") configure_file("${PROJECT_SOURCE_DIR}/src/diffpy/features.tpl" "${_features_header}") @@ -343,11 +352,19 @@ if(LIBDIFFPY_HAS_OBJCRYST) target_link_libraries(diffpy PUBLIC ObjCryst::ObjCryst) endif() include(CalculationKernels) +if(LIBDIFFPY_CUDA) + include(Cuda) +endif() # Installation ---------------------------------------------------------------- if(LIBDIFFPY_INSTALL) set(LIBDIFFPY_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/libdiffpy") + if(LIBDIFFPY_CUDA AND NOT _libdiffpy_library_type STREQUAL "SHARED_LIBRARY") + # A static libdiffpy leaves the CUDA kernels to the program that links it. + install(TARGETS diffpy_cuda EXPORT libdiffpyTargets + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT Development) + endif() install(TARGETS diffpy EXPORT libdiffpyTargets RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" COMPONENT Runtime diff --git a/CMakePresets.json b/CMakePresets.json index 28331e0..0873f53 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -26,6 +26,14 @@ "LIBDIFFPY_BUILD_TESTS": "ON" } }, + { + "name": "debug-cuda", + "inherits": "debug", + "displayName": "Debug build with CUDA kernels and unit tests", + "cacheVariables": { + "LIBDIFFPY_CUDA": "ON" + } + }, { "name": "release", "inherits": "ninja", @@ -34,6 +42,14 @@ "CMAKE_BUILD_TYPE": "Release" } }, + { + "name": "release-cuda", + "inherits": "release", + "displayName": "Optimized build with CUDA kernels", + "cacheVariables": { + "LIBDIFFPY_CUDA": "ON" + } + }, { "name": "coverage", "inherits": "debug", @@ -49,11 +65,21 @@ "configurePreset": "debug", "configuration": "Debug" }, + { + "name": "debug-cuda", + "configurePreset": "debug-cuda", + "configuration": "Debug" + }, { "name": "release", "configurePreset": "release", "configuration": "Release" }, + { + "name": "release-cuda", + "configurePreset": "release-cuda", + "configuration": "Release" + }, { "name": "coverage", "configurePreset": "coverage", @@ -72,12 +98,22 @@ "outputOnFailure": true } }, + { + "name": "debug-cuda", + "inherits": "debug", + "configurePreset": "debug-cuda" + }, { "name": "release", "inherits": "debug", "configurePreset": "release", "configuration": "Release" }, + { + "name": "release-cuda", + "inherits": "release", + "configurePreset": "release-cuda" + }, { "name": "coverage", "inherits": "debug", diff --git a/cmake/Cuda.cmake b/cmake/Cuda.cmake new file mode 100644 index 0000000..84e18b6 --- /dev/null +++ b/cmake/Cuda.cmake @@ -0,0 +1,37 @@ +# CUDA calculation kernels, see +# src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp. +# +# The CUDA source makes the static library diffpy_cuda, which the diffpy +# library links. It is compiled by nvcc and statically linked with its +# own C++ standard and without the options of the C++ sources. + +include(CheckLanguage) +check_language(CUDA) +if(NOT CMAKE_CUDA_COMPILER) + message(FATAL_ERROR + "LIBDIFFPY_CUDA needs the CUDA compiler nvcc. Set CMAKE_CUDA_COMPILER " + "or the environment variable CUDACXX to it.") +endif() +enable_language(CUDA) + +add_library(diffpy_cuda STATIC + "${PROJECT_SOURCE_DIR}/src/diffpy/srreal/kernels/CalculationKernelCuda.cu") +set_target_properties(diffpy_cuda PROPERTIES + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + CUDA_EXTENSIONS OFF + CUDA_RUNTIME_LIBRARY Static + POSITION_INDEPENDENT_CODE ON + # The objects of nvcc have no data for link-time optimization. + INTERPROCEDURAL_OPTIMIZATION OFF) +target_include_directories(diffpy_cuda PRIVATE "${PROJECT_SOURCE_DIR}/src") +# Use the CPU kernels' rounding order for distances, widths and grid cutoffs. +target_compile_options(diffpy_cuda PRIVATE + $<$:--fmad=false> + $<$:-ffp-contract=off>) +# The runtime is named here for the programs that link diffpy_cuda through +# a static or OBJECT diffpy library, whose link step is not one of CUDA. +find_package(CUDAToolkit REQUIRED) +target_link_libraries(diffpy_cuda PRIVATE CUDA::cudart_static) +target_link_libraries(diffpy PRIVATE diffpy_cuda) +target_compile_definitions(diffpy PRIVATE LIBDIFFPY_KERNEL_CUDA=1) diff --git a/cmake/libdiffpyConfig.cmake.in b/cmake/libdiffpyConfig.cmake.in index 75330b4..16d4319 100644 --- a/cmake/libdiffpyConfig.cmake.in +++ b/cmake/libdiffpyConfig.cmake.in @@ -8,6 +8,7 @@ # Variables: # # libdiffpy_HAS_OBJCRYST true when libdiffpy was built with ObjCryst +# libdiffpy_HAS_CUDA true when libdiffpy has the CUDA kernels # libdiffpy_RUNTIME_DIR directory of the installed data files @PACKAGE_INIT@ @@ -15,12 +16,16 @@ include(CMakeFindDependencyMacro) set(libdiffpy_HAS_OBJCRYST @LIBDIFFPY_HAS_OBJCRYST@) +set(libdiffpy_HAS_CUDA @LIBDIFFPY_CUDA@) set_and_check(libdiffpy_RUNTIME_DIR "@PACKAGE_LIBDIFFPY_RUNTIME_INSTALL_DIR@") find_dependency(Boost CONFIG COMPONENTS serialization) if(NOT "@_libdiffpy_library_type@" STREQUAL "SHARED_LIBRARY") find_dependency(GSL) find_dependency(Threads) + if(libdiffpy_HAS_CUDA) + find_dependency(CUDAToolkit) + endif() endif() if(libdiffpy_HAS_OBJCRYST) list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") diff --git a/docs/source/development.rst b/docs/source/development.rst index f651b27..18781a8 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -33,7 +33,9 @@ run them in a command prompt with the conda build environment activated. The build presets are * ``debug`` - unoptimized build with debug information and C assertion checks +* ``debug-cuda`` - debug build with CUDA calculation kernels * ``release`` - optimized build with C assertion checks disabled +* ``release-cuda`` - optimized build with CUDA calculation kernels * ``coverage`` - unoptimized build instrumented for coverage analysis The library is built in the ``build/`` directory, for example @@ -61,6 +63,26 @@ The SIMD kernels are ``avx512``, ``avx2`` and ``sse2`` on x86-64 and ``neon`` on arm64. On first use the library picks the best kernel that the CPU supports, for example ``avx2`` on a CPU without AVX-512. +CUDA calculation kernels +------------------------ + +Build and test with the ``debug-cuda`` preset :: + + cmake --preset debug-cuda + cmake --build --preset debug-cuda --parallel 4 + ctest --preset debug-cuda + +Use ``release-cuda`` for an optimized build. To run the tests with either +release preset, pass ``-DLIBDIFFPY_BUILD_TESTS=ON`` when configuring, then +build and run CTest with the same preset. +To select ``nvcc`` or its host C++ compiler, set ``CUDACXX`` or +``CUDAHOSTCXX``, respectively, before the first configure. + +On a compatible GPU, ``availableCalculationKernels`` lists ``cuda`` +(double precision) and ``cuda-mixed`` (single-precision sampling with +double-precision distances and sums). Without a compatible GPU, CPU +kernels remain available and the ``[cuda]`` tests skip. + Embed libdiffpy in another CMake project ---------------------------------------- diff --git a/news/cuda.rst b/news/cuda.rst index 01e345d..de3358d 100644 --- a/news/cuda.rst +++ b/news/cuda.rst @@ -1,5 +1,10 @@ **Added:** +* Build with ``LIBDIFFPY_CUDA=ON`` or either CUDA preset (``debug-cuda``, + ``release-cuda``) to enable the ``cuda`` and ``cuda-mixed`` kernels for full + PDF and Debye sums. Query build support with ``libdiffpy_HAS_CUDA`` (CMake) + or ``DIFFPY_HAS_CUDA`` (C++). + CUDA-enabled builds also run on CPU-only machines. * Derived calculators can implement ``addAllPairContributions`` or reuse ``addAllPairsOfPDF`` and ``addAllPairsOfDebyeSum``. The constant, Debye-Waller and Jeong peak width models now provide ``calculateFromMSD``. diff --git a/src/diffpy/features.tpl b/src/diffpy/features.tpl index 526bdf5..73d4fcf 100644 --- a/src/diffpy/features.tpl +++ b/src/diffpy/features.tpl @@ -25,6 +25,10 @@ # define DIFFPY_HAS_OBJCRYST #endif +#if ${DIFFPY_HAS_CUDA} +# define DIFFPY_HAS_CUDA +#endif + #endif // FEATURES_HPP_INCLUDED // vim:ft=cpp: diff --git a/src/diffpy/srreal/BaseDebyeSum.cpp b/src/diffpy/srreal/BaseDebyeSum.cpp index 9d3c089..7e027aa 100644 --- a/src/diffpy/srreal/BaseDebyeSum.cpp +++ b/src/diffpy/srreal/BaseDebyeSum.cpp @@ -200,7 +200,7 @@ bool BaseDebyeSum::addAllPairsOfDebyeSum() { args.prec = this->getDebyePrecision(); const auto& kernel = this->preparePairSum(args, range.getRmin(), range.getRmax()); - mallpairs.record(kernel.debyePairSum(args)); + mallpairs.record(kernel.debyePairSum(args), kernel.onDevice); return true; } diff --git a/src/diffpy/srreal/CalculationKernel.hpp b/src/diffpy/srreal/CalculationKernel.hpp index 35902a1..7f89610 100644 --- a/src/diffpy/srreal/CalculationKernel.hpp +++ b/src/diffpy/srreal/CalculationKernel.hpp @@ -24,12 +24,12 @@ namespace diffpy { namespace srreal { -/// Names of the kernels that this build and CPU can run, from the most to -/// the least preferred: "avx512", "avx2" and "sse2" on x86-64, "neon" on -/// arm64, and "scalar", which is last and always available. +/// Names of the kernels that this build and machine can run, CPU kernels +/// first in order of preference. A compatible CUDA device adds "cuda" and +/// "cuda-mixed", which only run full PDF and Debye sums. DLL_EXPORT std::vector availableCalculationKernels(); -/// Name of the kernel in use, by default the most preferred one. +/// Name of the kernel in use, by default the first available one. DLL_EXPORT std::string getCalculationKernel(); /// Use the named kernel for all calculators. diff --git a/src/diffpy/srreal/PDFCalculator.cpp b/src/diffpy/srreal/PDFCalculator.cpp index 791f711..2f58b30 100644 --- a/src/diffpy/srreal/PDFCalculator.cpp +++ b/src/diffpy/srreal/PDFCalculator.cpp @@ -454,7 +454,7 @@ bool PDFCalculator::addAllPairsOfPDF() { args.rstep = this->getRstep(); const auto& kernel = this->preparePairSum(args, this->rcalclo(), this->rcalchi()); - mallpairs.record(kernel.pdfPairSum(args)); + mallpairs.record(kernel.pdfPairSum(args), kernel.onDevice); return true; } diff --git a/src/diffpy/srreal/PQEvaluator.cpp b/src/diffpy/srreal/PQEvaluator.cpp index 4d676d0..0688087 100644 --- a/src/diffpy/srreal/PQEvaluator.cpp +++ b/src/diffpy/srreal/PQEvaluator.cpp @@ -88,6 +88,7 @@ void PQEvaluatorBasic::updateValue(PairQuantity& pq, StructureAdapterPtr stru) { mvalue_ticker = eventticker::EventTicker(); pq.setStructure(stru); // A batch covers the complete half sum of an unpartitioned evaluation. + pq.mallpairs.ondevice = false; const bool halfsum = !this->getFlag(USEFULLSUM) && !this->isParallel(); if (halfsum && pq.addAllPairContributions()) { maddedallpairs = true; @@ -289,6 +290,9 @@ bool PQEvaluatorOptimized::allPairsTakeLessWork( // A change of whether the calculator adds all pairs forces a full // calculation anyway. if (!maddedallpairs) return false; + // A sum on the device beats any fast update on the CPU. + const bool changes = !sd.pop0.empty() || !sd.add1.empty(); + if (pq.mallpairs.ondevice) return changes; // share of the pairs that have one of the sites auto pairshare = [](const SiteIndices& sites, const StructureAdapter& s) { if (sites.empty()) return 0.0; diff --git a/src/diffpy/srreal/PairQuantity.cpp b/src/diffpy/srreal/PairQuantity.cpp index 97616a1..89584cf 100644 --- a/src/diffpy/srreal/PairQuantity.cpp +++ b/src/diffpy/srreal/PairQuantity.cpp @@ -280,16 +280,18 @@ const kernels::CalculationKernel& PairQuantity::preparePairSum( mallpairs.pairsum = std::make_shared(); auto& state = *mallpairs.pairsum; args.mask = this->hasMask() ? this : nullptr; - const auto& kernel = kernels::activeCalculationKernel(); + const auto& kernel = kernels::pairSumKernel(args.mask != nullptr); args.g = state.table.geometryArgs(*mstructure, rmin, rmax); args.workspace = state.workspaceFor(kernel); args.out = mvalue.data(); return kernel; } -void PairQuantity::AllPairsState::record(kernels::PairSumStats stats) { +void PairQuantity::AllPairsState::record(kernels::PairSumStats stats, + bool usedDevice) { pairs = stats.pairs; peaks = stats.peaks; + ondevice = usedDevice; } void PairQuantity::configureBondGenerator(BaseBondGenerator& bnds) const { diff --git a/src/diffpy/srreal/PairQuantity.hpp b/src/diffpy/srreal/PairQuantity.hpp index 87b32b9..2e82beb 100644 --- a/src/diffpy/srreal/PairQuantity.hpp +++ b/src/diffpy/srreal/PairQuantity.hpp @@ -127,16 +127,19 @@ class DLL_EXPORT PairQuantity : public diffpy::Attributes { /// once. Neither copied nor serialized. struct AllPairsState { AllPairsState() = default; - void record(kernels::PairSumStats stats); + void record(kernels::PairSumStats stats, bool usedDevice); AllPairsState(const AllPairsState& other) - : pairs(other.pairs), peaks(other.peaks) {} + : ondevice(other.ondevice), pairs(other.pairs), peaks(other.peaks) {} AllPairsState& operator=(const AllPairsState& other) { pairsum.reset(); + ondevice = other.ondevice; pairs = other.pairs; peaks = other.peaks; return *this; } std::shared_ptr pairsum; + /// The last full sum ran on a GPU. + bool ondevice = false; /// Pairs and merged contributions of the last sum of all pairs, from /// which the OPTIMIZED evaluator estimates the work of the next one. double pairs = 0.0; diff --git a/src/diffpy/srreal/kernels/CalculationKernelCuda.cu b/src/diffpy/srreal/kernels/CalculationKernelCuda.cu new file mode 100644 index 0000000..43f1bb1 --- /dev/null +++ b/src/diffpy/srreal/kernels/CalculationKernelCuda.cu @@ -0,0 +1,586 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + ****************************************************************************** + * + * CUDA calculation kernels, which add all pairs of a structure on a device + * + * The device computes the distance and width of every PairTable pair in + * double precision. A periodic pair list stays on the device while its + * neighbor search remains valid. Each output point is summed in a fixed + * order: PDF peaks in per-warp rows, Debye terms in blocks of pairs, then + * the rows or blocks in order. The mixed kernel evaluates peaks and Debye + * terms in single precision from arguments reduced in double precision. + * + *****************************************************************************/ + +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace diffpy { +namespace srreal { +namespace kernels { + +namespace { + +// Device ---------------------------------------------------------------------- + +void check(cudaError_t err, const char* what) { + if (err == cudaSuccess) return; + throw std::runtime_error(std::string("the CUDA device failed in ") + what + + " with '" + cudaGetErrorString(err) + "'"); +} + +int currentDevice() { + int device; + check(cudaGetDevice(&device), "the current device lookup"); + return device; +} + +/// Memory on the device that grows as needed. +template +class Buffer { + public: + Buffer() {} + Buffer(const Buffer&) = delete; + Buffer& operator=(const Buffer&) = delete; + ~Buffer() { + if (mptr) cudaFree(mptr); + } + + T* reserve(size_t n) { + if (n > mcapacity) { + if (mptr) cudaFree(mptr); + mptr = nullptr; + mcapacity = 0; + void* p = nullptr; + check(cudaMalloc(&p, n * sizeof(T)), "the allocation of memory"); + mptr = static_cast(p); + mcapacity = n; + } + return mptr; + } + + T* upload(const T* src, size_t n) { + this->reserve(n); + if (n) { + check(cudaMemcpy(mptr, src, n * sizeof(T), cudaMemcpyHostToDevice), + "the copy of data to it"); + } + return mptr; + } + + T* zero(size_t n) { + this->reserve(n); + if (n) check(cudaMemset(mptr, 0, n * sizeof(T)), "the reset of memory"); + return mptr; + } + + void download(T* dst, size_t n) const { + if (n) { + check(cudaMemcpy(dst, mptr, n * sizeof(T), cudaMemcpyDeviceToHost), + "the copy of data from it"); + } + } + + T* get() const { return mptr; } + + private: + T* mptr = nullptr; + size_t mcapacity = 0; +}; + +// Pairs ----------------------------------------------------------------------- + +/// Maximum pairs in a Debye chunk. +const long long CHUNK = 1LL << 20; +/// Threads of a block. +const int THREADS = 128; +/// Pairs that one thread of the Debye sum adds for its Q point. +const int SUBCHUNK = 1024; + +const double FOUR_LN2 = 2.7725887222397811; +const double TWO_PI = 6.2831853071795865; + +// Peaks of the PDF +// -------------------------------------------------------------- + +struct PeakGrid { + int npts; + int rlosteps; + double rstep; + double height; + int cropped; + double halfboundrel; +}; + +// Keep the original lower grid index as the origin, including when cropping +// removes the first point. This is the coordinate used by the CPU kernels. +struct PeakWindow { + int origin, first, last; + double xlo; + + __device__ double x(int i, double step) const { + return xlo + (i - origin) * step; + } +}; + +__device__ PeakWindow peakWindow(const PeakGrid& grid, double dist, + double fwhm) { + const double half = grid.halfboundrel * fwhm; + const int lo = + gridIndex(dist - half, grid.rstep, grid.rlosteps, 0, grid.npts); + const int hi = + gridIndex(dist + half, grid.rstep, grid.rlosteps, 1, grid.npts); + PeakWindow win{lo, lo, hi, (grid.rlosteps + lo) * grid.rstep - dist}; + if (grid.cropped) { + while (win.first < win.last && + fabs(win.x(win.first, grid.rstep) / fwhm) >= grid.halfboundrel) + ++win.first; + while (win.first < win.last && + fabs(win.x(win.last - 1, grid.rstep) / fwhm) >= grid.halfboundrel) + --win.last; + } + return win; +} + +__device__ double samplePeak(double x, double dist, double fwhm, + double amplitude) { + const double xrel = x / fwhm; + return amplitude * exp(-FOUR_LN2 * xrel * xrel) * (x / dist + 1.0); +} + +enum { WARP = 32 }; +struct Peak { + PeakWindow win; + double dist, fwhm, amplitude, invdist, invwidth; +}; + +/// Each warp groups bit-identical distances and widths, adds their weights +/// in lane order, and accumulates the resulting peaks in its own output row. +template +__global__ void pdfPeaks(PairGeometryArgs g, PairWidthArgs w, + const double* sfsite, PeakGrid grid, long long count, + double* rows) { + __shared__ Peak peaks[THREADS]; + __shared__ double scales[THREADS]; + const int lane = threadIdx.x % WARP; + const int warp = (blockIdx.x * THREADS + threadIdx.x) / WARP; + double* row = rows + (size_t)warp * grid.npts; + Peak* batch = peaks + threadIdx.x - lane; + double* weights = scales + threadIdx.x - lane; + for (long long first = (long long)warp * WARP; first < count; + first += (long long)gridDim.x * THREADS) { + Peak pk{}; + double scale = 0.; + if (first + lane < count) { + const Pair p = pairAt(g, first + lane); + if (p.ok) { + pk.dist = p.dist; + pk.fwhm = widthOf(w, p.dist, p.msd); + scale = sfsite[p.site0] * sfsite[p.site1] * p.weight; + } + } + const unsigned valid = __ballot_sync(0xffffffffu, pk.fwhm > 0.); + const unsigned distMask = + __match_any_sync(0xffffffffu, __double_as_longlong(pk.dist)); + const unsigned widthMask = + __match_any_sync(0xffffffffu, __double_as_longlong(pk.fwhm)); + const unsigned same = valid & distMask & widthMask; + const bool leader = lane == __ffs(same) - 1; + unsigned leaders = __ballot_sync(0xffffffffu, leader); + weights[lane] = scale; + __syncwarp(); + if (leader) { + for (unsigned others = same & (same - 1); others; others &= others - 1) + scale += weights[__ffs(others) - 1]; + pk.win = peakWindow(grid, pk.dist, pk.fwhm); + pk.invdist = 1.0 / pk.dist; + pk.invwidth = 1.0 / pk.fwhm; + pk.amplitude = scale * grid.height / pk.fwhm; + batch[lane] = pk; + } + __syncwarp(); + for (; leaders; leaders &= leaders - 1) { + const Peak p = batch[__ffs(leaders) - 1]; + const int offset = (lane - p.win.first) & (WARP - 1); + for (long long i = (long long)p.win.first + offset; i < p.win.last; + i += WARP) { + const double x = p.win.x(int(i), grid.rstep); + // An unresolved peak can have an unrepresentable inverse width. + if (p.win.last - p.win.first <= 2) { + row[i] += samplePeak(x, p.dist, p.fwhm, p.amplitude); + } else { + using Real = std::conditional_t; + const Real xr = Real(x * p.invwidth); + const Real correction = Real(x) * Real(p.invdist) + Real(1); + row[i] += Real(p.amplitude) * exp(-Real(FOUR_LN2) * xr * xr) * + correction; + } + } + } + __syncwarp(); + } +} + +// Terms of the Debye sum +// -------------------------------------------------------- + +struct QGrid { + int nq; + int kqlo; + int kqhi; + double qstep; + double prec; +}; + +/// The double kernel stores distance in angstroms. The mixed kernel stores +/// the phase step as a fraction of a turn in units of 2^-64. +template +struct DebyePair { + using Phase = std::conditional_t, double, + unsigned long long>; + Phase phase; + Real sigma, amplitude; + int type0, type1, kqstop; +}; + +template +struct DebyeBuffers { + Buffer> pairs; + Buffer factors; +}; + +/// Set up the pairs of a chunk. logsf contains log(|sf|) for each type. +template +__global__ void debyePairs(PairGeometryArgs g, PairWidthArgs w, + const int* typeofsite, const int* curve, + const double* occ, const double* sf, + const float* logsf, QGrid grid, long long first, + long long count, DebyePair* pairs) { + const long long j = (long long)blockIdx.x * blockDim.x + threadIdx.x; + if (j >= count) return; + const Pair p = pairAt(g, first + j); + if (!p.ok) { + pairs[j].kqstop = grid.kqlo; + return; + } + const double sigma = fwhmToSigma(widthOf(w, p.dist, p.msd)); + const int tp0 = typeofsite[p.site0]; + const int tp1 = typeofsite[p.site1]; + const double amplitude = p.weight / p.dist; + DebyePair out; + if constexpr (std::is_same_v) { + out.phase = p.dist; + } else { + const double turns = p.dist * grid.qstep / TWO_PI; + out.phase = + (unsigned long long)((turns - floor(turns)) * 18446744073709551616.0); + } + out.sigma = Real(sigma); + out.amplitude = Real(amplitude); + out.type0 = tp0; + out.type1 = tp1; + // The sum of a pair stops at the first kq with + // |amplitude * exp(-(sigma q)^2 / 2) * sf0 * sf1| <= prec. + // Its logarithm in single precision finds where that may be, then the + // terms themselves decide. + const double* sf0 = sf + (size_t)curve[tp0] * grid.nq; + const double* sf1 = sf + (size_t)curve[tp1] * grid.nq; + const float* lsf0 = logsf + (size_t)curve[tp0] * grid.nq; + const float* lsf1 = logsf + (size_t)curve[tp1] * grid.nq; + const float decay = (float)(0.5 * sigma * sigma * grid.qstep * grid.qstep); + const float limit = (grid.prec > 0.0) + ? (float)(log(grid.prec / fabs(amplitude)) - + log(fabs(occ[tp0])) - log(fabs(occ[tp1]))) + : -800.0f; + int kq = grid.kqlo; + for (; kq < grid.kqhi; ++kq) { + const float lterm = lsf0[kq] + lsf1[kq] - decay * (float)kq * (float)kq; + if (!(lterm > limit + 0.01f)) break; + } + for (; kq < grid.kqhi; ++kq) { + const double sq = sigma * (kq * grid.qstep); + const double term = amplitude * exp(-0.5 * sq * sq) * (sf0[kq] * occ[tp0]) * + (sf1[kq] * occ[tp1]); + if (fabs(term) <= grid.prec) break; + } + out.kqstop = kq; + pairs[j] = out; +} + +/// Apply each occupancy once per Q tile in the selected precision. +template +__global__ void occupiedFactors(const double* sf, const int* curve, + const double* occupancy, QGrid tile, int ntypes, + Real* out) { + const int span = tile.kqhi - tile.kqlo; + const int q = blockIdx.x * blockDim.x + threadIdx.x; + if (q >= span) return; + for (int t = blockIdx.y; t < ntypes; t += gridDim.y) { + out[(size_t)t * span + q] = + Real(sf[(size_t)curve[t] * tile.nq + tile.kqlo + q] * occupancy[t]); + } +} + +__device__ double debyeSine(double distance, double q, int) { + return sin(q * distance); +} + +__device__ float debyeSine(unsigned long long turns, float, int kq) { + // Fold the phase to a quarter turn before converting to single precision. + const long long phase = (long long)((unsigned long long)kq * turns); + const long long quarter = 1LL << 62; + const long long folded = (phase > quarter || phase < -quarter) + ? (long long)(0x8000000000000000ULL - phase) + : phase; + return sinpif((float)folded * 0x1p-63f); +} + +/// Each thread adds one Q point for a contiguous subchunk of pairs. +template +__global__ void debyeTerms(const DebyePair* pairs, const Real* sf, + QGrid grid, int count, double* partial) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + const int span = grid.kqhi - grid.kqlo; + if (i >= span) return; + const int kq = grid.kqlo + i; + const Real q = Real(kq * grid.qstep); + const int first = blockIdx.y * SUBCHUNK; + const int end = min(first + SUBCHUNK, count); + double sum = 0.0; + for (int j = first; j < end; ++j) { + if (kq >= pairs[j].kqstop) continue; + const auto& p = pairs[j]; + const Real sq = p.sigma * q; + sum += p.amplitude * exp(Real(-0.5) * sq * sq) * + sf[(size_t)p.type0 * span + i] * sf[(size_t)p.type1 * span + i] * + debyeSine(p.phase, q, kq); + } + partial[(size_t)blockIdx.y * span + i] = sum; +} + +/// Add the sums of the blocks of pairs in their order. +__global__ void finishRows(const double* partial, int cntsub, int span, + double* total) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= span) return; + double sum = 0.0; + for (int sub = 0; sub < cntsub; ++sub) sum += partial[(size_t)sub * span + i]; + total[i] += sum; +} + +int blocksFor(long long n) { return (int)((n + THREADS - 1) / THREADS); } + +// Sums ------------------------------------------------------------------------ + +/// What the kernels keep on the device between the sums of a calculator. +struct DeviceState : PairSumWorkspace { + const int device = currentDevice(); + bool isCurrentDevice() const override { return device == currentDevice(); } + + // structure + Buffer sitebegin; + Buffer possite; + Buffer pos; + Buffer posuij; + Buffer siteuij; + Buffer anisotropy; + Buffer multiplicity; + Buffer trans; + Buffer pairsite0; + Buffer pairpos; + Buffer pairtrans; + // the list of pairs on the device + const int* listowner = nullptr; + int listversion = -1; + long long listsize = -1; + // peaks + Buffer sfsite; + Buffer total; + // Debye sum + Buffer typeofsite, curve; + Buffer occupancy; + Buffer sf; + Buffer logsf; + DebyeBuffers debye; + DebyeBuffers debyeMixed; + Buffer partial; + + /// Copy the structure to the device. Return the number of its pairs. + long long upload(const PairGeometryArgs& v, PairGeometryArgs& g) { + g = v; + g.sitebegin = sitebegin.upload(v.sitebegin, v.cntsites + 1); + g.possite = possite.upload(v.possite, v.cntpositions); + g.pos = pos.upload(v.pos, 3 * (size_t)v.cntpositions); + g.posuij = posuij.upload(v.posuij, 6 * (size_t)v.cntpositions); + g.siteuij = siteuij.upload(v.siteuij, 6 * (size_t)v.cntsites); + g.anisotropy = anisotropy.upload(v.anisotropy, v.cntsites); + g.multiplicity = multiplicity.upload(v.multiplicity, v.cntsites); + g.trans = trans.upload(v.trans, 3 * (size_t)v.cnttrans); + if (!v.translations) { + listversion = -1; + g.pairsite0 = g.pairpos = g.pairtrans = nullptr; + return pairCount(v); + } + // The list stays on the device for the structures that keep it. + const bool same = (listowner == v.pairsite0) && + (listversion == v.pairversion) && + (listsize == v.cntpairs); + if (!same) { + pairsite0.upload(v.pairsite0, v.cntpairs); + pairpos.upload(v.pairpos, v.cntpairs); + pairtrans.upload(v.pairtrans, v.cntpairs); + listowner = v.pairsite0; + listversion = v.pairversion; + listsize = v.cntpairs; + } + g.pairsite0 = pairsite0.get(); + g.pairpos = pairpos.get(); + g.pairtrans = pairtrans.get(); + return v.cntpairs; + } +}; + +template +void addPDFPairs(const PDFPairSumArgs& job) { + if (job.npts <= 0 || !pairCount(job.g)) return; + auto& m = static_cast(*job.workspace); + PairGeometryArgs g; + const long long cntpairs = m.upload(job.g, g); + const double* sfsite = m.sfsite.upload(job.sfsite, job.g.cntsites); + const PeakGrid grid = {job.npts, job.rlosteps, job.rstep, + job.height, job.cropped, job.halfboundrel}; + // Bound temporary storage to 32 MiB or four rows for a very long grid. + const int rowLimit = std::max(4, std::min(4096, (4 << 20) / job.npts)); + const int blocks = int(std::min((cntpairs + THREADS - 1) / THREADS, + rowLimit / (THREADS / WARP))); + const int rows = blocks * (THREADS / WARP); + double* partial = m.partial.zero((size_t)rows * job.npts); + double* total = m.total.zero(job.npts); + pdfPeaks + <<>>(g, job.w, sfsite, grid, cntpairs, partial); + check(cudaGetLastError(), "the start of a PDF kernel"); + finishRows<<>>(partial, rows, job.npts, total); + check(cudaGetLastError(), "the reduction of the PDF"); + std::vector result(job.npts); + m.total.download(result.data(), job.npts); + if (std::any_of(result.begin(), result.end(), + [](double x) { return !std::isfinite(x); })) + throw std::runtime_error("nonfinite PDF sum on the CUDA device"); + for (int i = 0; i < job.npts; ++i) job.out[i] += result[i]; +} + +template +void addDebyePairs(const DebyePairSumArgs& job, DeviceState& m, + DebyeBuffers& work) { + const int span = job.kqhi - job.kqlo; + if (span <= 0) return; + PairGeometryArgs g; + const long long cntpairs = m.upload(job.g, g); + if (!cntpairs) return; + const QGrid grid = {job.nq, job.kqlo, job.kqhi, job.qstep, job.prec}; + const int* typeofsite = m.typeofsite.upload(job.typeofsite, job.g.cntsites); + const int* curve = m.curve.upload(job.curve, job.ntypes); + const double* occ = m.occupancy.upload(job.occupancy, job.ntypes); + const size_t sfsize = (size_t)job.ncurves * job.nq; + const double* sf = m.sf.upload(job.sf, sfsize); + std::vector logsf(sfsize); + for (size_t i = 0; i < sfsize; ++i) { + logsf[i] = (float)std::log(std::fabs(job.sf[i])); + } + const float* dlogsf = m.logsf.upload(logsf.data(), logsf.size()); + const size_t chunk = (size_t)std::min(CHUNK, std::max(cntpairs, 1LL)); + auto* pairs = work.pairs.reserve(chunk); + const size_t maxsub = (chunk + SUBCHUNK - 1) / SUBCHUNK; + // Scratch is bounded even when each site has its own occupancy. + const int tileSize = std::min(span, std::max(1, (1 << 20) / job.ntypes)); + Real* weighted = work.factors.reserve((size_t)tileSize * job.ntypes); + double* partial = m.partial.reserve(maxsub * tileSize); + double* total = m.total.zero(span); + for (long long first = 0; first < cntpairs; first += CHUNK) { + const long long count = std::min(CHUNK, cntpairs - first); + const int cntsub = (int)((count + SUBCHUNK - 1) / SUBCHUNK); + debyePairs<<>>( + g, job.w, typeofsite, curve, occ, sf, dlogsf, grid, first, count, pairs); + check(cudaGetLastError(), "the start of a kernel"); + for (int lo = grid.kqlo; lo < grid.kqhi;) { + const int n = std::min(tileSize, grid.kqhi - lo); + QGrid tile = grid; + tile.kqlo = lo; + tile.kqhi = lo + n; + const dim3 factorBlocks(blocksFor(n), std::min(job.ntypes, 65535)); + occupiedFactors<<>>(sf, curve, occ, tile, + job.ntypes, weighted); + check(cudaGetLastError(), "the occupied scattering factors"); + const dim3 blocks(blocksFor(n), cntsub); + debyeTerms<<>>(pairs, weighted, tile, (int)count, + partial); + check(cudaGetLastError(), "the start of a kernel"); + finishRows<<>>(partial, cntsub, n, + total + (lo - grid.kqlo)); + check(cudaGetLastError(), "the reduction of the Debye sum"); + lo += n; + } + } + check(cudaDeviceSynchronize(), "the Debye sum"); + std::vector result(span); + m.total.download(result.data(), span); + for (int i = 0; i < span; ++i) job.out[job.kqlo + i] += result[i]; +} + +} // namespace + +// Public Functions ------------------------------------------------------------ + +const char* cudaKernelUnavailable() { + // Loading an actual kernel checks the binary or PTX image against the + // calling thread's current device and driver without launching any work. + cudaFuncAttributes attributes; + const cudaError_t err = cudaFuncGetAttributes(&attributes, pdfPeaks); + return err == cudaSuccess ? nullptr : cudaGetErrorString(err); +} + +PairSumStats cudaPDFPairSum(const PDFPairSumArgs& a) { + addPDFPairs(a); + return {}; +} + +PairSumStats cudaPDFPairSumMixed(const PDFPairSumArgs& a) { + addPDFPairs(a); + return {}; +} + +PairSumStats cudaDebyePairSum(const DebyePairSumArgs& a) { + auto& state = static_cast(*a.workspace); + addDebyePairs(a, state, state.debye); + return {}; +} + +PairSumStats cudaDebyePairSumMixed(const DebyePairSumArgs& a) { + auto& state = static_cast(*a.workspace); + addDebyePairs(a, state, state.debyeMixed); + return {}; +} + +PairSumWorkspace* makeCudaWorkspace() { return new DeviceState; } + +} // namespace kernels +} // namespace srreal +} // namespace diffpy + +// End of file diff --git a/src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp b/src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp index 7a94175..a477a70 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp +++ b/src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp @@ -36,7 +36,7 @@ namespace { /// Kernels of this build that the CPU can run, from the most to the least /// preferred. CMake defines LIBDIFFPY_KERNEL_ for each SIMD kernel it /// compiles. The scalar kernel is last and always available. -const std::vector& availableKernels() { +const std::vector& cpuKernels() { static const std::vector available = [] { std::vector rv; #if defined(LIBDIFFPY_SIMD_KERNELS) @@ -63,9 +63,66 @@ const std::vector& availableKernels() { return available; } -/// The active kernel, the most preferred one is selected on first use. +// Kernel tables have process lifetime. Availability is checked on the +// current device whenever a caller lists or explicitly selects CUDA. +const char* cudaUnavailable() { +#if defined(LIBDIFFPY_KERNEL_CUDA) + return kernels::cudaKernelUnavailable(); +#else + return "libdiffpy was built without CUDA"; +#endif +} + +const std::vector& cudaKernels() { + static const std::vector cuda = [] { + std::vector rv; +#if defined(LIBDIFFPY_KERNEL_CUDA) + const CalculationKernel& cpu = *cpuKernels().front(); + static const CalculationKernel doubles = { + "cuda", + cpu.gaussianWindow, + cpu.debyeSum, + cpu.bondCandidates, + kernels::cudaPDFPairSum, + kernels::cudaDebyePairSum, + kernels::makeCudaWorkspace, + true, + }; + static const CalculationKernel mixed = { + "cuda-mixed", + cpu.gaussianWindow, + cpu.debyeSum, + cpu.bondCandidates, + kernels::cudaPDFPairSumMixed, + kernels::cudaDebyePairSumMixed, + kernels::makeCudaWorkspace, + true, + }; + rv = {&doubles, &mixed}; +#endif + return rv; + }(); + return cuda; +} + +bool isCudaName(const std::string& name) { + return name == "cuda" || name == "cuda-mixed"; +} + +/// All kernels that setCalculationKernel takes. +std::vector availableKernels() { + std::vector rv = cpuKernels(); + if (!cudaUnavailable()) { + const auto& cuda = cudaKernels(); + rv.insert(rv.end(), cuda.begin(), cuda.end()); + } + return rv; +} + +/// The active kernel, the most preferred one of the CPU is selected on +/// first use. std::atomic& activeKernelSlot() { - static std::atomic slot{availableKernels().front()}; + static std::atomic slot{cpuKernels().front()}; return slot; } @@ -85,14 +142,22 @@ std::string getCalculationKernel() { } void setCalculationKernel(const std::string& name) { - const auto& available = availableKernels(); + // No device is looked for when the name is one of the CPU. + const bool cuda = isCudaName(name); + if (cuda) { + if (const char* reason = cudaUnavailable()) { + throw std::invalid_argument("Calculation kernel '" + name + + "' is not available: " + reason + "."); + } + } + const auto& available = cuda ? cudaKernels() : cpuKernels(); const auto it = std::ranges::find_if( available, [&](const CalculationKernel* k) { return name == k->name; }); if (it == available.end()) { std::string names; - for (const CalculationKernel* k : available) { + for (const std::string& kname : availableCalculationKernels()) { if (!names.empty()) names += ", "; - names += k->name; + names += kname; } throw std::invalid_argument("Calculation kernel '" + name + "' is not available, use one of: " + names + @@ -107,6 +172,11 @@ const CalculationKernel& activeCalculationKernel() { return *activeKernelSlot().load(std::memory_order_relaxed); } +const CalculationKernel& pairSumKernel(bool cpuOnly) { + const auto& kernel = activeCalculationKernel(); + return cpuOnly && kernel.onDevice ? *cpuKernels().front() : kernel; +} + } // namespace kernels } // namespace srreal diff --git a/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp b/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp index dcc47ed..58e4b88 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp +++ b/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp @@ -150,6 +150,7 @@ struct PairSumStats { /// Owned by the calculator and reused by kernels with the same backend. struct PairSumWorkspace { virtual ~PairSumWorkspace() = default; + virtual bool isCurrentDevice() const { return true; } }; using MakeWorkspace = PairSumWorkspace* (*)(); @@ -158,7 +159,7 @@ struct PairSumArgs { PairWidthArgs w; double* out; PairSumWorkspace* workspace; - /// A null mask includes every pair. + /// Only the CPU reads the mask. A null pointer includes every pair. const PairQuantity* mask; }; @@ -210,11 +211,16 @@ struct CalculationKernel { PairSumStats (*pdfPairSum)(const PDFPairSumArgs&); PairSumStats (*debyePairSum)(const DebyePairSumArgs&); MakeWorkspace makeWorkspace; + bool onDevice = false; }; /// The active kernel, exported for the unit tests. DLL_EXPORT const CalculationKernel& activeCalculationKernel(); +/// Use the selected CPU kernel for CPU-only features, or the preferred CPU +/// kernel when the active kernel is on a device. +const CalculationKernel& pairSumKernel(bool cpuOnly); + PairSumStats cpuPDFPairSum(const PDFPairSumArgs&, GaussianWindow); PairSumStats cpuDebyePairSum(const DebyePairSumArgs&, DebyeSum); PairSumWorkspace* makeCPUWorkspace(); @@ -240,6 +246,18 @@ constexpr CalculationKernel makeCPUKernel( makeCPUWorkspace}; } +// Functions of the CUDA kernels, which exist only when CMake compiles +// them, see cmake/Cuda.cmake. The Mixed ones add in single precision +// where the result keeps six digits. + +/// Null when a CUDA device can run the pair sums, otherwise why it cannot. +const char* cudaKernelUnavailable(); +PairSumStats cudaPDFPairSum(const PDFPairSumArgs&); +PairSumStats cudaPDFPairSumMixed(const PDFPairSumArgs&); +PairSumStats cudaDebyePairSum(const DebyePairSumArgs&); +PairSumStats cudaDebyePairSumMixed(const DebyePairSumArgs&); +PairSumWorkspace* makeCudaWorkspace(); + } // namespace kernels } // namespace srreal } // namespace diffpy diff --git a/src/diffpy/srreal/kernels/PairGeometry.hpp b/src/diffpy/srreal/kernels/PairGeometry.hpp index 8921e80..f76428b 100644 --- a/src/diffpy/srreal/kernels/PairGeometry.hpp +++ b/src/diffpy/srreal/kernels/PairGeometry.hpp @@ -17,6 +17,12 @@ #include +#ifdef __CUDACC__ +#define DIFFPY_HOST_DEVICE __host__ __device__ +#else +#define DIFFPY_HOST_DEVICE +#endif + namespace diffpy { namespace srreal { namespace kernels { @@ -24,7 +30,7 @@ namespace kernels { constexpr double PAIR_SIGMA_TO_FWHM = 2.3548200450309493; /// Use the per-bond rounding order for the Debye precision cutoff. -inline double fwhmToSigma(double fwhm) { +DIFFPY_HOST_DEVICE inline double fwhmToSigma(double fwhm) { constexpr double scale = 1.0 / PAIR_SIGMA_TO_FWHM; return fwhm * scale; } @@ -35,7 +41,8 @@ inline long long pairCount(const PairGeometryArgs& g) { } /// Clip before converting to an integer, including for very wide peaks. -inline int gridIndex(double r, double step, int lo, int offset, int npts) { +DIFFPY_HOST_DEVICE inline int gridIndex(double r, double step, int lo, + int offset, int npts) { const double i = floor(r / step) - lo + offset; return i > 0 ? (i < npts ? int(i) : npts) : 0; } @@ -50,8 +57,8 @@ struct Pair { }; /// As meanSquareDisplacement for the tensor xx, yy, zz, xy, xz, yz. -inline double siteMSD(const double* U, double rx, double ry, double rz, - int anisotropy) { +DIFFPY_HOST_DEVICE inline double siteMSD(const double* U, double rx, double ry, + double rz, int anisotropy) { if (!anisotropy) return U[0]; const double sUs = U[0] * rx * rx + U[1] * ry * ry + U[2] * rz * rz + 2 * (U[3] * rx * ry + U[4] * rx * rz + U[5] * ry * rz); @@ -62,7 +69,7 @@ inline double siteMSD(const double* U, double rx, double ry, double rz, struct ClusterPair { int site0, site1; - static ClusterPair at(long long k) { + DIFFPY_HOST_DEVICE static ClusterPair at(long long k) { // k = i0 * (i0 - 1) / 2 + i1 with i1 < i0. long long row = (long long)((1.0 + sqrt(1.0 + 8.0 * (double)k)) / 2); while (row * (row - 1) / 2 > k) --row; @@ -78,8 +85,8 @@ struct ClusterPair { } }; -inline Pair pairAt(const PairGeometryArgs& g, long long k, - ClusterPair cluster) { +DIFFPY_HOST_DEVICE inline Pair pairAt(const PairGeometryArgs& g, long long k, + ClusterPair cluster) { Pair rv{}; int p1 = cluster.site1; rv.site0 = cluster.site0; @@ -109,30 +116,31 @@ inline Pair pairAt(const PairGeometryArgs& g, long long k, return rv; } -/// Look up a pair directly from its index. -inline Pair pairAt(const PairGeometryArgs& g, long long k) { +/// Random access for independent CUDA threads. +DIFFPY_HOST_DEVICE inline Pair pairAt(const PairGeometryArgs& g, long long k) { return pairAt(g, k, g.translations ? ClusterPair{} : ClusterPair::at(k)); } -inline double debyeWallerWidth(double msd) { +DIFFPY_HOST_DEVICE inline double debyeWallerWidth(double msd) { return msd < 0.0 ? 0.0 : PAIR_SIGMA_TO_FWHM * sqrt(msd); } -inline double sharpeningRatio(double r, double delta1, double delta2, - double qbroad) { +DIFFPY_HOST_DEVICE inline double sharpeningRatio(double r, double delta1, + double delta2, double qbroad) { if (r < DBL_EPSILON) return 0.0; const double qr = qbroad * r; return 1.0 - delta1 / r - delta2 / (r * r) + qr * qr; } -inline double sharpenedWidth(double r, double corr, double msd, - double qbroadsep) { +DIFFPY_HOST_DEVICE inline double sharpenedWidth(double r, double corr, + double msd, double qbroadsep) { const double qr = qbroadsep * r; return sqrt(corr) * debyeWallerWidth(msd) + qr * qr; } /// FWHM of the built-in peak width models for one pair. -inline double widthOf(const PairWidthArgs& w, double dist, double msd) { +DIFFPY_HOST_DEVICE inline double widthOf(const PairWidthArgs& w, double dist, + double msd) { if (w.kind == PairWidthArgs::Constant) return w.width; if (w.kind == PairWidthArgs::DebyeWaller) return debyeWallerWidth(msd); const double corr = sharpeningRatio(dist, w.delta1, w.delta2, w.qbroad); @@ -143,4 +151,5 @@ inline double widthOf(const PairWidthArgs& w, double dist, double msd) { } // namespace srreal } // namespace diffpy +#undef DIFFPY_HOST_DEVICE #endif diff --git a/src/diffpy/srreal/kernels/PairTable.cpp b/src/diffpy/srreal/kernels/PairTable.cpp index 9ac5614..6269bbe 100644 --- a/src/diffpy/srreal/kernels/PairTable.cpp +++ b/src/diffpy/srreal/kernels/PairTable.cpp @@ -322,7 +322,7 @@ bool assignPairWidth(PairWidthArgs& rv, const PeakWidthModel& model) { ////////////////////////////////////////////////////////////////////////////// PairSumWorkspace* PairSumState::workspaceFor(const CalculationKernel& kernel) { - if (mfactory != kernel.makeWorkspace) { + if (mfactory != kernel.makeWorkspace || !mworkspace->isCurrentDevice()) { mworkspace.reset(kernel.makeWorkspace()); mfactory = kernel.makeWorkspace; } diff --git a/src/diffpy/srreal/kernels/PairTable.hpp b/src/diffpy/srreal/kernels/PairTable.hpp index 3d9c38b..e2d799a 100644 --- a/src/diffpy/srreal/kernels/PairTable.hpp +++ b/src/diffpy/srreal/kernels/PairTable.hpp @@ -84,7 +84,7 @@ class DLL_EXPORT PairTable { }; static View view(const StructureAdapter& stru); - /// Contiguous storage for the arrays read by the pair-sum kernels. + /// Contiguous storage for the arrays read by both CPU and CUDA kernels. struct Geometry { std::vector pos, posuij, siteuij, trans; std::vector anisotropy; diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 08632a3..1e00ad6 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -61,6 +61,11 @@ set_property(SOURCE TestCalculationKernel.cpp test_helpers.cpp target_include_directories(alltests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}") target_link_libraries(alltests PRIVATE libdiffpy::diffpy Catch2::Catch2WithMain Threads::Threads) +if(TARGET diffpy_cuda) + # The device-switch test selects the current GPU through the CUDA runtime. + target_link_libraries(alltests PRIVATE CUDA::cudart_static) + target_compile_definitions(alltests PRIVATE LIBDIFFPY_TEST_CUDA_RUNTIME) +endif() # test_helpers.cpp finds the testdata directory from this macro. set_property(SOURCE test_helpers.cpp APPEND PROPERTY COMPILE_DEFINITIONS "DIFFPYTESTSDIRPATH=${CMAKE_CURRENT_SOURCE_DIR}") @@ -97,6 +102,9 @@ list(REMOVE_DUPLICATES _suite_tags) foreach(tag IN LISTS _suite_tags) string(REGEX REPLACE "^\"\\[|\\]\"$" "" name "${tag}") add_test(NAME "${name}" COMMAND alltests "[${name}]") + # Catch2 returns 4 when it skipped all tests of the tag, as those of the + # CUDA kernels without a device. + set_tests_properties("${name}" PROPERTIES SKIP_RETURN_CODE 4) if(WIN32 AND LIBDIFFPY_TEST_INSTALLED) set_tests_properties("${name}" PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") diff --git a/src/tests/TestCalculationKernel.cpp b/src/tests/TestCalculationKernel.cpp index accbce8..9081e98 100644 --- a/src/tests/TestCalculationKernel.cpp +++ b/src/tests/TestCalculationKernel.cpp @@ -234,10 +234,17 @@ TEST_CASE("TestCalculationKernel::test_available", "[CalculationKernel]") { return std::ranges::find(names, name) != names.end(); }; REQUIRE(!names.empty()); - // The scalar kernel is the last resort. - CHECK(names.back() == "scalar"); - const std::vector known = {"avx512", "avx2", "sse2", "neon", - "scalar"}; + // The scalar kernel is the last resort of the CPU. The CUDA kernels + // follow it, so that they are never the default. + const size_t cntcpu = names.size() - (has("cuda") ? 2 : 0); + CHECK(names[cntcpu - 1] == "scalar"); + CHECK(has("cuda") == has("cuda-mixed")); + if (has("cuda")) { + CHECK(names[cntcpu] == "cuda"); + CHECK(names[cntcpu + 1] == "cuda-mixed"); + } + const std::vector known = {"avx512", "avx2", "sse2", "neon", + "scalar", "cuda", "cuda-mixed"}; for (const std::string& kname : names) { INFO("kernel " << kname); CHECK(std::ranges::count(names, kname) == 1); @@ -273,6 +280,22 @@ TEST_CASE("TestCalculationKernel::test_selection", "[CalculationKernel]") { } else { CHECK_THROWS_AS(setCalculationKernel("sse2"), std::invalid_argument); } + // The CUDA kernels tell why they are not available. + if (std::ranges::find(names, "cuda") == names.end()) { + for (const char* cuda : {"cuda", "cuda-mixed"}) { + INFO("kernel '" << cuda << "'"); + try { + setCalculationKernel(cuda); + FAIL("The kernel was taken without a device."); + } catch (const std::invalid_argument& e) { + const std::string what = e.what(); + INFO(what); + CHECK(what.find("is not available: ") != std::string::npos); + CHECK(what.find("use one of") == std::string::npos); + } + CHECK(active == getCalculationKernel()); + } + } } TEST_CASE("TestCalculationKernel::test_match_scalar", "[CalculationKernel]") { diff --git a/src/tests/TestCalculationKernelCuda.cpp b/src/tests/TestCalculationKernelCuda.cpp new file mode 100644 index 0000000..627e31d --- /dev/null +++ b/src/tests/TestCalculationKernelCuda.cpp @@ -0,0 +1,527 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + ****************************************************************************** + * + * TestCalculationKernelCuda -- the "cuda" and "cuda-mixed" kernels must + * give the results of the CPU. Skipped without a CUDA device. + * + *****************************************************************************/ + +#include + +#ifdef LIBDIFFPY_TEST_CUDA_RUNTIME +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// The internal kernel header is missing when testing an installed library. +#if __has_include() +#include +#define LIBDIFFPY_TEST_KERNEL_INTERNALS 1 +#endif + +#include "pair_test_helpers.hpp" + +using namespace diffpy::srreal; +using namespace pairtests; + +namespace { + +/// Agreement with the CPU. In mixed precision, the results other than the +/// raw PDF peaks cancel larger terms (Debye sums, PDF after the baseline) +/// and get the looser tolerance. +const double TOLERANCE = 1e-12; +const double MIXED_TOLERANCE = 1e-6; +const double MIXED_REST_TOLERANCE = 1e-5; + +void skipWithoutDevice() { + KernelGuard guard; + try { + setCalculationKernel("cuda"); + } catch (const std::invalid_argument& e) { + SKIP(e.what()); + } +} + +/// The kernel of the CPU that the calculators use by default. +std::string cpuKernel() { return availableCalculationKernels().front(); } + +/// Evaluate with the kernel, then restore the one that was active. +const QuantityType& evalWith(const std::string& kernel, PairQuantity& calc, + StructureAdapterPtr stru) { + KernelGuard guard; + setCalculationKernel(kernel); + return calc.eval(stru); +} + +/// Ni atoms of the fcc lattice within a sphere. +std::shared_ptr makeSphere(double radius) { + auto rv = std::make_shared(); + Atom a; + a.atomtype = "Ni"; + a.uij_cartn = 0.005 * R3::identity(); + const double alat = 3.52; + const int n = int(radius / alat) + 1; + const R3::Vector basis[] = {R3::Vector(0, 0, 0), R3::Vector(0, 0.5, 0.5), + R3::Vector(0.5, 0, 0.5), R3::Vector(0.5, 0.5, 0)}; + for (int i = -n; i <= n; ++i) { + for (int j = -n; j <= n; ++j) { + for (int k = -n; k <= n; ++k) { + for (const R3::Vector& b : basis) { + a.xyz_cartn = alat * (b + R3::Vector(i, j, k)); + if (R3::norm(a.xyz_cartn) <= radius) rv->append(a); + } + } + } + } + return rv; +} + +/// Results of a calculator of the class T with the kernel. +template +std::vector evaluate(const Setup& setup, StructureAdapterPtr s, + const std::string& kernel) { + KernelGuard guard; + setCalculationKernel(kernel); + AllPairs calc; + calc.setEvaluatorType(BASIC); + setup(calc); + calc.eval(s); + CHECK(calc.addedAllPairs()); + CHECK((kernel != cpuKernel()) == calc.addedOnDevice()); + std::vector rv = {calc.value(), calc.getPDF(), calc.getF()}; + // The same again to the last bit, also from the calculator of the + // library. + T again; + again.setEvaluatorType(BASIC); + setup(again); + again.eval(s); + CHECK(rv[0] == again.value()); + return rv; +} + +/// Check that the device gives what the CPU does. +template +void checkAsCPU(const Setup& setup, StructureAdapterPtr s) { + const auto expected = evaluate(setup, s, cpuKernel()); + REQUIRE(*std::ranges::max_element(expected[0]) > 0.0); + const auto results = evaluate(setup, s, "cuda"); + const auto mixed = evaluate(setup, s, "cuda-mixed"); + for (size_t i = 0; i < expected.size(); ++i) { + CHECK(maxRelativeError(results[i], expected[i]) <= TOLERANCE); + const bool peaks = !i && std::is_same_v; + const double tolerance = peaks ? MIXED_TOLERANCE : MIXED_REST_TOLERANCE; + CHECK(maxRelativeError(mixed[i], expected[i]) <= tolerance); + } +} + +} // namespace + +TEST_CASE("TestCalculationKernelCuda::test_results_of_calculators", "[cuda]") { + skipWithoutDevice(); + forEachCalculatorCase([](const Setup& setup, StructureAdapterPtr structure) { + checkAsCPU(setup, structure); + checkAsCPU(setup, structure); + }); +} + +#ifdef LIBDIFFPY_TEST_CUDA_RUNTIME +TEST_CASE("TestCalculationKernelCuda::test_device_switch", "[cuda]") { + skipWithoutDevice(); + int count, original; + REQUIRE(cudaGetDeviceCount(&count) == cudaSuccess); + if (count < 2) SKIP("Requires two CUDA devices"); + REQUIRE(cudaGetDevice(&original) == cudaSuccess); + struct RestoreDevice { + int device; + ~RestoreDevice() { cudaSetDevice(device); } + } restore{original}; + + const auto stru = loadPeriodic("CaTiO3.stru"); + PDFCalculator pdf; + DebyePDFCalculator debye; + PairQuantity* calculators[] = {&pdf, &debye}; + for (PairQuantity* calc : calculators) { + setupDefault(*calc); + calc->setEvaluatorType(BASIC); + const QuantityType expected = evalWith("scalar", *calc, stru); + for (const int device : {original, (original + 1) % count, original}) { + REQUIRE(cudaSetDevice(device) == cudaSuccess); + skipWithoutDevice(); + CHECK(maxRelativeError(evalWith("cuda", *calc, stru), expected) <= + TOLERANCE); + } + } +} +#endif + +TEST_CASE("TestCalculationKernelCuda::test_peak_profiles_and_cutoffs", + "[cuda]") { + skipWithoutDevice(); + const StructureAdapterPtr stru = loadPeriodic("CaTiO3.stru", true); + const Setup cropped = [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c).setPeakProfileByType("croppedgaussian"); + }; + const Setup precise = [](PairQuantity& c) { + setupDefault(c); + c.setDoubleAttr("peakprecision", 1e-12); + }; + // peaks that are cut at one percent of their height + const Setup cut = [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c).setPeakProfileByType("croppedgaussian"); + c.setDoubleAttr("peakprecision", 0.01); + }; + checkAsCPU(cropped, stru); + checkAsCPU(cut, stru); + checkAsCPU(precise, stru); + // Debye sums that stop early, late and never + for (double prec : {1e-3, 1e-6, 1e-12, 0.0}) { + INFO("debyeprecision " << prec); + const Setup debye = [=](PairQuantity& c) { + setupDefault(c); + c.setDoubleAttr("debyeprecision", prec); + }; + checkAsCPU(debye, stru); + checkAsCPU(debye, + asCluster(*loadPeriodic("Ni.stru", true))); + } +} + +TEST_CASE("TestCalculationKernelCuda::test_narrow_jeong_peak", "[cuda]") { + skipWithoutDevice(); + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "Ni"; + atom.uij_cartn = 0.05 * R3::identity(); + stru->append(atom); + atom.xyz_cartn[0] = 1.0; + stru->append(atom); + checkAsCPU( + [](PairQuantity& c) { + c.setRmax(2.0); + c.setDoubleAttr("rstep", 0.01); + c.setDoubleAttr("delta2", 1.0 - 2.5e-8); + }, + stru); +} + +TEST_CASE("TestCalculationKernelCuda::test_sampled_peak_windows", "[cuda]") { + skipWithoutDevice(); + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "C"; + stru->append(atom); + stru->append(atom); + PDFCalculator calc; + calc.setRmax(4); + calc.setRstep(0.01); + calc.setMaxExtension(0); + calc.setEvaluatorType(BASIC); + SECTION("an unresolved peak between samples leaves the other peaks intact") { + (*stru)[1].xyz_cartn[0] = 1.005; + atom.xyz_cartn[1] = 2; + atom.uij_cartn = 0.01 * R3::identity(); + stru->append(atom); + calc.setDoubleAttr("qbroad_seperable", 1e-10); + } + SECTION("a 3 angstrom bond at the inclusive upper distance cutoff") { + (*stru)[1].xyz_cartn = R3::Vector(0.4, 2.0, 2.2); + (*stru)[1].uij_cartn = 0.02 * R3::identity(); + calc.setRmax(3); + } + SECTION("a sample at the cropped Gaussian boundary") { + (*stru)[1].xyz_cartn[0] = 3; + calc.setPeakWidthModelByType("constant"); + calc.setDoubleAttr("width", 0.2); + calc.setPeakProfileByType("croppedgaussian"); + calc.setDoubleAttr("peakprecision", 0.5); + } + const QuantityType reference = evalWith("scalar", calc, stru); + REQUIRE(*std::ranges::max_element(reference) > 0.0); + CHECK(maxRelativeError(evalWith("cuda", calc, stru), reference) <= TOLERANCE); + CHECK(maxRelativeError(evalWith("cuda-mixed", calc, stru), reference) <= + MIXED_TOLERANCE); +} + +TEST_CASE("TestCalculationKernelCuda::test_weak_peaks", "[cuda]") { + skipWithoutDevice(); + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "Ni"; + atom.occupancy = 0.01; + for (int x = 0; x < 8; ++x) + for (int y = 0; y < 8; ++y) + for (int z = 0; z < 8; ++z) { + atom.xyz_cartn = R3::Vector(x, y, z); + stru->append(atom); + } + atom.occupancy = 1; + atom.xyz_cartn = R3::Vector(0, 100, 0); + stru->append(atom); + atom.xyz_cartn[0] = 2.7; + stru->append(atom); + PDFCalculator calc; + calc.setEvaluatorType(BASIC); + calc.setRmax(5); + calc.setMaxExtension(0); + calc.setPeakWidthModelByType("constant"); + calc.setDoubleAttr("width", 0.1); + const QuantityType reference = evalWith("scalar", calc, stru); + const QuantityType actual = evalWith("cuda", calc, stru); + CHECK(maxRelativeError(actual, reference) <= TOLERANCE); + CHECK(actual == evalWith("cuda", calc, stru)); +} + +TEST_CASE("TestCalculationKernelCuda::test_split_site_debye_sum", "[cuda]") { + skipWithoutDevice(); + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "Ni"; + atom.uij_cartn = 0.01 * R3::identity(); + stru->append(atom); + atom.xyz_cartn[0] = 2.0; + stru->append(atom); + DebyePDFCalculator calc; + calc.setRmax(4); + calc.setQstep(0.01); + calc.setEvaluatorType(BASIC); + const QuantityType reference = evalWith("scalar", calc, stru); + // Split the first site into coincident partial occupancies. All its + // terms for a given Q have the same sign and their sum is unchanged. + (*stru)[0].occupancy = 1.0 / 2048; + atom = (*stru)[0]; + for (int i = 1; i < 2048; ++i) stru->append(atom); + // Put the unsplit site last so that one subchunk sums 1024 equal terms. + std::swap((*stru)[1], stru->at(2048)); + CHECK(maxRelativeError(evalWith("cuda-mixed", calc, stru), reference) <= + MIXED_TOLERANCE); +} + +TEST_CASE("TestCalculationKernelCuda::test_many_occupancies_on_a_fine_q_grid", + "[cuda]") { + skipWithoutDevice(); + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "Ni"; + // Only the first two sites form a bond. All 129 occupancies enter the + // factor table, which spans more than one Q tile at this Q step. + for (int i = 0; i < 129; ++i) { + atom.occupancy = (i + 1) / 129.0; + atom.xyz_cartn[0] = (i == 1) ? 2.0 : 10.0 * i; + stru->append(atom); + } + DebyePDFCalculator calc; + calc.setEvaluatorType(BASIC); + calc.setRmax(4); + calc.setMaxExtension(0); + calc.setQmax(9); + calc.setQstep(0.001); + const QuantityType expected = evalWith("scalar", calc, stru); + REQUIRE(*std::ranges::max_element(expected) > 0); + CHECK(maxRelativeError(evalWith("cuda", calc, stru), expected) <= TOLERANCE); + CHECK(maxRelativeError(evalWith("cuda-mixed", calc, stru), expected) <= + MIXED_TOLERANCE); +} + +TEST_CASE("TestCalculationKernelCuda::test_change_precision", "[cuda]") { + skipWithoutDevice(); + const auto stru = loadPeriodic("CaTiO3.stru", true); + PDFCalculator pdf; + DebyePDFCalculator debye; + PairQuantity* calculators[] = {&pdf, &debye}; + for (PairQuantity* calc : calculators) { + setupDefault(*calc); + calc->setEvaluatorType(OPTIMIZED); + const QuantityType mixed = evalWith("cuda-mixed", *calc, stru); + const QuantityType doubles = evalWith("cuda", *calc, stru); + CHECK(calc->getEvaluatorTypeUsed() == BASIC); + calc->setEvaluatorType(BASIC); + const QuantityType expected = evalWith("cuda", *calc, stru); + REQUIRE(mixed != expected); + CHECK(maxRelativeError(doubles, expected) == 0.0); + } +} + +TEST_CASE("TestCalculationKernelCuda::test_many_pairs", "[cuda]") { + skipWithoutDevice(); + // more pairs than the device takes at once + const StructureAdapterPtr sphere = makeSphere(19.5); + const long long n = sphere->countSites(); + INFO(n << " atoms"); + REQUIRE(n * (n - 1) / 2 > (3LL << 20)); + const Setup setup = [](PairQuantity& c) { + c.setRmax(41.0); + c.setDoubleAttr("qmax", 20.0); + }; + checkAsCPU(setup, sphere); + checkAsCPU(setup, sphere); + // and of a periodic structure + const StructureAdapterPtr periodic = + loadPeriodic("alpha_K2Bi8Se13.stru", true); + const Setup far = [](PairQuantity& c) { + c.setRmax(60.0); + c.setDoubleAttr("qmax", 15.0); + }; + checkAsCPU(far, periodic); +} + +TEST_CASE("TestCalculationKernelCuda::test_evaluations_in_turn", "[cuda]") { + skipWithoutDevice(); + // The list of pairs stays on the device while the structure changes + // little. + const auto periodic = loadPeriodic("CaTiO3.stru", true); + for (std::shared_ptr stru : + {periodic, + std::shared_ptr(asCrystal(*periodic))}) { + AllPairs cpu, device; + AllPairs dcpu, ddevice; + for (PairQuantity* c : {(PairQuantity*)&cpu, (PairQuantity*)&device, + (PairQuantity*)&dcpu, (PairQuantity*)&ddevice}) { + setupDefault(*c); + } + const Lattice L = stru->getLattice(); + for (int k = 0; k < 12; ++k) { + INFO("evaluation " << k); + const int n = stru->countSites(); + // every fourth change is beyond the margin of the pairs + const double step = (k % 4 == 3) ? 0.3 : 0.004; + stru->at(k % n).xyz_cartn[k % 3] += step * ((k % 5) - 2); + stru->at((3 * k) % n).uij_cartn *= 1.0 + 0.01 * ((k % 3) - 1); + for (Atom& a : *stru) stru->toFractional(a); + const double f = 1.0 + 1e-4 * ((k % 7) - 3); + stru->setLatPar(f * L.a(), L.b(), L.c() / f, L.alpha(), L.beta(), + L.gamma()); + for (Atom& a : *stru) stru->toCartesian(a); + StructureAdapterPtr copy = stru->clone(); + // every third sum of the device is in mixed precision + const bool mixed = (k % 3 == 2); + const std::string kernel = mixed ? "cuda-mixed" : "cuda"; + const double tolerance = mixed ? MIXED_REST_TOLERANCE : TOLERANCE; + CHECK(maxRelativeError(evalWith(kernel, device, copy), cpu.eval(copy)) <= + tolerance); + CHECK(maxRelativeError(evalWith(kernel, ddevice, copy), + dcpu.eval(copy)) <= tolerance); + CHECK(device.addedOnDevice()); + CHECK(ddevice.addedOnDevice()); + CHECK(!cpu.addedOnDevice()); + // The device adds all pairs again rather than those of a few sites. + if (k > 0) CHECK(BASIC == device.getEvaluatorTypeUsed()); + } + // a structure of the same content takes no sum + evalWith("cuda-mixed", device, stru->clone()); + CHECK(OPTIMIZED == device.getEvaluatorTypeUsed()); + } +} + +TEST_CASE("TestCalculationKernelCuda::test_sums_on_the_cpu", "[cuda]") { + skipWithoutDevice(); + struct DerivedProfile : GaussianProfile {}; + const StructureAdapterPtr stru = loadPeriodic("CaTiO3.stru", true); + // What the device does not take gives the results of the CPU to the + // last bit. + auto check = [&](PairQuantity& calc, PairQuantity& cpu) { + for (PairQuantity* c : {&calc, &cpu}) { + setupDefault(*c); + c->setEvaluatorType(BASIC); + } + for (const char* kernel : {"cuda", "cuda-mixed"}) { + INFO("kernel " << kernel); + CHECK(evalWith(kernel, calc, stru) == cpu.eval(stru)); + } + }; + // all pairs at once, with the kernel of the CPU + AllPairs masked, maskedcpu; + masked.setPairMask(0, 3, false); + maskedcpu.setPairMask(0, 3, false); + check(masked, maskedcpu); + CHECK(masked.addedAllPairs()); + CHECK(!masked.addedOnDevice()); + AllPairs dmasked, dmaskedcpu; + dmasked.setTypeMask("Ti4+", "O2-", false); + dmaskedcpu.setTypeMask("Ti4+", "O2-", false); + check(dmasked, dmaskedcpu); + CHECK(dmasked.addedAllPairs()); + CHECK(!dmasked.addedOnDevice()); + AllPairs profiled, profiledcpu; + profiled.setPeakProfile(PeakProfilePtr(new DerivedProfile)); + profiledcpu.setPeakProfile(PeakProfilePtr(new DerivedProfile)); + check(profiled, profiledcpu); + CHECK_FALSE(profiled.addedAllPairs()); + CHECK(!profiled.addedOnDevice()); + // the pairs one by one + PairByPair derived, derivedcpu; + check(derived, derivedcpu); + BVSCalculator bvsc, bvscpu; + CHECK(evalWith("cuda", bvsc, stru) == bvscpu.eval(stru)); + // all pairs of a calculator without any of these go to the device + AllPairs pdfc; + PDFCalculator pdfcpu; + for (PairQuantity* c : {(PairQuantity*)&pdfc, (PairQuantity*)&pdfcpu}) { + setupDefault(*c); + c->setEvaluatorType(BASIC); + } + CHECK(maxRelativeError(evalWith("cuda", pdfc, stru), pdfcpu.eval(stru)) <= + TOLERANCE); + CHECK(pdfc.addedOnDevice()); + // and back to the CPU with its kernels + CHECK(pdfc.eval(stru) == pdfcpu.value()); + CHECK(pdfc.addedAllPairs()); + CHECK(!pdfc.addedOnDevice()); + // then to the device again + CHECK(maxRelativeError(evalWith("cuda-mixed", pdfc, stru), pdfcpu.value()) <= + MIXED_TOLERANCE); + CHECK(pdfc.addedOnDevice()); +} + +#ifdef LIBDIFFPY_TEST_KERNEL_INTERNALS + +TEST_CASE("TestCalculationKernelCuda::test_kernels_of_the_cpu", "[cuda]") { + skipWithoutDevice(); + // The CUDA kernels are those of the CPU for everything else. + KernelGuard guard; + setCalculationKernel(cpuKernel()); + const kernels::CalculationKernel cpu = kernels::activeCalculationKernel(); + CHECK_FALSE(cpu.onDevice); + CHECK(cpu.pdfPairSum); + CHECK(cpu.debyePairSum); + for (const char* name : {"cuda", "cuda-mixed"}) { + INFO("kernel " << name); + setCalculationKernel(name); + const kernels::CalculationKernel& k = kernels::activeCalculationKernel(); + CHECK(std::string(name) == k.name); + CHECK(k.gaussianWindow == cpu.gaussianWindow); + CHECK(k.debyeSum == cpu.debyeSum); + CHECK(k.bondCandidates == cpu.bondCandidates); + CHECK(k.pdfPairSum); + CHECK(k.debyePairSum); + CHECK(k.makeWorkspace); + CHECK(k.onDevice); + } +} + +#endif // LIBDIFFPY_TEST_KERNEL_INTERNALS diff --git a/src/tests/test_helpers.hpp b/src/tests/test_helpers.hpp index e5054d4..1120929 100644 --- a/src/tests/test_helpers.hpp +++ b/src/tests/test_helpers.hpp @@ -58,6 +58,7 @@ void testAllKernels(Fn&& fn) { KernelGuard guard; for (const std::string& kname : diffpy::srreal::availableCalculationKernels()) { + if (kname.starts_with("cuda")) continue; diffpy::srreal::setCalculationKernel(kname); INFO("kernel " << kname); fn(); @@ -78,6 +79,10 @@ class AllPairs : public T { public: /// True when the last full calculation added all pairs at once. bool addedAllPairs() const { return maddedallpairs; } + /// True when it added them on the GPU. + bool addedOnDevice() const { + return maddedallpairs && this->mallpairs.ondevice; + } protected: // every calculation starts with the value reset From 88406875bed5985aea10c4f2bfc4939227d67d7a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 19 Nov 2026 05:50:06 +0000 Subject: [PATCH 3/6] [pre-commit.ci] auto fixes from pre-commit hooks --- src/diffpy/srreal/PairQuantity.hpp | 2 +- src/diffpy/srreal/kernels/CalculationKernelCuda.cu | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/diffpy/srreal/PairQuantity.hpp b/src/diffpy/srreal/PairQuantity.hpp index 2e82beb..e50b640 100644 --- a/src/diffpy/srreal/PairQuantity.hpp +++ b/src/diffpy/srreal/PairQuantity.hpp @@ -44,7 +44,7 @@ struct PairSumState; struct PairSumArgs; struct PairSumStats; struct CalculationKernel; -} +} // namespace kernels class DLL_EXPORT PairQuantity : public diffpy::Attributes { public: diff --git a/src/diffpy/srreal/kernels/CalculationKernelCuda.cu b/src/diffpy/srreal/kernels/CalculationKernelCuda.cu index 43f1bb1..a24a294 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelCuda.cu +++ b/src/diffpy/srreal/kernels/CalculationKernelCuda.cu @@ -228,8 +228,8 @@ __global__ void pdfPeaks(PairGeometryArgs g, PairWidthArgs w, using Real = std::conditional_t; const Real xr = Real(x * p.invwidth); const Real correction = Real(x) * Real(p.invdist) + Real(1); - row[i] += Real(p.amplitude) * exp(-Real(FOUR_LN2) * xr * xr) * - correction; + row[i] += + Real(p.amplitude) * exp(-Real(FOUR_LN2) * xr * xr) * correction; } } } From c6e661ed1a2eda66b3a8aa2a862acf4687060531 Mon Sep 17 00:00:00 2001 From: SYMSCAE Date: Wed, 18 Nov 2026 22:37:21 -0800 Subject: [PATCH 4/6] fix: crop Gaussian peak windows on the absolute r-grid --- .../srreal/kernels/CalculationKernelCuda.cu | 16 ++--- .../kernels/CalculationKernelDispatch.cpp | 2 +- .../srreal/kernels/CalculationKernelImpl.hpp | 5 +- .../srreal/kernels/CalculationKernelSIMD.ipp | 62 ++++++------------- .../kernels/CalculationKernelScalar.cpp | 11 ++-- src/diffpy/srreal/kernels/PairGeometry.hpp | 13 ++++ 6 files changed, 49 insertions(+), 60 deletions(-) diff --git a/src/diffpy/srreal/kernels/CalculationKernelCuda.cu b/src/diffpy/srreal/kernels/CalculationKernelCuda.cu index a24a294..576fb37 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelCuda.cu +++ b/src/diffpy/srreal/kernels/CalculationKernelCuda.cu @@ -128,8 +128,7 @@ struct PeakGrid { double halfboundrel; }; -// Keep the original lower grid index as the origin, including when cropping -// removes the first point. This is the coordinate used by the CPU kernels. +// Keep the original lower grid index as the origin for sampling the peak. struct PeakWindow { int origin, first, last; double xlo; @@ -146,15 +145,10 @@ __device__ PeakWindow peakWindow(const PeakGrid& grid, double dist, gridIndex(dist - half, grid.rstep, grid.rlosteps, 0, grid.npts); const int hi = gridIndex(dist + half, grid.rstep, grid.rlosteps, 1, grid.npts); - PeakWindow win{lo, lo, hi, (grid.rlosteps + lo) * grid.rstep - dist}; - if (grid.cropped) { - while (win.first < win.last && - fabs(win.x(win.first, grid.rstep) / fwhm) >= grid.halfboundrel) - ++win.first; - while (win.first < win.last && - fabs(win.x(win.last - 1, grid.rstep) / fwhm) >= grid.halfboundrel) - --win.last; - } + PeakWindow win{lo, lo, hi, (double(grid.rlosteps) + lo) * grid.rstep - dist}; + if (grid.cropped) + cropGaussianWindow(win.first, win.last, grid.rlosteps, grid.rstep, dist, + fwhm, grid.halfboundrel); return win; } diff --git a/src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp b/src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp index a477a70..0cdabe5 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp +++ b/src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp @@ -20,7 +20,7 @@ #include #if defined(LIBDIFFPY_SIMD_KERNELS) -#include +#include #endif #include diff --git a/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp b/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp index 58e4b88..517ed8a 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp +++ b/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp @@ -31,8 +31,9 @@ namespace kernels { /// xlo = (rlosteps + ilo) * rstep - dist: /// out[i] += scale * G(x, fwhm) * (x / dist + 1) /// where G(x, fwhm) = peakheight * exp(-4 * ln(2) * (x / fwhm)^2), or zero -/// for |x / fwhm| >= halfboundrel when cropped. peakheight is the profile -/// value at x = 0. Nothing is added for fwhm <= 0. +/// outside the cropped support. The cutoff uses the absolute grid offset +/// (rlosteps + i) * rstep - dist. peakheight is the profile value at x = 0. +/// Nothing is added for fwhm <= 0. struct GaussianWindowArgs { double* out; int ilo; diff --git a/src/diffpy/srreal/kernels/CalculationKernelSIMD.ipp b/src/diffpy/srreal/kernels/CalculationKernelSIMD.ipp index 3ec5cf6..cef41d4 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelSIMD.ipp +++ b/src/diffpy/srreal/kernels/CalculationKernelSIMD.ipp @@ -47,7 +47,7 @@ #include #include -#include +#include // The xsimd headers and all code below use the renamed namespace. #define xsimd LIBDIFFPY_XSIMD_NAMESPACE @@ -152,58 +152,36 @@ struct SIMDKernel { const B twowh(2 * wh); const B whwh(wh * wh); const B ratiostep(exp(-2 * b * wh * wh)); - const B vw(1.0 * W); const B vrstep(a.rstep); const B vxlo(xlo); const B vinvdist(1.0 / a.dist); const B vnorm(a.scale * a.peakheight); - const B vfwhm(a.fwhm); - const B vhalfbound(a.halfboundrel); const B one(1.0); - const B zero(0.0); - // Grid steps of the lanes from ilo, whole numbers are exact in double. - B steps = laneIndex(); + int first = a.ilo, last = a.ihi; + if (a.cropped) + cropGaussianWindow(first, last, a.rlosteps, a.rstep, a.dist, a.fwhm, + a.halfboundrel); + B steps = B(double(first - a.ilo)) + laneIndex(); // x / dist + 1 grows by a constant per vector step. const B vrscalestep(wh / a.dist); - for (int i = a.ilo; i < a.ihi;) { - const int iend = blockEnd(i, a.ihi); + for (int i = first; i < last;) { + const int iend = blockEnd(i, last); const int ivec = wholeEnd(i, iend); - // Same x as the scalar kernel, so cropping agrees. const B xseed = steps * vrstep + vxlo; B g = vnorm * xsimd::exp(negb * xseed * xseed); B ratio = xsimd::exp(negb * (twowh * xseed + whwh)); - if (a.cropped) { - for (; i < ivec; i += W) { - const B x = steps * vrstep + vxlo; - const B y = - xsimd::select(xsimd::abs(x / vfwhm) >= vhalfbound, zero, g); - addProduct(a.out + i, y, xsimd::fma(x, vinvdist, one)); - g *= ratio; - ratio *= ratiostep; - steps += vw; - } - if (i < iend) { - const B x = steps * vrstep + vxlo; - const B y = - xsimd::select(xsimd::abs(x / vfwhm) >= vhalfbound, zero, g); - addProductPartial(a.out + i, y, xsimd::fma(x, vinvdist, one), - iend - i); - return; - } - } else { - B rscaled = xsimd::fma(xseed, vinvdist, one); - // Advance before the loop, the next seeds then do not wait for it. - steps += B(double(ivec - i)); - for (; i < ivec; i += W) { - addProduct(a.out + i, g, rscaled); - g *= ratio; - ratio *= ratiostep; - rscaled += vrscalestep; - } - if (i < iend) { - addProductPartial(a.out + i, g, rscaled, iend - i); - return; - } + B rscaled = xsimd::fma(xseed, vinvdist, one); + // Advance before the loop, the next seeds then do not wait for it. + steps += B(double(ivec - i)); + for (; i < ivec; i += W) { + addProduct(a.out + i, g, rscaled); + g *= ratio; + ratio *= ratiostep; + rscaled += vrscalestep; + } + if (i < iend) { + addProductPartial(a.out + i, g, rscaled, iend - i); + return; } } } diff --git a/src/diffpy/srreal/kernels/CalculationKernelScalar.cpp b/src/diffpy/srreal/kernels/CalculationKernelScalar.cpp index cc874eb..01dddad 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelScalar.cpp +++ b/src/diffpy/srreal/kernels/CalculationKernelScalar.cpp @@ -16,7 +16,7 @@ #include #include -#include +#include namespace diffpy { namespace srreal { @@ -26,11 +26,14 @@ namespace { void gaussianWindowScalar(const GaussianWindowArgs& a) { if (a.fwhm <= 0.0) return; - const double xlo = (a.rlosteps + a.ilo) * a.rstep - a.dist; - for (int i = a.ilo; i < a.ihi; ++i) { + int first = a.ilo, last = a.ihi; + if (a.cropped) + cropGaussianWindow(first, last, a.rlosteps, a.rstep, a.dist, a.fwhm, + a.halfboundrel); + const double xlo = (double(a.rlosteps) + a.ilo) * a.rstep - a.dist; + for (int i = first; i < last; ++i) { const double x = xlo + (i - a.ilo) * a.rstep; const double xrel = x / a.fwhm; - if (a.cropped && fabs(xrel) >= a.halfboundrel) continue; const double y = a.peakheight * exp(-4 * std::numbers::ln2 * xrel * xrel); // Normalized by dist instead of r, see PeakProfile::addRDFPeak. const double yrdf = y * (x / a.dist + 1); diff --git a/src/diffpy/srreal/kernels/PairGeometry.hpp b/src/diffpy/srreal/kernels/PairGeometry.hpp index f76428b..3be414c 100644 --- a/src/diffpy/srreal/kernels/PairGeometry.hpp +++ b/src/diffpy/srreal/kernels/PairGeometry.hpp @@ -47,6 +47,19 @@ DIFFPY_HOST_DEVICE inline int gridIndex(double r, double step, int lo, return i > 0 ? (i < npts ? int(i) : npts) : 0; } +/// Crop against the given r-grid. Reconstructing x from the start of +/// the window can round a boundary sample to the other side of the cutoff. +DIFFPY_HOST_DEVICE static inline void cropGaussianWindow( + int& first, int& last, int rlosteps, double rstep, double dist, double fwhm, + double halfboundrel) { + auto outside = [&](int i) { + const double x = (double(rlosteps) + i) * rstep - dist; + return fabs(x / fwhm) >= halfboundrel; + }; + while (first < last && outside(first)) ++first; + while (first < last && outside(last - 1)) --last; +} + struct Pair { bool ok; int site0; From 476cdd6ffa95fb090401afc31db3fac688a3b7f3 Mon Sep 17 00:00:00 2001 From: SYMSCAE Date: Thu, 19 Nov 2026 10:17:24 -0800 Subject: [PATCH 5/6] fix: make CUDA kernel coefficients constant expressions --- src/diffpy/srreal/kernels/CalculationKernelCuda.cu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffpy/srreal/kernels/CalculationKernelCuda.cu b/src/diffpy/srreal/kernels/CalculationKernelCuda.cu index 576fb37..23c8909 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelCuda.cu +++ b/src/diffpy/srreal/kernels/CalculationKernelCuda.cu @@ -113,8 +113,8 @@ const int THREADS = 128; /// Pairs that one thread of the Debye sum adds for its Q point. const int SUBCHUNK = 1024; -const double FOUR_LN2 = 2.7725887222397811; -const double TWO_PI = 6.2831853071795865; +constexpr double FOUR_LN2 = 2.7725887222397811; +constexpr double TWO_PI = 6.2831853071795865; // Peaks of the PDF // -------------------------------------------------------------- From 9a1959f4ef1c9a5f6144e9305c016c497ac5f4f1 Mon Sep 17 00:00:00 2001 From: SYMSCAE Date: Thu, 19 Nov 2026 10:58:42 -0800 Subject: [PATCH 6/6] fix: handle missing CUDA error descriptions without a driver --- src/diffpy/srreal/kernels/CalculationKernelCuda.cu | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/diffpy/srreal/kernels/CalculationKernelCuda.cu b/src/diffpy/srreal/kernels/CalculationKernelCuda.cu index 23c8909..c9b05bb 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelCuda.cu +++ b/src/diffpy/srreal/kernels/CalculationKernelCuda.cu @@ -39,10 +39,17 @@ namespace { // Device ---------------------------------------------------------------------- +const char* errorDescription(cudaError_t err) { + // The Windows runtime can return nullptr when the driver is unavailable. + const char* description = cudaGetErrorString(err); + return description ? description + : "CUDA runtime error (no description available)"; +} + void check(cudaError_t err, const char* what) { if (err == cudaSuccess) return; throw std::runtime_error(std::string("the CUDA device failed in ") + what + - " with '" + cudaGetErrorString(err) + "'"); + " with '" + errorDescription(err) + "'"); } int currentDevice() { @@ -546,7 +553,7 @@ const char* cudaKernelUnavailable() { // calling thread's current device and driver without launching any work. cudaFuncAttributes attributes; const cudaError_t err = cudaFuncGetAttributes(&attributes, pdfPeaks); - return err == cudaSuccess ? nullptr : cudaGetErrorString(err); + return err == cudaSuccess ? nullptr : errorDescription(err); } PairSumStats cudaPDFPairSum(const PDFPairSumArgs& a) {