diff --git a/Detectors/TPC/spacecharge/CMakeLists.txt b/Detectors/TPC/spacecharge/CMakeLists.txt index 5615f0699546f..cdd2f7a069fc6 100644 --- a/Detectors/TPC/spacecharge/CMakeLists.txt +++ b/Detectors/TPC/spacecharge/CMakeLists.txt @@ -17,7 +17,6 @@ o2_add_library(TPCSpaceCharge src/DataContainer3D.cxx PUBLIC_LINK_LIBRARIES O2::TPCBaseRecSim O2::Field - Vc::Vc ROOT::Core O2::DataFormatsParameters) @@ -35,7 +34,6 @@ o2_target_root_dictionary(TPCSpaceCharge include/TPCSpaceCharge/PoissonSolverHelpers.h include/TPCSpaceCharge/SpaceChargeHelpers.h include/TPCSpaceCharge/TriCubic.h - include/TPCSpaceCharge/Vector.h include/TPCSpaceCharge/Vector3D.h include/TPCSpaceCharge/SpaceChargeParameter.h LINKDEF src/TPCSpacechargeLinkDef.h) @@ -49,7 +47,7 @@ o2_add_test_root_macro(macro/createResidualDistortionObject.C PUBLIC_LINK_LIBRARIES O2::TPCSpaceCharge O2::TPCSpaceChargeIO O2::CommonUtils - LABELS tpc) + LABELS tpc COMPILE_ONLY) install(FILES macro/createSCHistosFromHits.C @@ -81,6 +79,14 @@ o2_add_test(TriCubic LABELS tpc CONFIGURATIONS RelWithDebInfo Release MinSizeRel) +o2_add_test(SpaceCharge + COMPONENT_NAME spacecharge + PUBLIC_LINK_LIBRARIES O2::TPCSpaceCharge + SOURCES test/testO2TPCSpaceCharge.cxx + ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage + LABELS tpc + CONFIGURATIONS RelWithDebInfo Release MinSizeRel) + if (OpenMP_CXX_FOUND) target_compile_definitions(${targetName} PRIVATE WITH_OPENMP) target_link_libraries(${targetName} PRIVATE OpenMP::OpenMP_CXX) diff --git a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/DataContainer3D.h b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/DataContainer3D.h index 73638c5b2982f..88451320679cb 100644 --- a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/DataContainer3D.h +++ b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/DataContainer3D.h @@ -20,6 +20,8 @@ #include "Rtypes.h" #include +#include +#include class TFile; class TTree; diff --git a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/PoissonSolver.h b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/PoissonSolver.h index 8265a83ad2501..0d542ca22fe87 100644 --- a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/PoissonSolver.h +++ b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/PoissonSolver.h @@ -26,6 +26,7 @@ #include "CommonConstants/MathConstants.h" #include "TPCSpaceCharge/SpaceChargeParameter.h" #include +#include namespace o2 { @@ -41,7 +42,9 @@ class DataContainer3D; /// \class PoissonSolver /// The PoissonSolver class represents methods to solve the poisson equation. /// Original version with more methods can be found in AliTPCPoissonSolver. -/// Following methods are implemented: poissonSolver3D, poissonSolver3D2D, poissonSolver2D +/// Implemented: full multi grid (F cycle of V cycles) in 3D with red-black Gauss-Seidel relaxation, full weighting grid transfer and periodic boundary in phi. +/// The phi direction is coarsened only if it is not weakly coupled (MGParameters::phiCoarseningThreshold). On the levels where phi is strongly coupled (fine grid in phi) +/// line relaxation in phi is used instead of the point relaxation (MGParameters::phiLineRelaxThreshold). /// \tparam DataT the type of data which is used during the calculations template @@ -57,38 +60,28 @@ class PoissonSolver /// Provides poisson solver in Cylindrical 3D (TPC geometry) /// - /// Strategy based on parameter settings (mMgParameters)provided - /// * Cascaded multi grid with S.O.R - /// * Geometric MultiGrid - /// * Cycles: V, W, Full - /// * Relaxation: Jacobi, Weighted-Jacobi, Gauss-Seidel - /// * Grid transfer operators: Full, Half - /// * Spectral Methods (TODO) + /// Geometric full multi grid (F cycle of V cycles) with red-black Gauss-Seidel relaxation, full weighting grid transfer and periodic boundary in phi. + /// The parameters are set in MGParameters /// /// \param matricesV potential in 3D - /// \param matricesCharge charge density in 3D (side effect) - /// \param symmetry symmetry or not + /// \param matricesCharge charge density in 3D /// /// \pre Charge density distribution in **matricesCharge** is known and boundary values for **matricesV** are set /// \post Numerical solution for potential distribution is calculated and stored in each rod at **matricesV** - void poissonSolver3D(DataContainer& matricesV, const DataContainer& matricesCharge, const int symmetry); - - /// Provides poisson solver in 2D - /// - /// Based on the strategy (multi grid) - /// - /// \param matricesV potential in matrix - /// \param matricesCharge charge density in matrix (side effect - void poissonSolver2D(DataContainer& matricesV, const DataContainer& matricesCharge); + void poissonSolver3D(DataContainer& matricesV, const DataContainer& matricesCharge); DataT getSpacingZ() const { return mGrid3D.getSpacingZ(); } DataT getSpacingR() const { return mGrid3D.getSpacingR(); } DataT getSpacingPhi() const { return mGrid3D.getSpacingPhi(); } + /// set the stopping criterion of the multi grid V cycles. For MGParameters::relativeConvergence (default): maximum relative change sqrt(sum dV^2 / sum V^2) of the potential between two V cycles static void setConvergenceError(const DataT error) { sConvergenceError = error; } static DataT getConvergenceError() { return sConvergenceError; } + /// default stopping criterion: relative change of 1e-8 for double, 1e-6 for float (the relative change of float stays at ~1e-7 because of the rounding) + static constexpr DataT getDefaultConvergenceError() { return std::is_same_v ? DataT(1e-6) : DataT(1e-8); } + /// get the number of threads used for some of the calculations static int getNThreads() { return sNThreads; } @@ -98,45 +91,64 @@ class PoissonSolver private: const RegularGrid& mGrid3D{}; ///< grid properties const ParamSpaceCharge mParamGrid{mGrid3D.getParamSC()}; ///< parameters of the grid on which the calculations are performed - inline static DataT sConvergenceError{1e-6}; ///< Error tolerated + inline static DataT sConvergenceError{getDefaultConvergenceError()}; ///< stopping criterion of the V cycles (see MGParameters::relativeConvergence) static constexpr DataT INVTWOPI = 1. / o2::constants::math::TwoPI; ///< inverse of 2*pi - inline static int sNThreads{4}; ///< number of threads which are used during some of the calculations (increasing this number has no big impact) + inline static int sNThreads{4}; ///< number of threads used for the relaxation, residue, restriction, interpolation and convergence calculation on the larger multi grid levels (set by SpaceCharge::poissonSolver() to SpaceCharge::getNThreads()) + static constexpr long MINPOINTSPARALLEL{20000}; ///< minimum number of grid points of a multi grid level for which the calculations are performed in parallel /// \returns inverse grid size in phi (either 1/2Pi or NSECTORSPERSIDE/2Pi) static DataT getGridSizePhiInv(); - /// Relative error calculation: comparison with exact solution + /// \return returns the change of the potential between two V cycles (stopping criterion of the multi grid): + /// MGParameters::relativeConvergence: sqrt(sum dV^2 / sum V^2), otherwise the maximum of the sum of dV^2 of the phi slices /// /// \param matricesCurrentV current potential (numerical solution) /// \param prevArrayV content from matricesCurrentV from previous iteration - DataT getConvergenceError(const Vector& matricesCurrentV, Vector& prevArrayV) const; + DataT getConvergenceError(const Vector& matricesCurrentV, const Vector& prevArrayV) const; + + /// grid of one level of the multi grid + struct MGLevel { + int nr{}; ///< number of vertices in r + int nz{}; ///< number of vertices in z + int nphi{}; ///< number of vertices in phi + int rzFactor{}; ///< grid spacing in r and z in units of the grid spacing of the finest grid + bool lineRelaxPhi{}; ///< use line relaxation in phi (phi strongly coupled, see MGParameters::phiLineRelaxThreshold) instead of point relaxation + }; - /// 3D - Solve Poisson's Equation in 3D by MultiGrid with constant phi slices - /// - /// NOTE: In order for this algorithm to work, the number of Nr and Nz must be a power of 2 plus one. - /// The number of Nr and Z Column can be different. - /// - /// R Row == 2**M + 1 - /// Z Column == 2**N + 1 - /// Phi Slice == Arbitrary but greater than 3 - /// - /// Solving: \f$ \nabla^{2}V(r,\phi,z) = - f(r,\phi,z) \f$ - /// - /// Algorithm for MultiGrid Full Cycle (FMG) - /// - Relax on the coarsest grid - /// - Do from coarsest to finest - /// - Interpolate potential from coarse -> fine - /// - Do V-Cycle to the current coarse level to the coarsest - /// - Stop if converged - /// - /// DeltaPhi in Radians - /// \param matricesV potential in 3D matrix \f$ V(r,\phi,z) \f$ - /// \param matricesCharge charge density in 3D matrix (side effect) \f$ - f(r,\phi,z) \f$ - /// \param symmetry symmetry (TODO for symmetry = 1) - // - /// SYMMETRY = 0 if no phi symmetries, and no phi boundary condition - /// = 1 if we have reflection symmetry at the boundaries (eg. sector symmetry or half sector symmetries). - void poissonMultiGrid3D2D(DataContainer& matricesV, const DataContainer& matricesCharge, const int symmetry); + /// relaxation of the given level: line relaxation in phi (MGLevel::lineRelaxPhi) or point relaxation (relax3D()) + void relaxLevel(const MGLevel& level, Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const DataT h2, const DataT tempRatioZ, + const std::vector& coefficient1, const std::vector& coefficient2, const std::vector& coefficient3, const std::vector& coefficient4) const; + + /// red-black Gauss-Seidel line relaxation in phi direction: the potential of all phi slices on one line (r, z) is solved exactly (periodic tridiagonal system) + /// for given potential on the neighbouring lines. The lines with (ir + iz) even and odd are relaxed one after the other. + /// Robust if the phi direction is strongly coupled compared to r and z (fine grid in phi), where the point relaxation smooths the error badly. + void relaxLinePhi(Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const int iPhi, const DataT h2, const DataT tempRatioZ, + const std::vector& coefficient1, const std::vector& coefficient2, const std::vector& coefficient3, const std::vector& coefficient4) const; + + /// \returns the grids of the multi grid levels. From one level to the next the r and z direction are coarsened and the phi direction is coarsened (if the number of slices is even) + /// only if it is not weakly coupled compared to r and z (see MGParameters::phiCoarseningThreshold). Line relaxation in phi is used on the levels where phi is strongly coupled (see MGParameters::phiLineRelaxThreshold) + /// \param nLevelsRZ number of levels for the coarsening in r and z + std::vector getMultiGridLevels(const int nLevelsRZ) const; + + /// \return returns the relative coupling (h_{r,z} / (radius * h_{phi}))^2 in phi direction compared to r and z for given grid and radius + DataT getRelCouplingPhi(const MGLevel& level, const DataT radius) const; + + /// restriction of the fine level to the coarse level (full weighting) + void restrictLevel(Vector& coarse, const Vector& fine, const MGLevel& levelCoarse, const MGLevel& levelFine) const; + + /// restriction of the boundary values of the fine level to the coarse level + void restrictBoundaryLevel(Vector& coarse, const Vector& fine, const MGLevel& levelCoarse, const MGLevel& levelFine) const; + + /// interpolation of the coarse level to the fine level: set (Add = false) or add (Add = true) the interpolated values + template + void interpolateLevel(Vector& fine, const Vector& coarse, const MGLevel& levelFine, const MGLevel& levelCoarse) const; + + /// get the indices of the neighbouring phi slices for given phi slice (periodic boundary in phi) + /// \param m phi slice + /// \param nPhi number of phi slices + /// \param mp1 index of the slice m + 1 (output) + /// \param mm1 index of the slice m - 1 (output) + static void getPhiNeighbours(const int m, const int nPhi, int& mp1, int& mm1); /// 3D - Solve Poisson's Equation in 3D in all direction by MultiGrid /// @@ -150,27 +162,15 @@ class PoissonSolver /// Solving: \f$ \nabla^{2}V(r,\phi,z) = - f(r,\phi,z) \f$ /// /// Algorithm for MultiGrid Full Cycle (FMG) + /// - Restrict the charge density and the boundary potential to the coarser grids (see getMultiGridLevels()) /// - Relax on the coarsest grid /// - Do from coarsest to finest - /// - Interpolate potential from coarse -> fine - /// - Do V-Cycle to the current coarse level to the coarsest - /// - Stop if converged + /// - Interpolate potential from coarse -> fine + /// - Do V-Cycles from the current level to the coarsest level until converged (see getConvergenceError()) /// - /// \param matricesV potential in 3D matrix - /// \param matricesCharge charge density in 3D matrix (side effect) - /// \param symmetry symmetry or not: symmetry = 0 if no phi symmetries, and no phi boundary condition. - /// symmetry = 1 if we have reflection symmetry at the boundaries (eg. sector symmetry or half sector symmetries). - void poissonMultiGrid3D(DataContainer& matricesV, const DataContainer& matricesCharge, const int symmetry); - - /// Solve Poisson's Equation by MultiGrid Technique in 2D (assuming cylindrical symmetry) - /// - /// NOTE: In order for this algorithm to work, the number of nRRow and nZColumn must be a power of 2 plus one. - /// So nRRow == 2**M + 1 and nZColumn == 2**N + 1. The number of nRRow and nZColumn can be different. - /// - /// \param matricesV potential in matrix - /// \param matricesCharge charge density in matrix (side effect - /// \param iPhi phi vertex - void poissonMultiGrid2D(DataContainer& matricesV, const DataContainer& matricesCharge, const int iPhi = 0); + /// \param matricesV potential in 3D matrix (input: boundary potential, output: solution) + /// \param matricesCharge charge density in 3D matrix + void poissonMultiGrid3D(DataContainer& matricesV, const DataContainer& matricesCharge); /// Restrict2D /// @@ -231,128 +231,30 @@ class PoissonSolver /// \param tnRRow number of grid in in r-direction for coarser grid should be 2^N + 1, finer grid in 2^{N+1} + 1 /// \param tnZColumn number of grid in in z-direction for coarser grid should be 2^M + 1, finer grid in 2^{M+1} + 1 /// \param iPhi phi vertex - /// \param symmetry is the cylinder has symmetry /// \param h2 \f$ h_{r}^{2} \f$ /// \param tempRatioZ ration between grid size in z-direction and r-direction /// \param coefficient1 coefficients for \f$ V_{x+1,y,z} \f$ /// \param coefficient2 coefficients for \f$ V_{x-1,y,z} \f$ /// \param coefficient3 coefficients for z /// \param coefficient4 coefficients for f(r,\phi,z) - void relax3D(Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const int iPhi, const int symmetry, const DataT h2, const DataT tempRatioZ, + void relax3D(Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const int iPhi, const DataT h2, const DataT tempRatioZ, const std::vector& coefficient1, const std::vector& coefficient2, const std::vector& coefficient3, const std::vector& coefficient4) const; - /// Relax2D - /// - /// Relaxation operation for multiGrid - /// relaxation used 5 stencil in cylindrical coordinate - /// - /// Using the following equations - /// \f$ U_{i,j,k} = (1 + \frac{1}{r_{i}h_{r}}) U_{i+1,j,k} + (1 - \frac{1}{r_{i}h_{r}}) U_{i+1,j,k} \f$ - /// - /// \param matricesCurrentV potential in 3D (matrices of matrix) - /// \param matricesCurrentCharge charge in 3D - /// \param tnRRow number of vertices in r direction of TPC - /// \param tnZColumn number of vertices in z direction of TPC - /// \param h2 \f$ h_{r}^{2} \f$ - /// \param tempFourth coefficient for h - /// \param tempRatio ratio between grid size in z-direction and r-direction - /// \param coefficient1 coefficient for \f$ V_{x+1,y,z} \f$ - /// \param coefficient2 coefficient for \f$ V_{x-1,y,z} \f$ - void relax2D(Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const DataT h2, const DataT tempFourth, const DataT tempRatio, - std::vector& coefficient1, std::vector& coefficient2); - - /// Interpolation/Prolongation in 2D - /// - /// Interpolation is a map from coarse grid (h) to fine grid (2h) - /// - /// In case of 2D - /// Full weighting: - /// \f[ (R u)_{i,j,k} = \frac{1}{2} u_{2i,2j,2k} + \frac{1}{4} S_{1} + \frac{1}{8} S_{2} + \frac{1}{16} S_{3}\f] - /// - /// - /// Restriction in all direction r-phi-z - /// \param matricesCurrentV finer grid h - /// \param matricesCurrentVC coarse grid 2h - /// \param tnRRow number of grid in r-direction for coarser grid should be 2^N + 1, finer grid in 2^{N+1} + 1 - /// \param tnZColumn number of grid in z-direction for coarser grid should be 2^M + 1, finer grid in 2^{M+1} + 1 - /// \param iphi phi vertex - void interp2D(Vector& matricesCurrentV, const Vector& matricesCurrentVC, const int tnRRow, const int tnZColumn, const int iphi) const; - - /// Interpolation/Prolongation in 3D - /// - /// Interpolation is a map from coarse grid (h) to fine grid (2h) - /// - /// In case of 3D - /// Full weighting: - /// \f[ (R u)_{i,j,k} = \frac{1}{2} u_{2i,2j,2k} + \frac{1}{4} S_{1} + \frac{1}{8} S_{2} + \frac{1}{16} S_{3}\f] - /// - /// - /// Restriction in all direction r-phi-z - /// restriction in phi only if oldPhi == 2*newPhi - /// \param matricesCurrentV finer grid h - /// \param matricesCurrentVC coarse grid 2h - /// \param tnRRow number of vertices in r-direction for coarser grid should be 2^N + 1, finer grid in 2^{N+1} + 1 - /// \param tnZColumn number of vertices in z-direction for coarser grid should be 2^M + 1, finer grid in 2^{M+1} + 1 - /// \param newPhiSlice number of vertices in phi-direction for coarser grid - /// \param oldPhiSlice number of vertices in phi-direction for finer grid - void interp3D(Vector& matricesCurrentV, const Vector& matricesCurrentVC, const int tnRRow, const int tnZColumn, const int newPhiSlice, const int oldPhiSlice) const; - - /// Prolongation with Addition for 3D - /// - /// Interpolation with addition from coarse level (2h) --> fine level (h) - /// - /// Interpolation in all direction r-phi-z - /// Interpolation in phi only if oldPhi == 2*newPhi - /// \param matricesCurrentV fine grid h - /// \param matricesCurrentVC coarse grid 2h - /// \param tnRRow number of vertices in r-direction for coarser grid should be 2^N + 1, finer grid in 2^{N+1} + 1 - /// \param tnZColumn number of vertices in z-direction for coarser grid should be 2^M + 1, finer grid in 2^{M+1} + 1a - /// \param newPhiSlice number of vertices in phi-direction for coarser grid - /// \param oldPhiSlice number of vertices in phi-direction for finer grid - void addInterp3D(Vector& matricesCurrentV, const Vector& matricesCurrentVC, const int tnRRow, const int tnZColumn, const int newPhiSlice, const int oldPhiSlice) const; - - /// Prolongation with Addition for 2D - /// - /// Interpolation with addition from coarse level (2h) --> fine level (h) - /// - /// Interpolation in all direction r-phi-z + /// Interpolation/prolongation from the coarse grid (2h) to the fine grid (h): set (Add = false) or add (Add = true) the interpolated values + /// Interpolation in all directions r-phi-z, in phi only if newPhiSlice == 2 * oldPhiSlice /// \param matricesCurrentV fine grid h /// \param matricesCurrentVC coarse grid 2h - /// \param tnRRow number of vertices in r-direction for coarser grid should be 2^N + 1, finer grid in 2^{N+1} + 1 - /// \param tnZColumn number of vertices in z-direction for coarser grid should be 2^M + 1, finer grid in 2^{M+1} + 1a - /// \param tnPhi phi vertices - void addInterp2D(Vector& matricesCurrentV, const Vector& matricesCurrentVC, const int tnRRow, const int tnZColumn, const int tnPhi) const; - - /// VCycle 3D2D, V Cycle 3D in multiGrid with constant Nphi - /// fine-->coarsest-->fine, propagating the residue to correct initial guess of V - /// - /// Algorithm: - /// - /// NOTE: In order for this algorithm to work, the number of Nr and Nz must be a power of 2 plus one. - /// The number of Nr and Z Column can be different. - /// - /// R Row == 2**M + 1 - /// Z Column == 2**N + 1 - /// Phi Slice == Arbitrary but greater than 3 - /// - /// DeltaPhi in Radians - /// \param gridFrom finest level of grid - /// \param gridTo coarsest level of grid - /// \param nPre number of smoothing before coarsening - /// \param nPost number of smoothing after coarsening - /// \param ratioZ ratio between square of grid r and grid z (OPTION, recalculate) - /// \param ratioPhi ratio between square of grid r and grid phi (OPTION, recalculate) - /// \param tvArrayV vector of V potential in different grids - /// \param tvCharge vector of charge distribution in different grids - /// \param tvResidue vector of residue calculation in different grids - /// \param coefficient1 coefficient for relaxation (r direction) - /// \param coefficient2 coefficient for relaxation (r direction) - /// \param coefficient3 coefficient for relaxation (ratio r/z) - /// \param coefficient4 coefficient for relaxation (ratio for grid_r) - /// \param inverseCoefficient4 coefficient for relaxation (inverse coefficient4) - void vCycle3D2D(const int symmetry, const int gridFrom, const int gridTo, const int nPre, const int nPost, const DataT ratioZ, const DataT ratioPhi, std::vector& tvArrayV, - std::vector& tvCharge, std::vector& tvResidue, std::vector& coefficient1, std::vector& coefficient2, std::vector& coefficient3, - std::vector& coefficient4, std::vector& inverseCoefficient4) const; + /// \param tnRRow number of vertices in r-direction of the fine grid + /// \param tnZColumn number of vertices in z-direction of the fine grid + /// \param newPhiSlice number of vertices in phi-direction of the fine grid + /// \param oldPhiSlice number of vertices in phi-direction of the coarse grid + template + void interpolate3D(Vector& matricesCurrentV, const Vector& matricesCurrentVC, const int tnRRow, const int tnZColumn, const int newPhiSlice, const int oldPhiSlice) const; + + /// Interpolation/prolongation in r and z from the coarse grid (2h) to the fine grid (h) for one phi slice: set (Add = false) or add (Add = true) the interpolated values + /// \param iphi phi slice + template + void interpolate2D(Vector& matricesCurrentV, const Vector& matricesCurrentVC, const int tnRRow, const int tnZColumn, const int iphi) const; /// VCycle 3D, V Cycle in multiGrid, fine-->coarsest-->fine, propagating the residue to correct initial guess of V /// @@ -365,12 +267,12 @@ class PoissonSolver /// /// DeltaPhi in Radians /// - /// \param symmetry symmetry or not /// \param gridFrom finest level of grid /// \param gridTo coarsest level of grid /// \param nPre number of smoothing before coarsening /// \param nPost number of smoothing after coarsening - /// \param ratioZ ratio between square of grid r and grid z (OPTION, recalculate) + /// \param ratioZ ratio between square of grid r and grid z + /// \param levels grids of the multi grid levels /// \param tvArrayV vector of V potential in different grids /// \param tvCharge vector of charge distribution in different grids /// \param tvResidue vector of residue calculation in different grids @@ -379,47 +281,10 @@ class PoissonSolver /// \param coefficient3 coefficient for relaxation (ratio r/z) /// \param coefficient4 coefficient for relaxation (ratio for grid_r) /// \param inverseCoefficient4 coefficient for relaxation (inverse coefficient4) - void vCycle3D(const int symmetry, const int gridFrom, const int gridTo, const int nPre, const int nPost, const DataT ratioZ, std::vector& tvArrayV, std::vector& tvCharge, + void vCycle3D(const int gridFrom, const int gridTo, const int nPre, const int nPost, const DataT ratioZ, const std::vector& levels, std::vector& tvArrayV, std::vector& tvCharge, std::vector& tvResidue, std::vector& coefficient1, std::vector& coefficient2, std::vector& coefficient3, std::vector& coefficient4, std::vector& inverseCoefficient4) const; - /// V-Cycle 2D - /// - /// Implementation non-recursive V-cycle for 2D - /// - /// Algorithms: - /// - /// \param gridFrom finest level of grid - /// \param gridTo coarsest level of grid - /// \param nPre number of smoothing before coarsening - /// \param nPost number of smoothing after coarsening - /// \param gridSizeR grid size in r direction (OPTION, recalculate) - /// \param ratio ratio between square of grid r and grid z (OPTION, recalculate) - /// \param tvArrayV vector of V potential in different grids - /// \param tvCharge vector of charge distribution in different grids - /// \param tvResidue vector of residue calculation in different grids - void vCycle2D(const int gridFrom, const int gridTo, const int nPre, const int nPost, const DataT gridSizeR, const DataT ratio, std::vector& tvArrayV, - std::vector& tvCharge, std::vector& tvResidue); - - /// W-Cycle 2D - /// - /// Implementation non-recursive W-cycle for 2D - /// - /// Algorithms: - /// - /// \param gridFrom finest level of grid - /// \param gridTo coarsest level of grid - /// \param gamma number of iterations at coarsest level - /// \param nPre number of smoothing before coarsening - /// \param nPost number of smoothing after coarsening - /// \param gridSizeR grid size in r direction (OPTION, recalculate) - /// \param ratio ratio between square of grid r and grid z (OPTION, recalculate) - /// \param tvArrayV vector of V potential in different grids - /// \param tvCharge vector of charge distribution in different grids - /// \param tvResidue vector of residue calculation in different grids - void wCycle2D(const int gridFrom, const int gridTo, const int gamma, const int nPre, const int nPost, const DataT gridSizeR, const DataT ratio, - std::vector& tvArrayV, std::vector& tvCharge, std::vector& tvResidue); - /// Residue3D /// /// Compute residue from V(.) where V(.) is numerical potential and f(.). @@ -434,53 +299,29 @@ class PoissonSolver /// \param tnRRow number of vertices in the r direction of TPC /// \param tnZColumn number of vertices in z direction of TPC /// \param tnPhi number of vertices in phi direction of TPC - /// \param symmetry if the cylinder has symmetry /// \param ih2 \f$ 1/ h_{r}^{2} \f$ /// \param tempRatioZ ration between grid size in z-direction and r-direction /// \param coefficient1 coefficient for \f$ V_{x+1,y,z} \f$ /// \param coefficient2 coefficient for \f$ V_{x-1,y,z} \f$ /// \param coefficient3 coefficient for z /// \param inverseCoefficient4 inverse coefficient for f(r,\phi,z) - void residue3D(Vector& residue, const Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const int tnPhi, const int symmetry, const DataT ih2, const DataT tempRatioZ, + void residue3D(Vector& residue, const Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const int tnPhi, const DataT ih2, const DataT tempRatioZ, const std::vector& coefficient1, const std::vector& coefficient2, const std::vector& coefficient3, const std::vector& inverseCoefficient4) const; - /// Residue2D - /// - /// Compute residue from V(.) where V(.) is numerical potential and f(.). - /// residue used 5 stencil in cylindrical coordinate - /// - /// Using the following equations - /// \f$ U_{i,j,k} = (1 + \frac{1}{r_{i}h_{r}}) U_{i+1,j,k} + (1 - \frac{1}{r_{i}h_{r}}) U_{i+1,j,k} \f$ - /// - /// \param residue potential in 2D - /// \param matricesCurrentV potential in 2D - /// \param matricesCurrentCharge charge in 2D - /// \param nRRow number of nRRow in the r direction of TPC - /// \param nZColumn number of nZColumn in z direction of TPC - /// \param ih2 \f$ h_{r}^{2} \f$ - /// \param iTempFourth coefficient for h - /// \param tempRatio ratio between grid size in z-direction and r-direction - /// \param coefficient1 coefficient for \f$ V_{x+1,y,z} \f$ - /// \param coefficient2 coefficient for \f$ V_{x-1,y,z} \f$ - void residue2D(Vector& residue, const Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const DataT ih2, const DataT inverseTempFourth, - const DataT tempRatio, std::vector& coefficient1, std::vector& coefficient2); - /// Boundary transfer restrict from fine -> coarse grid /// /// \param matricesCurrentCharge coarse grid (2h) /// \param residue fine grid (h) /// \param tnRRow number of vertices in the r direction of TPC /// \param tnZColumn number of vertices in z direction of TPC - /// \param tnPhi phi vertices - void restrictBoundary2D(Vector& matricesCurrentCharge, const Vector& residue, const int tnRRow, const int tnZColumn, const int tnPhi) const; + /// \param phiCoarse phi slice of the coarse grid + /// \param phiFine phi slice of the fine grid + void restrictBoundary2D(Vector& matricesCurrentCharge, const Vector& residue, const int tnRRow, const int tnZColumn, const int phiCoarse, const int phiFine) const; // calculate coefficients void calcCoefficients(unsigned int from, unsigned int to, const DataT h, const DataT tempRatioZ, const DataT tempRatioPhi, std::vector& coefficient1, std::vector& coefficient2, std::vector& coefficient3, std::vector& coefficient4) const; - // calculate coefficients for 2D poisson solver - void calcCoefficients2D(unsigned int from, unsigned int to, const DataT h, std::vector& coefficient1, std::vector& coefficient2) const; - /// Helper function to check if the integer is equal to a power of two /// \param i the number /// \return 1 if it is a power of two, else 0 diff --git a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/PoissonSolverHelpers.h b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/PoissonSolverHelpers.h index 933273d0b5eb9..b30f63dbfaa4d 100644 --- a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/PoissonSolverHelpers.h +++ b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/PoissonSolverHelpers.h @@ -26,37 +26,14 @@ namespace o2 namespace tpc { -///< Enumeration of Cycles Type -enum class CycleType { - VCycle = 0, ///< V Cycle - WCycle = 1, ///< W Cycle (TODO) - FCycle = 2 ///< Full Cycle -}; - -///< Fine -> Coarse Grid transfer operator types -enum class GridTransferType { - Half = 0, ///< Half weighting - Full = 1, ///< Full weighting -}; - -///< Smoothing (Relax) operator types -enum class RelaxType { - Jacobi = 0, ///< Jacobi (5 Stencil 2D, 7 Stencil 3D_ - WeightedJacobi = 1, ///< (TODO) - GaussSeidel = 2 ///< Gauss Seidel 2D (2 Color, 5 Stencil), 3D (7 Stencil) -}; - -struct MGParameters { ///< Parameters choice for MultiGrid algorithm - inline static bool isFull3D = true; ///< TRUE: full coarsening, FALSE: semi coarsening - inline static CycleType cycleType = CycleType::FCycle; ///< cycleType follow CycleType - inline static GridTransferType gtType = GridTransferType::Full; ///< gtType grid transfer type follow GridTransferType - inline static RelaxType relaxType = RelaxType::GaussSeidel; ///< relaxType follow RelaxType - inline static int nPre = 2; ///< number of iteration for pre smoothing - inline static int nPost = 2; ///< number of iteration for post smoothing - inline static int nMGCycle = 200; ///< number of multi grid cycle (V type) - inline static int maxLoop = 7; ///< the number of tree-deep of multi grid - inline static int gamma = 1; ///< number of iteration at coarsest level !TODO SET TO REASONABLE VALUE! - inline static int normalizeGridToNSector = SECTORSPERSIDE; ///< the grid in phi direction is squashed from 2 Pi to (2 Pi / SECTORSPERSIDE). This can used to get the potential for phi symmetric sc density or boundary potentials +struct MGParameters { ///< Parameters of the multi grid algorithm + inline static int nPre = 2; ///< number of iteration for pre smoothing + inline static int nPost = 2; ///< number of iteration for post smoothing + inline static int nMGCycle = 200; ///< maximum number of V cycles on each level of the full multi grid + inline static int normalizeGridToNSector = SECTORSPERSIDE; ///< the grid in phi direction is squashed from 2 Pi to (2 Pi / SECTORSPERSIDE). This can used to get the potential for phi symmetric sc density or boundary potentials + inline static float phiCoarseningThreshold = 0.5; ///< full 3D: the phi direction is only coarsened if the relative coupling in phi (h_{r,z} / (r_{min} * h_{phi}))^2 is larger than this value (weakly coupled directions should not be coarsened). 0: always coarsen phi if possible + inline static float phiLineRelaxThreshold = 4; ///< full 3D: line relaxation in phi instead of point relaxation on the multi grid levels where the relative coupling in phi (h_{r,z} / (r_{min} * h_{phi}))^2 is larger than this value. 0: always point relaxation + inline static bool relativeConvergence = true; ///< stopping criterion of the multi grid V cycles: true: relative change of the potential between two V cycles sqrt(sum dV^2 / sum V^2) (independent of the units of the potential and the grid size), false: maximum over the phi slices of the sum of dV^2 (depends on the units and on the grid size) }; template diff --git a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/RegularGrid3D.h b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/RegularGrid3D.h index fa9831b81f5bd..8faa18c8cfa39 100644 --- a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/RegularGrid3D.h +++ b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/RegularGrid3D.h @@ -17,9 +17,10 @@ #ifndef ALICEO2_TPC_REGULARGRID3D_H_ #define ALICEO2_TPC_REGULARGRID3D_H_ -#include "TPCSpaceCharge/Vector.h" #include "Rtypes.h" // for ClassDefNV #include "TPCSpaceCharge/SpaceChargeParameter.h" +#include +#include #include namespace o2 @@ -90,17 +91,17 @@ struct RegularGrid3D { static constexpr unsigned int getFR() { return FR; } /// \return returns the index for dimension y (1) static constexpr unsigned int getFPhi() { return FPHI; } /// \return returns the index for dimension z (2) - const Vector& getGridMin() const { return mMin; } /// \return returns the minimum coordinates of the grid in all dimensions - DataT getGridMinZ() const { return mMin[FZ]; } /// \return returns the minimum coordinate of the grid in x dimension - DataT getGridMinR() const { return mMin[FR]; } /// \return returns the minimum coordinate of the grid in y dimension - DataT getGridMinPhi() const { return mMin[FPHI]; } /// \return returns the minimum coordinate of the grid in z dimension + const std::array& getGridMin() const { return mMin; } /// \return returns the minimum coordinates of the grid in all dimensions + DataT getGridMinZ() const { return mMin[FZ]; } /// \return returns the minimum coordinate of the grid in x dimension + DataT getGridMinR() const { return mMin[FR]; } /// \return returns the minimum coordinate of the grid in y dimension + DataT getGridMinPhi() const { return mMin[FPHI]; } /// \return returns the minimum coordinate of the grid in z dimension DataT getGridMaxZ() const { return mMax[FZ]; } DataT getGridMaxR() const { return mMax[FR]; } DataT getGridMaxPhi() const { return mMax[FPHI]; } /// \return returns the inversed spacing of the grid for all dimensions - const Vector& getInvSpacing() const { return mInvSpacing; } + const std::array& getInvSpacing() const { return mInvSpacing; } DataT getInvSpacingZ() const { return mInvSpacing[FZ]; } DataT getInvSpacingR() const { return mInvSpacing[FR]; } DataT getInvSpacingPhi() const { return mInvSpacing[FPHI]; } @@ -141,7 +142,7 @@ struct RegularGrid3D { /// \return returns the z positon for given vertex DataT getPhiVertex(const size_t vertexZ) const { return mPhiVertices[vertexZ]; } - const Vector& getMaxIndices() const { return sMaxIndex; } /// get max indices for all dimensions + const std::array& getMaxIndices() const { return sMaxIndex; } /// get max indices for all dimensions DataT getMaxIndexZ() const { return sMaxIndex[0]; } /// get max index in x direction DataT getMaxIndexR() const { return sMaxIndex[1]; } /// get max index in y direction @@ -159,19 +160,19 @@ struct RegularGrid3D { private: ParamSpaceCharge mParamGrid{}; - static constexpr unsigned int FDIM = 3; ///< dimensions of the grid (only 3 supported) - static constexpr unsigned int FZ = 0; ///< index for x coordinate - static constexpr unsigned int FR = 1; ///< index for y coordinate - static constexpr unsigned int FPHI = 2; ///< index for z coordinate - Vector mMin{}; /// mMax{}; /// mSpacing{}; /// mInvSpacing{}; /// sMaxIndex{{static_cast(mParamGrid.NZVertices - 1.), static_cast(mParamGrid.NRVertices - 1), static_cast(mParamGrid.NPhiVertices - 1)}}; /// sNdim{{static_cast(mParamGrid.NZVertices), static_cast(mParamGrid.NRVertices), static_cast(mParamGrid.NPhiVertices)}}; /// mZVertices{}; ///< positions of vertices in x direction - std::vector mRVertices{}; ///< positions of vertices in y direction - std::vector mPhiVertices{}; ///< positions of vertices in z direction + static constexpr unsigned int FDIM = 3; ///< dimensions of the grid (only 3 supported) + static constexpr unsigned int FZ = 0; ///< index for x coordinate + static constexpr unsigned int FR = 1; ///< index for y coordinate + static constexpr unsigned int FPHI = 2; ///< index for z coordinate + std::array mMin{}; /// mMax{}; /// mSpacing{}; /// mInvSpacing{}; /// sMaxIndex{{static_cast(mParamGrid.NZVertices - 1.), static_cast(mParamGrid.NRVertices - 1), static_cast(mParamGrid.NPhiVertices - 1)}}; /// sNdim{{static_cast(mParamGrid.NZVertices), static_cast(mParamGrid.NRVertices), static_cast(mParamGrid.NPhiVertices)}}; /// mZVertices{}; ///< positions of vertices in x direction + std::vector mRVertices{}; ///< positions of vertices in y direction + std::vector mPhiVertices{}; ///< positions of vertices in z direction void initLists(); @@ -217,6 +218,11 @@ DataT RegularGrid3D::clampToGridRel(const DataT pos, const unsigned int d template DataT RegularGrid3D::clampToGridCircular(DataT pos, const unsigned int dim) const { + const DataT period = mMax[dim] - mMin[dim] + mSpacing[dim]; + if (!(std::abs(pos - mMin[dim]) < 4 * period)) { + // far outside of the grid or not finite: wrap directly with the exact remainder instead of looping (a non-finite position becomes NaN) + pos = mMin[dim] + std::fmod(pos - mMin[dim], period); + } while (pos < mMin[dim]) { pos += mMax[dim] - mMin[dim] + mSpacing[dim]; } @@ -229,6 +235,11 @@ DataT RegularGrid3D::clampToGridCircular(DataT pos, const unsigned int di template DataT RegularGrid3D::clampToGridCircularRel(DataT pos, const unsigned int dim) const { + const DataT period = sNdim[dim]; + if (!(std::abs(pos) < 4 * period)) { + // far outside of the grid or not finite: wrap directly with the exact remainder instead of looping (a non-finite position becomes NaN) + pos = std::fmod(pos, period); + } while (pos < 0) { pos += sNdim[dim]; } diff --git a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/SpaceCharge.h b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/SpaceCharge.h index d0c66d0ff3df1..0762f14890f74 100644 --- a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/SpaceCharge.h +++ b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/SpaceCharge.h @@ -26,8 +26,10 @@ #include "TPCSpaceCharge/DataContainer3D.h" #include "TPCSpaceCharge/SpaceChargeParameter.h" #include "DataFormatsTPC/Defs.h" +#include class TH3; +class TAxis; class TH3D; class TH3F; class TH2F; @@ -282,14 +284,12 @@ class SpaceCharge /// step 1: use the O2TPCPoissonSolver class to numerically calculate the potential with set space charge density and boundary conditions from potential /// \param side side of the TPC - /// \param stoppingConvergence stopping criterion used in the poisson solver - /// \param symmetry use symmetry or not in the poisson solver - void poissonSolver(const Side side, const DataT stoppingConvergence = 1e-6, const int symmetry = 0); + /// \param stoppingConvergence stopping criterion used in the poisson solver (relative change of the potential between two multi grid V cycles, see MGParameters::relativeConvergence; default 1e-8 for double, 1e-6 for float) + void poissonSolver(const Side side, const DataT stoppingConvergence = std::is_same_v ? DataT(1e-6) : DataT(1e-8)); /// step 1: use the O2TPCPoissonSolver class to numerically calculate the potential with set space charge density and boundary conditions from potential for A and C side in parallel - /// \param stoppingConvergence stopping criterion used in the poisson solver - /// \param symmetry use symmetry or not in the poisson solver - void poissonSolver(const DataT stoppingConvergence = 1e-6, const int symmetry = 0); + /// \param stoppingConvergence stopping criterion used in the poisson solver (relative change of the potential between two multi grid V cycles, see MGParameters::relativeConvergence; default 1e-8 for double, 1e-6 for float) + void poissonSolver(const DataT stoppingConvergence = std::is_same_v ? DataT(1e-6) : DataT(1e-8)); /// step 2: calculate numerically the electric field from the potential /// \param side side of the TPC @@ -865,7 +865,7 @@ class SpaceCharge DataT getPotential(const size_t iz, const size_t ir, const size_t iphi, const Side side) const { return mPotential[side](iz, ir, iphi); } /// Get the step width which is used for the calculation of the correction/distortions in units of the z-bin - static int getStepWidth() { return 1 / sSteps; } + static DataT getStepWidth() { return DataT(1) / sSteps; } /// Get phi vertex position for index in phi direction /// \param indexPhi index in phi direction @@ -1365,8 +1365,9 @@ class SpaceCharge void integrateEFieldsSimpson(const DataT p1r, const DataT p1phi, const DataT p1z, const DataT p2z, DataT& localIntErOverEz, DataT& localIntEPhiOverEz, DataT& localIntDeltaEz, const Fields& formulaStruct, const DataT ezField, const Side side) const; /// integrate electrical fields using simpson integration method with non straight drift of electrons + /// \param fieldsStart fields (z, r, phi) at the start point p1 template > - void integrateEFieldsSimpsonIterative(const DataT p1r, const DataT p2r, const DataT p1phi, const DataT p2phi, const DataT p1z, const DataT p2z, DataT& localIntErOverEz, DataT& localIntEPhiOverEz, DataT& localIntDeltaEz, const Fields& formulaStruct, const DataT ezField, const Side side) const; + void integrateEFieldsSimpsonIterative(const DataT p1r, const DataT p2r, const DataT p1phi, const DataT p2phi, const DataT p1z, const DataT p2z, const DataT (&fieldsStart)[3], DataT& localIntErOverEz, DataT& localIntEPhiOverEz, DataT& localIntDeltaEz, const Fields& formulaStruct, const DataT ezField, const Side side) const; /// calculate distortions/corrections using analytical electric fields void processGlobalDistCorr(const DataT radius, const DataT phi, const DataT z0Tmp, const DataT z1Tmp, DataT& ddR, DataT& ddPhi, DataT& ddZ, const AnalyticalFields& formulaStruct) const { calcDistCorr(radius, phi, z0Tmp, z1Tmp, ddR, ddPhi, ddZ, formulaStruct, false, formulaStruct.getSide()); } @@ -1377,9 +1378,9 @@ class SpaceCharge /// calculate distortions/corrections by interpolation of local distortions/corrections void processGlobalDistCorr(const DataT radius, const DataT phi, const DataT z0Tmp, [[maybe_unused]] const DataT z1Tmp, DataT& ddR, DataT& ddPhi, DataT& ddZ, const DistCorrInterpolator& localDistCorr) const { - ddR = localDistCorr.evaldR(z0Tmp, radius, phi); - ddZ = localDistCorr.evaldZ(z0Tmp, radius, phi); - ddPhi = localDistCorr.evaldRPhi(z0Tmp, radius, phi) / radius; + DataT ddRPhi; + localDistCorr.evalDistCorr(z0Tmp, radius, phi, ddZ, ddR, ddRPhi); + ddPhi = ddRPhi / radius; } /// dump the created electron tracks with calculateElectronDriftPath function to a tree @@ -1394,6 +1395,14 @@ class SpaceCharge void initContainer(DataContainer& data, const bool initMem = true); + /// initialize several containers concurrently (allocation and first access to the memory of large containers) + void initContainers(const std::vector& containers); + + /// rebin the space-charge density of a histogram (phi, r, z with z in [-zMax, zMax]) to the grid of both sides + /// \param getBinContentOrig function (iPhi, iR, iZ) returning the bin content of the original histogram (ROOT bin numbering) + template + void fillChargeDensityFromHisto(const TAxis& axisPhiOrig, const TAxis& axisROrig, const TAxis& axisZOrig, const BinContent& getBinContentOrig); + void initAllBuffers(); void setBoundaryFromIndices(const std::function& potentialFunc, const std::vector>& indices, const Side side); diff --git a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/SpaceChargeHelpers.h b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/SpaceChargeHelpers.h index 954bf90d2ebcf..d4e0f1c6cbd5f 100644 --- a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/SpaceChargeHelpers.h +++ b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/SpaceChargeHelpers.h @@ -107,6 +107,20 @@ class AnalyticalFields /// \return returns the function value for electric field Ephi for given coordinate DataT evalFieldPhi(DataT z, DataT r, DataT phi) const { return mEphiFunc(z, r, phi); } + /// evaluate all three components of the electric field at the same coordinate + /// \param z z coordinate + /// \param r r coordinate + /// \param phi phi coordinate + /// \param eZ field in z direction (output) + /// \param eR field in r direction (output) + /// \param ePhi field in phi direction (output) + void evalFields(DataT z, DataT r, DataT phi, DataT& eZ, DataT& eR, DataT& ePhi) const + { + eZ = evalFieldZ(z, r, phi); + eR = evalFieldR(z, r, phi); + ePhi = evalFieldPhi(z, r, phi); + } + /// \param r r coordinate /// \param phi phi coordinate /// \param z z coordinate @@ -274,6 +288,15 @@ class NumericalFields /// \return returns the function value for electric field Ephi for given coordinate DataT evalFieldPhi(DataT z, DataT r, DataT phi) const { return mInterpolatorEphi(z, r, phi); } + /// evaluate all three components of the electric field at the same coordinate (the position in the grid and the weights of the interpolation are calculated only once) + /// \param z z coordinate + /// \param r r coordinate + /// \param phi phi coordinate + /// \param eZ field in z direction (output) + /// \param eR field in r direction (output) + /// \param ePhi field in phi direction (output) + void evalFields(DataT z, DataT r, DataT phi, DataT& eZ, DataT& eR, DataT& ePhi) const { mInterpolatorEz.interpolate(z, r, phi, mInterpolatorEr, mInterpolatorEphi, eZ, eR, ePhi); } + o2::tpc::Side getSide() const { return mSide; } static constexpr unsigned int getID() { return ID; } @@ -330,6 +353,23 @@ class BField return getBPhi(rphiz); } + /// evaluate all three components of the B field at the same coordinate + /// \param z z coordinate + /// \param r r coordinate + /// \param phi phi coordinate + /// \param bZ field in z direction (output) + /// \param bR field in r direction (output) + /// \param bPhi field in phi direction (output) + /// templated on the value type so it can be used from SpaceCharge and SpaceCharge + template + void evalFields(T z, T r, T phi, T& bZ, T& bR, T& bPhi) const + { + const double rphiz[]{static_cast(r), static_cast(phi), static_cast(z)}; + bZ = static_cast(getBZ(rphiz)); + bR = static_cast(getBR(rphiz)); + bPhi = static_cast(getBPhi(rphiz)); + } + double getBR(const double rphiz[]) const { return (rphiz[2] >= 0) ? getBR_A(rphiz) : getBR_C(rphiz); } double getBPhi(const double rphiz[]) const { return (rphiz[2] >= 0) ? getBPhi_A(rphiz) : getBPhi_C(rphiz); } double getBZ(const double rphiz[]) const { return (rphiz[2] >= 0) ? getBZ_A(rphiz) : getBZ_C(rphiz); } @@ -408,6 +448,15 @@ class DistCorrInterpolator /// \return returns the function value for the local distortion or correction dRPhi for given coordinate DataT evaldRPhi(const DataT z, const DataT r, const DataT phi) const { return interpolatorDistCorrdRPhi(z, r, phi); } + /// evaluate the local distortions or corrections dZ, dR, dRPhi at the same coordinate (the position in the grid and the weights of the interpolation are calculated only once) + /// \param z z coordinate + /// \param r r coordinate + /// \param phi phi coordinate + /// \param dZ distortion or correction dZ (output) + /// \param dR distortion or correction dR (output) + /// \param dRPhi distortion or correction dRPhi (output) + void evalDistCorr(const DataT z, const DataT r, const DataT phi, DataT& dZ, DataT& dR, DataT& dRPhi) const { interpolatorDistCorrdZ.interpolate(z, r, phi, interpolatorDistCorrdR, interpolatorDistCorrdRPhi, dZ, dR, dRPhi); } + o2::tpc::Side getSide() const { return mSide; } static constexpr unsigned int getID() { return ID; } diff --git a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/TriCubic.h b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/TriCubic.h index a565091e24431..8d4a4f8605cf8 100644 --- a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/TriCubic.h +++ b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/TriCubic.h @@ -19,10 +19,6 @@ #include "TPCSpaceCharge/RegularGrid3D.h" -// forward declare VC Memory -template -class Memory; - namespace o2 { namespace tpc @@ -32,19 +28,27 @@ template class DataContainer3D; /// \class TriCubicInterpolator -/// The TriCubic class represents tricubic interpolation on a regular 3-Dim grid. -/// The algorithm which is used is based on the method developed by F. Lekien and J. Marsden and is described -/// in 'Tricubic Interpolation in Three Dimensions (2005)' http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.89.7835 -/// In this method in a first step 64 coefficients are computed by using a predefined 64*64 matrix. -/// These coefficients have to be computed for each cell in the grid, but are only computed when querying a point in a given cell. -/// The calculated coefficient is then stored for only the last cell and will be reused if the next query point lies in the same cell. -/// -/// Additionally the classical one dimensional approach of interpolating values is implemented. This algorithm is faster when interpolating only a few values inside each cube. -/// -/// periodic boundary conditions are used in phi direction. - -template -class Vector; +/// The TriCubic class represents tricubic interpolation on a regular 3-Dim grid (z, r, phi). +/// The interpolation is separable: the 4 x 4 x 4 grid points around the query point are weighted with the 1D cubic (Catmull-Rom) weights in each dimension. +/// Periodic boundary conditions are used in phi direction. Grid points outside of the grid in r and z are extrapolated (linear or parabolic) from the next +/// three points towards the inside of the grid. Query points outside of the grid are extrapolated or, with setExtrapolateValues(false), restricted to the grid. +/// Several data containers on the same grid can be interpolated at the same position with interpolate(), which calculates the position in the grid and +/// the weights only once (e.g. for the three components of the electric field). + +/// cell of the grid which is used for the tricubic interpolation of a query point and the weights of the interpolation (independent of the interpolated data) +template +struct TriCubicCell { + int indexZ{}; ///< index of the cell in z + int indexR{}; ///< index of the cell in r + int indexPhi{}; ///< index of the cell in phi + bool inGrid{}; ///< all 64 points used for the interpolation are on the grid (no extrapolation needed) + int dataIndex{}; ///< inGrid: index of the data at (iz - 1, ir - 1, iphi) + int sliceOffset[4]{}; ///< inGrid: offsets of the data for the 4 phi slices + int rowStride{}; ///< inGrid: offset of the data between the rows in r direction + DataT weightsZ[4]{}; ///< weights of the interpolation in z + DataT weightsR[4]{}; ///< weights of the interpolation in r + DataT weightsPhi[4]{}; ///< weights of the interpolation in phi +}; /// \tparam DataT the type of data which is used during the calculations template @@ -70,14 +74,26 @@ class TriCubicInterpolator Parabola = 1, ///< assume parabolic dependency at the boundaries of the grid }; - // interpolate value at given coordinate + /// interpolate value at given coordinate /// \param z z coordinate /// \param r r coordinate /// \param phi phi coordinate - /// \param type interpolation algorithm /// \return returns the interpolated value at given coordinate DataT operator()(const DataT z, const DataT r, const DataT phi) const { return interpolateSparse(z, r, phi); } + /// interpolate the data of this and of two other interpolators at the same coordinate (e.g. the three components of a vector field). + /// The position in the grid and the weights of the interpolation are calculated only once, which is faster than three separate interpolations. + /// The other interpolators have to use the same grid. The extrapolation settings of this interpolator are used for all three. + /// \param z z coordinate + /// \param r r coordinate + /// \param phi phi coordinate + /// \param other1 second interpolator + /// \param other2 third interpolator + /// \param val0 interpolated value of this interpolator (output) + /// \param val1 interpolated value of other1 (output) + /// \param val2 interpolated value of other2 (output) + void interpolate(const DataT z, const DataT r, const DataT phi, const TriCubicInterpolator& other1, const TriCubicInterpolator& other2, DataT& val0, DataT& val1, DataT& val2) const; + /// set which type of extrapolation is used at the grid boundaries (linear or parabol can be used with periodic phi axis and non periodic z and r axis). /// \param extrapolationType sets type of extrapolation. See enum ExtrapolationType for different types void setExtrapolationType(const ExtrapolationType extrapolationType) { mExtrapolationType = extrapolationType; } @@ -101,61 +117,18 @@ class TriCubicInterpolator ExtrapolationType mExtrapolationType = ExtrapolationType::Parabola; ///< sets which type of extrapolation for missing points at boundary is used bool mExtraPolateValues = true; ///< extrapolating values outside of the grid or restricting query to inside of the grid - // DEFINITION OF enum GridPos - //======================================================== - // r - // | 6------F---7 - // | / | / | - // | K G YR L H - // | / | / | - // | 2---B------3 | - // | | | | | - // | | 4---|---E--5 - // | C XL / D XR / - // | | I YL | J - // | | / | / - // | 0---A------1 - // |------------------------------- z - // / - // / - // / - // phi - //======================================================== - - enum class GridPos { - None = 27, - InnerVolume = 26, - Edge0 = 0, - Edge1 = 1, - Edge2 = 2, - Edge3 = 3, - Edge4 = 4, - Edge5 = 5, - Edge6 = 6, - Edge7 = 7, - LineA = 8, - LineB = 9, - LineC = 10, - LineD = 11, - LineE = 12, - LineF = 13, - LineG = 14, - LineH = 15, - LineI = 16, - LineJ = 17, - LineK = 18, - LineL = 19, - SideXRight = 20, - SideXLeft = 21, - SideYRight = 22, - SideYLeft = 23, - SideZRight = 24, - SideZLeft = 25 - }; + /// \return returns the cell and the weights of the interpolation for given coordinate + TriCubicCell getCell(const DataT z, const DataT r, const DataT phi) const; + + /// \return returns the interpolated value of the data for given cell (the data has to be defined on the grid of this interpolator) + DataT interpolateData(const DataContainer& data, const TriCubicCell& cell) const; - void setValues(const int iz, const int ir, const int iphi, std::array, 16>& cVals) const; + /// set the 64 values (4 x 4 x 4) used for the interpolation in the cell with given indices. Values outside of the grid in r and z are extrapolated + /// \param data data which is interpolated + /// \param cVals values of the points: cVals[4 * iphi + ir][iz] with iphi, ir, iz the relative index (-1, 0, 1, 2) + 1 to the cell + void setValues(const DataContainer& data, const int iz, const int ir, const int iphi, DataT (&cVals)[16][4]) const; - // interpolate value at given coordinate - this method doesnt compute and stores the coefficients and is faster when quering only a few values per cube + /// interpolate value at given coordinate /// \param z z coordinate /// \param r r coordinate /// \param phi phi coordinate @@ -173,20 +146,6 @@ class TriCubicInterpolator DataT linearExtrapolation(const DataT valk, const DataT valk1) const; DataT parabolExtrapolation(const DataT valk, const DataT valk1, const DataT valk2) const; - - GridPos findPos(const int iz, const int ir, const int iphi) const; - - bool isInInnerVolume(const int iz, const int ir, const int iphi, GridPos& posType) const; - - bool findEdge(const int iz, const int ir, const int iphi, GridPos& posType) const; - - bool findLine(const int iz, const int ir, const int iphi, GridPos& posType) const; - - bool findSide(const int iz, const int ir, const int iphi, GridPos& posType) const; - - bool isSideRight(const int ind, const int dim) const; - - bool isSideLeft(const int ind) const; }; } // namespace tpc diff --git a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/Vector.h b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/Vector.h deleted file mode 100644 index d72fbdd5dabda..0000000000000 --- a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/Vector.h +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// \file matrix.h -/// \brief Definition of Vector and Matrix class -/// -/// \author Matthias Kleiner - -#ifndef ALICEO2_TPC_VECTOR_H_ -#define ALICEO2_TPC_VECTOR_H_ - -#include -#include - -namespace o2 -{ -namespace tpc -{ - -template -class Vector -{ - using VDataT = Vc::Vector; - - public: - /// default constructor - Vector() = default; - - /// constructor - Vector(Vc::Memory&& dataVector) : mDataVector(std::move(dataVector)) {} - - /// operator access - const DataT operator[](size_t i) const { return mDataVector.scalar(i); } - DataT& operator[](size_t i) { return mDataVector.scalar(i); } - - /// sets the vector with index j - void setVector(const size_t j, const VDataT& vector) { mDataVector.vector(j) = vector; } - - /// \return returns the vector with index j - const VDataT getVector(const size_t j) const { return mDataVector.vector(j); } - - /// \return returns the number of Vc::Vector stored in the Vector - size_t getvectorsCount() const { return mDataVector.vectorsCount(); } - - private: - Vc::Memory mDataVector{}; ///< storage for the data -}; - -template -inline Vector operator*(const std::array, N>, N>& a, const Vector& b) -{ - using V = Vc::Vector; - // resulting vector c - Vector c; - for (size_t i = 0; i < N; ++i) { - V c_ij{}; - for (size_t j = 0; j < a[i].vectorsCount(); ++j) { - c_ij += a[i].vector(j) * b.getVector(j); - } - c[i] = c_ij.sum(); - } - return c; -} - -template -inline Vector floor_vec(const Vector& a) -{ - Vector c; - for (size_t j = 0; j < a.getvectorsCount(); ++j) { - c.setVector(j, Vc::floor(a.getVector(j))); - } - return c; -} - -template -inline Vector operator-(const Vector& a, const Vector& b) -{ - // resulting matrix c - Vector c; - for (size_t j = 0; j < a.getvectorsCount(); ++j) { - c.setVector(j, a.getVector(j) - b.getVector(j)); - } - return c; -} - -template -inline Vector operator*(const DataT a, const Vector& b) -{ - // resulting matrix c - Vector c; - for (size_t j = 0; j < b.getvectorsCount(); ++j) { - c.setVector(j, a * b.getVector(j)); - } - return c; -} - -// compute the sum of one Vector -template -inline DataT sum(const Vector& a) -{ - // resulting matrix c - Vc::Vector b = a.getVector(0); - for (size_t j = 1; j < a.getvectorsCount(); ++j) { - b += a.getVector(j); - } - return b.sum(); -} - -// multiply each row from a vector with the row from a second vector -template -inline Vector operator*(const Vector& a, const Vector& b) -{ - // resulting matrix c - Vector c{}; - for (size_t j = 0; j < a.getvectorsCount(); ++j) { - c.setVector(j, a.getVector(j) * b.getVector(j)); - } - return c; -} - -} // namespace tpc -} // namespace o2 - -#endif diff --git a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/Vector3D.h b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/Vector3D.h index 9338230130932..1d3be5ec648ac 100644 --- a/Detectors/TPC/spacecharge/include/TPCSpaceCharge/Vector3D.h +++ b/Detectors/TPC/spacecharge/include/TPCSpaceCharge/Vector3D.h @@ -18,6 +18,8 @@ #ifndef ALICEO2_TPC_VECTOR3D_H_ #define ALICEO2_TPC_VECTOR3D_H_ +#include + namespace o2 { namespace tpc @@ -30,11 +32,11 @@ template class Vector3D { public: - /// Constructor for a tricubic interpolator - /// \param nr number of data points in r directions - /// \param nz number of data points in r directions - /// \param nphi number of data points in r directions - Vector3D(const unsigned int nr, const unsigned int nz, const unsigned int nphi) : mNr{nr}, mNz{nz}, mNphi{nphi}, mStorage{nr * nz * nphi} {}; + /// constructor + /// \param nr number of data points in r direction + /// \param nz number of data points in z direction + /// \param nphi number of data points in phi direction + Vector3D(const unsigned int nr, const unsigned int nz, const unsigned int nphi) : mNr{nr}, mNz{nz}, mNphi{nphi}, mStorage(nr * nz * nphi) {}; /// default constructor Vector3D() = default; @@ -72,9 +74,9 @@ class Vector3D } /// resize the vector - /// \param nr number of data points in r directions - /// \param nz number of data points in r directions - /// \param nphi number of data points in r directions + /// \param nr number of data points in r direction + /// \param nz number of data points in z direction + /// \param nphi number of data points in phi direction void resize(const unsigned int nr, const unsigned int nz, const unsigned int nphi) { mNr = nr; @@ -86,10 +88,10 @@ class Vector3D const auto& data() const { return mStorage; } auto& data() { return mStorage; } - unsigned int getNr() const { return mNr; } ///< get number of data points in r direction - unsigned int getNz() const { return mNz; } ///< get number of data points in z direction - unsigned int getNphi() const { return mNphi; } ///< get number of data points in phi direction - unsigned int size() const { return mStorage.size; } ///< get number of data points + unsigned int getNr() const { return mNr; } ///< get number of data points in r direction + unsigned int getNz() const { return mNz; } ///< get number of data points in z direction + unsigned int getNphi() const { return mNphi; } ///< get number of data points in phi direction + unsigned int size() const { return mStorage.size(); } ///< get number of data points auto begin() const { return mStorage.begin(); } auto begin() { return mStorage.begin(); } diff --git a/Detectors/TPC/spacecharge/src/DataContainer3D.cxx b/Detectors/TPC/spacecharge/src/DataContainer3D.cxx index 77c8fd0691b3b..23453384aadd9 100644 --- a/Detectors/TPC/spacecharge/src/DataContainer3D.cxx +++ b/Detectors/TPC/spacecharge/src/DataContainer3D.cxx @@ -131,9 +131,26 @@ DataContainer3D& DataContainer3D::operator*=(const DataT value) return *this; } +namespace +{ +/// \return returns true if both containers have the same grid and the same number of stored values (otherwise the element-wise operations would access the data of the other container out of bounds) +template +bool isSameGrid(const DataContainer3D& a, const DataContainer3D& b, const char* operation) +{ + if ((a.getNZ() != b.getNZ()) || (a.getNR() != b.getNR()) || (a.getNPhi() != b.getNPhi()) || (a.getNDataPoints() != b.getNDataPoints())) { + LOGP(error, "DataContainer3D::operator{}: different grids or number of values ({}x{}x{} with {} values and {}x{}x{} with {} values). Data is not modified!", operation, a.getNZ(), a.getNR(), a.getNPhi(), a.getNDataPoints(), b.getNZ(), b.getNR(), b.getNPhi(), b.getNDataPoints()); + return false; + } + return true; +} +} // namespace + template DataContainer3D& DataContainer3D::operator+=(const DataContainer3D& other) { + if (!isSameGrid(*this, other, "+=")) { + return *this; + } std::transform(mData.begin(), mData.end(), other.mData.begin(), mData.begin(), std::plus<>()); return *this; } @@ -141,6 +158,9 @@ DataContainer3D& DataContainer3D::operator+=(const DataContainer3D template DataContainer3D& DataContainer3D::operator-=(const DataContainer3D& other) { + if (!isSameGrid(*this, other, "-=")) { + return *this; + } std::transform(mData.begin(), mData.end(), other.mData.begin(), mData.begin(), std::minus<>()); return *this; } @@ -148,6 +168,9 @@ DataContainer3D& DataContainer3D::operator-=(const DataContainer3D template DataContainer3D& DataContainer3D::operator*=(const DataContainer3D& other) { + if (!isSameGrid(*this, other, "*=")) { + return *this; + } std::transform(mData.begin(), mData.end(), other.mData.begin(), mData.begin(), std::multiplies<>()); return *this; } diff --git a/Detectors/TPC/spacecharge/src/PoissonSolver.cxx b/Detectors/TPC/spacecharge/src/PoissonSolver.cxx index d00e268f64125..3f210114036a2 100644 --- a/Detectors/TPC/spacecharge/src/PoissonSolver.cxx +++ b/Detectors/TPC/spacecharge/src/PoissonSolver.cxx @@ -22,7 +22,12 @@ #include "TPCSpaceCharge/PoissonSolverHelpers.h" #include "Framework/Logger.h" #include +#include +#include +#include #include +#include +#include #include "TPCSpaceCharge/Vector3D.h" #include "TPCSpaceCharge/DataContainer3D.h" #include "DataFormatsTPC/Defs.h" @@ -34,319 +39,51 @@ using namespace o2::tpc; template -void PoissonSolver::poissonSolver3D(DataContainer& matricesV, const DataContainer& matricesCharge, const int symmetry) +void PoissonSolver::poissonSolver3D(DataContainer& matricesV, const DataContainer& matricesCharge) { using timer = std::chrono::high_resolution_clock; auto start = timer::now(); - if (MGParameters::isFull3D) { - poissonMultiGrid3D(matricesV, matricesCharge, symmetry); - } else { - poissonMultiGrid3D2D(matricesV, matricesCharge, symmetry); - } + poissonMultiGrid3D(matricesV, matricesCharge); auto stop = timer::now(); std::chrono::duration time = stop - start; const float totalTime = time.count(); LOGP(detail, "poissonSolver3D took {}s", totalTime); } -template -void PoissonSolver::poissonSolver2D(DataContainer& matricesV, const DataContainer& matricesCharge) +namespace { - poissonMultiGrid2D(matricesV, matricesCharge); -} - -template -void PoissonSolver::poissonMultiGrid2D(DataContainer& matricesV, const DataContainer& matricesCharge, const int iPhi) +/// set (Add = false) or add (Add = true) the value to the target +template +inline void setOrAdd(DataT& target, const DataT value) { - /// Geometry of TPC -- should be use AliTPCParams instead - const DataT gridSpacingR = getSpacingR(); - const DataT gridSpacingZ = getSpacingZ(); - const DataT ratioZ = gridSpacingR * gridSpacingR / (gridSpacingZ * gridSpacingZ); // ratio_{Z} = gridSize_{r} / gridSize_{z} - - int nGridRow = 0; // number grid - int nGridCol = 0; // number grid - - int nnRow = mParamGrid.NRVertices; - while (nnRow >>= 1) { - ++nGridRow; - } - - int nnCol = mParamGrid.NZVertices; - while (nnCol >>= 1) { - ++nGridCol; - } - - // Check that number of mParamGrid.NRVertices and mParamGrid.NZVertices is suitable for multi grid - if (!isPowerOfTwo(mParamGrid.NRVertices - 1)) { - LOGP(error, "PoissonMultiGrid2D: PoissonMultiGrid - Error in the number of mParamGrid.NRVertices. Must be 2**M + 1"); - return; - } - if (!isPowerOfTwo(mParamGrid.NZVertices - 1)) { - LOGP(error, "PoissonMultiGrid2D: PoissonMultiGrid - Error in the number of mParamGrid.NZVertices. Must be 2**N + 1"); - return; - } - - const int nLoop = std::max(nGridRow, nGridCol); // Calculate the number of nLoop for the binary expansion - - LOGP(detail, "{}", fmt::format("PoissonMultiGrid2D: nGridRow={}, nGridCol={}, nLoop={}, nMGCycle={}", nGridRow, nGridCol, nLoop, MGParameters::nMGCycle)); - - unsigned int iOne = 1; // index - unsigned int jOne = 1; // index - int tnRRow = mParamGrid.NRVertices; - int tnZColumn = mParamGrid.NZVertices; - - // Vector for storing multi grid array - std::vector tvArrayV(nLoop); // potential <--> error - std::vector tvChargeFMG(nLoop); // charge is restricted in full multiGrid - std::vector tvCharge(nLoop); // charge <--> residue - std::vector tvResidue(nLoop); // residue calculation - - // Allocate memory for temporary grid - for (int count = 1; count <= nLoop; ++count) { - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - // if one just address to matrixV - const int index = count - 1; - tvResidue[index].resize(tnRRow, tnZColumn, 1); - tvChargeFMG[index].resize(tnRRow, tnZColumn, 1); - tvArrayV[index].resize(tnRRow, tnZColumn, 1); - tvCharge[index].resize(tnRRow, tnZColumn, 1); - - if (count == 1) { - for (int iphi = iPhi; iphi <= iPhi; ++iphi) { - for (int ir = 0; ir < mParamGrid.NRVertices; ++ir) { - for (int iz = 0; iz < mParamGrid.NZVertices; ++iz) { - tvChargeFMG[index](ir, iz, iphi) = matricesCharge(iz, ir, iphi); - tvCharge[index](ir, iz, iphi) = matricesCharge(iz, ir, iphi); - tvArrayV[index](ir, iz, iphi) = matricesV(iz, ir, iphi); - } - } - } - } else { - restrict2D(tvChargeFMG[index], tvChargeFMG[count - 2], tnRRow, tnZColumn, 0); - } - iOne *= 2; - jOne *= 2; - } - - /// full multi grid - if (MGParameters::cycleType == CycleType::FCycle) { - - LOGP(detail, "PoissonMultiGrid2D: Do full cycle"); - // FMG - // 1) Relax on the coarsest grid - iOne /= 2; - jOne /= 2; - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - const DataT h = gridSpacingR * nLoop; - const DataT h2 = h * h; - const DataT tempRatio = ratioZ * iOne * iOne / (jOne * jOne); - const DataT tempFourth = 1 / (2 + 2 * tempRatio); - - std::vector coefficient1(tnRRow); - std::vector coefficient2(tnRRow); - calcCoefficients2D(1, tnRRow - 1, h, coefficient1, coefficient2); - - relax2D(tvArrayV[nLoop - 1], tvChargeFMG[nLoop - 1], tnRRow, tnZColumn, h2, tempFourth, tempRatio, coefficient1, coefficient2); - - // Do VCycle from nLoop H to h - for (int count = nLoop - 2; count >= 0; --count) { - iOne /= 2; - jOne /= 2; - - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - - interp2D(tvArrayV[count], tvArrayV[count + 1], tnRRow, tnZColumn, iPhi); - - // Copy the relax charge to the tvCharge - tvCharge[count] = tvChargeFMG[count]; // copy - - // Do V cycle - for (int mgCycle = 0; mgCycle < MGParameters::nMGCycle; ++mgCycle) { - vCycle2D(count + 1, nLoop, MGParameters::nPre, MGParameters::nPost, gridSpacingR, ratioZ, tvArrayV, tvCharge, tvResidue); - } - } - } else if (MGParameters::cycleType == CycleType::VCycle) { - // 2. VCycle - LOGP(detail, "PoissonMultiGrid2D: Do V cycle"); - - int gridFrom = 1; - int gridTo = nLoop; - - // Do MGCycle - for (int mgCycle = 0; mgCycle < MGParameters::nMGCycle; ++mgCycle) { - vCycle2D(gridFrom, gridTo, MGParameters::nPre, MGParameters::nPost, gridSpacingR, ratioZ, tvArrayV, tvCharge, tvResidue); - } - } else if (MGParameters::cycleType == CycleType::WCycle) { - // 3. W Cycle (TODO:) - int gridFrom = 1; - int gridTo = nLoop; - // Do MGCycle - for (Int_t mgCycle = 0; mgCycle < MGParameters::nMGCycle; mgCycle++) { - wCycle2D(gridFrom, gridTo, MGParameters::gamma, MGParameters::nPre, MGParameters::nPost, gridSpacingR, ratioZ, tvArrayV, tvCharge, tvResidue); - } - } - - // fill output - for (int iphi = iPhi; iphi <= iPhi; ++iphi) { - for (int ir = 0; ir < mParamGrid.NRVertices; ++ir) { - for (int iz = 0; iz < mParamGrid.NZVertices; ++iz) { - matricesV(iz, ir, iphi) = tvArrayV[0](ir, iz, iphi); - } - } + if constexpr (Add) { + target += value; + } else { + target = value; } } +/// copy src to dst (same size) in parallel template -void PoissonSolver::poissonMultiGrid3D2D(DataContainer& matricesV, const DataContainer& matricesCharge, const int symmetry) +void copyParallel(o2::tpc::Vector3D& dst, const o2::tpc::Vector3D& src, const int nThreads, const long minPointsParallel) { - LOGP(detail, "{}", fmt::format("PoissonMultiGrid3D2D: in Poisson Solver 3D multiGrid semi coarsening mParamGrid.NRVertices={}, cols={}, mParamGrid.NPhiVertices={}", mParamGrid.NZVertices, mParamGrid.NRVertices, mParamGrid.NPhiVertices)); - - // Check that the number of mParamGrid.NRVertices and mParamGrid.NZVertices is suitable for a binary expansion - if (!isPowerOfTwo((mParamGrid.NRVertices - 1))) { - LOGP(error, "PoissonMultiGrid3D2D: Poisson3DMultiGrid - Error in the number of mParamGrid.NRVertices. Must be 2**M + 1"); + if ((dst.size() != src.size()) || (dst.getNr() != src.getNr()) || (dst.getNz() != src.getNz())) { + dst = src; return; } - if (!isPowerOfTwo((mParamGrid.NZVertices - 1))) { - LOGP(error, "PoissonMultiGrid3D2D: Poisson3DMultiGrid - Error in the number of mParamGrid.NZVertices. Must be 2**N + 1"); - return; - } - if (mParamGrid.NPhiVertices <= 3) { - LOGP(error, "PoissonMultiGrid3D2D: Poisson3DMultiGrid - Error in the number of mParamGrid.NPhiVertices. Must be larger than 3"); - return; - } - - const DataT gridSpacingR = getSpacingR(); - const DataT gridSpacingZ = getSpacingZ(); - const DataT gridSpacingPhi = getSpacingPhi(); - const DataT ratioPhi = gridSpacingR * gridSpacingR / (gridSpacingPhi * gridSpacingPhi); // ratio_{phi} = gridSize_{r} / gridSize_{phi} - const DataT ratioZ = gridSpacingR * gridSpacingR / (gridSpacingZ * gridSpacingZ); // ratio_{Z} = gridSize_{r} / gridSize_{z} - - // Solve Poisson's equation in cylindrical coordinates by multiGrid technique - // Allow for different size grid spacing in R and Z directions - int nGridRow = 0; // number grid - int nGridCol = 0; // number grid - int nnRow = mParamGrid.NRVertices; - int nnCol = mParamGrid.NZVertices; - - while (nnRow >>= 1) { - ++nGridRow; - } - while (nnCol >>= 1) { - ++nGridCol; - } - - const int maxVal = std::max(nGridRow, nGridCol); // Calculate the number of nLoop for the binary expansion - const size_t nLoop = (maxVal > MGParameters::maxLoop) ? MGParameters::maxLoop : maxVal; - unsigned int iOne = 1; // index i in gridSize r (original) - unsigned int jOne = 1; // index j in gridSize z (original) - - std::vector tvArrayV(nLoop); // potential <--> error - std::vector tvChargeFMG(nLoop); // charge is restricted in full multiGrid - std::vector tvCharge(nLoop); // charge <--> residue - std::vector tvPrevArrayV(nLoop); // error calculation - std::vector tvResidue(nLoop); // residue calculation - - for (unsigned int count = 1; count <= nLoop; count++) { - const unsigned int tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - const unsigned int tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - const unsigned int index = count - 1; - tvResidue[index].resize(tnRRow, tnZColumn, mParamGrid.NPhiVertices); - tvPrevArrayV[index].resize(tnRRow, tnZColumn, mParamGrid.NPhiVertices); - - // memory for the finest grid is from parameters - tvChargeFMG[index].resize(tnRRow, tnZColumn, mParamGrid.NPhiVertices); - tvArrayV[index].resize(tnRRow, tnZColumn, mParamGrid.NPhiVertices); - tvCharge[index].resize(tnRRow, tnZColumn, mParamGrid.NPhiVertices); - - if (count == 1) { - for (int iphi = 0; iphi < mParamGrid.NPhiVertices; ++iphi) { - for (int ir = 0; ir < mParamGrid.NRVertices; ++ir) { - for (int iz = 0; iz < mParamGrid.NZVertices; ++iz) { - tvChargeFMG[index](ir, iz, iphi) = matricesCharge(iz, ir, iphi); - tvArrayV[index](ir, iz, iphi) = matricesV(iz, ir, iphi); - } - } - } - } else { - restrict3D(tvChargeFMG[index], tvChargeFMG[count - 2], tnRRow, tnZColumn, mParamGrid.NPhiVertices, mParamGrid.NPhiVertices); - restrictBoundary3D(tvArrayV[index], tvArrayV[count - 2], tnRRow, tnZColumn, mParamGrid.NPhiVertices, mParamGrid.NPhiVertices); - } - iOne *= 2; // doubling - jOne *= 2; // doubling - } - - std::vector coefficient1(mParamGrid.NRVertices); // coefficient1(mParamGrid.NRVertices) for storing (1 + h_{r}/2r_{i}) from central differences in r direction - std::vector coefficient2(mParamGrid.NRVertices); // coefficient2(mParamGrid.NRVertices) for storing (1 + h_{r}/2r_{i}) from central differences in r direction - std::vector coefficient3(mParamGrid.NRVertices); // coefficient3(mParamGrid.NRVertices) for storing (1/r_{i}^2) from central differences in phi direction - std::vector coefficient4(mParamGrid.NRVertices); // coefficient4(mParamGrid.NRVertices) for storing 1/2 - std::vector inverseCoefficient4(mParamGrid.NRVertices); // inverse of coefficient4(mParamGrid.NRVertices) - - // Case full multi grid (FMG) - if (MGParameters::cycleType == CycleType::FCycle) { - // 1) Relax on the coarsest grid - iOne /= 2; - jOne /= 2; - int tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - int tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - - const DataT h = getSpacingR() * iOne; - const DataT h2 = h * h; - const DataT iOne2 = iOne * iOne; - const DataT tempRatioPhi = ratioPhi * iOne2; // Used tobe divided by ( m_one * m_one ) when m_one was != 1 - const DataT tempRatioZ = ratioZ * iOne2 / (jOne * jOne); - - calcCoefficients(1, tnRRow - 1, h, tempRatioZ, tempRatioPhi, coefficient1, coefficient2, coefficient3, coefficient4); - - // relax on the coarsest level - relax3D(tvArrayV[nLoop - 1], tvChargeFMG[nLoop - 1], tnRRow, tnZColumn, mParamGrid.NPhiVertices, symmetry, h2, tempRatioZ, coefficient1, coefficient2, coefficient3, coefficient4); - - // 2) Do multiGrid v-cycle from coarsest to finest - for (int count = nLoop - 2; count >= 0; --count) { - // move to finer grid - iOne /= 2; - jOne /= 2; - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - - // 2) a) Interpolate potential for h -> 2h (coarse -> fine) - interp3D(tvArrayV[count], tvArrayV[count + 1], tnRRow, tnZColumn, mParamGrid.NPhiVertices, mParamGrid.NPhiVertices); - - // 2) c) Copy the restricted charge to charge for calculation - tvCharge[count] = tvChargeFMG[count]; // copy - - // 2) c) Do V cycle MGParameters::nMGCycle times at most - for (int mgCycle = 0; mgCycle < MGParameters::nMGCycle; ++mgCycle) { - // Copy the potential to temp array for convergence calculation - tvPrevArrayV[count] = tvArrayV[count]; - - // 2) c) i) Call V cycle from grid count+1 (current fine level) to nLoop (coarsest) - vCycle3D2D(symmetry, count + 1, nLoop, MGParameters::nPre, MGParameters::nPost, ratioZ, ratioPhi, tvArrayV, tvCharge, tvResidue, coefficient1, coefficient2, coefficient3, coefficient4, inverseCoefficient4); - - const DataT convergenceError = getConvergenceError(tvArrayV[count], tvPrevArrayV[count]); - - /// if already converge just break move to finer grid - if (convergenceError <= sConvergenceError) { - break; - } - } - } - } - - // fill output - for (int iphi = 0; iphi < mParamGrid.NPhiVertices; ++iphi) { - for (int ir = 0; ir < mParamGrid.NRVertices; ++ir) { - for (int iz = 0; iz < mParamGrid.NZVertices; ++iz) { - matricesV(iz, ir, iphi) = tvArrayV[0](ir, iz, iphi); - } - } + const size_t n = src.size(); + const int nChunks = (nThreads > 1 && static_cast(n) > minPointsParallel) ? nThreads : 1; +#pragma omp parallel for num_threads(nThreads) if (nChunks > 1) + for (int chunk = 0; chunk < nChunks; ++chunk) { + const size_t begin = n * chunk / nChunks; + const size_t end = n * (chunk + 1) / nChunks; + std::copy(src.begin() + begin, src.begin() + end, dst.begin() + begin); } } +} // namespace template -void PoissonSolver::poissonMultiGrid3D(DataContainer& matricesV, const DataContainer& matricesCharge, const int symmetry) +void PoissonSolver::poissonMultiGrid3D(DataContainer& matricesV, const DataContainer& matricesCharge) { const DataT gridSpacingR = getSpacingR(); const DataT gridSpacingZ = getSpacingZ(); @@ -368,38 +105,22 @@ void PoissonSolver::poissonMultiGrid3D(DataContainer& matricesV, const Da return; } - // Solve Poisson's equation in cylindrical coordinates by multi grid technique - // Allow for different size grid spacing in R and Z directions - int nGridRow = 0; // number grid - int nGridCol = 0; // number grid - int nGridPhi = 0; - + // number of levels for the coarsening in r and z + int nGridRow = 0; int nnRow = mParamGrid.NRVertices; while (nnRow >>= 1) { ++nGridRow; } - + int nGridCol = 0; int nnCol = mParamGrid.NZVertices; while (nnCol >>= 1) { ++nGridCol; } + LOGP(detail, "{}", fmt::format("PoissonMultiGrid3D: nGridRow={}, nGridCol={}", nGridRow, nGridCol)); - int nnPhi = mParamGrid.NPhiVertices; - while (nnPhi % 2 == 0) { - ++nGridPhi; - nnPhi /= 2; - } - - LOGP(detail, "{}", fmt::format("PoissonMultiGrid3D: nGridRow={}, nGridCol={}, nGridPhi={}", nGridRow, nGridCol, nGridPhi)); - const int nLoop = std::max({nGridRow, nGridCol, nGridPhi}); // Calculate the number of nLoop for the binary expansion - - // Vector for storing multi grid array - unsigned int iOne = 1; // index i in gridSize r (original) - unsigned int jOne = 1; // index j in gridSize z (original) - unsigned int kOne = 1; // index k in gridSize phi - int tnRRow = mParamGrid.NRVertices; - int tnZColumn = mParamGrid.NZVertices; - int tPhiSlice = mParamGrid.NPhiVertices; + // grids of the multi grid levels + const std::vector levels = getMultiGridLevels(std::max(nGridRow, nGridCol)); + const int nLoop = levels.size(); // 1) Memory allocation for multi grid std::vector tvArrayV(nLoop); // potential <--> error @@ -414,168 +135,98 @@ void PoissonSolver::poissonMultiGrid3D(DataContainer& matricesV, const Da std::vector coefficient4(mParamGrid.NRVertices); // coefficient4(mParamGrid.NRVertices) for storing 1/2 std::vector inverseCoefficient4(mParamGrid.NRVertices); // inverse of coefficient4(mParamGrid.NRVertices) - for (int count = 1; count <= nLoop; ++count) { - // tnRRow,tnZColumn in new grid - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - tPhiSlice = kOne == 1 ? mParamGrid.NPhiVertices : mParamGrid.NPhiVertices / kOne; - tPhiSlice = tPhiSlice < nnPhi ? nnPhi : tPhiSlice; + // the buffers are allocated concurrently: for large grids the first access to the memory (page faults) by the single thread zeroing a buffer dominates + std::vector> buffers; + for (int index = 0; index < nLoop; ++index) { + for (Vector* buffer : {&tvResidue[index], &tvPrevArrayV[index], &tvChargeFMG[index], &tvArrayV[index], &tvCharge[index]}) { + buffers.emplace_back(buffer, &levels[index]); + } + } +#pragma omp parallel for num_threads(sNThreads) schedule(dynamic, 1) + for (size_t i = 0; i < buffers.size(); ++i) { + const MGLevel& level = *buffers[i].second; + buffers[i].first->resize(level.nr, level.nz, level.nphi); + } - // allocate memory for residue - const int index = count - 1; - tvResidue[index].resize(tnRRow, tnZColumn, tPhiSlice); - tvPrevArrayV[index].resize(tnRRow, tnZColumn, tPhiSlice); - tvChargeFMG[index].resize(tnRRow, tnZColumn, tPhiSlice); - tvArrayV[index].resize(tnRRow, tnZColumn, tPhiSlice); - tvCharge[index].resize(tnRRow, tnZColumn, tPhiSlice); - - // memory for the finest grid is from parameters - if (count == 1) { - for (int iphi = 0; iphi < mParamGrid.NPhiVertices; ++iphi) { - for (int ir = 0; ir < mParamGrid.NRVertices; ++ir) { - for (int iz = 0; iz < mParamGrid.NZVertices; ++iz) { - tvChargeFMG[index](ir, iz, iphi) = matricesCharge(iz, ir, iphi); - tvArrayV[index](ir, iz, iphi) = matricesV(iz, ir, iphi); - } - } + // memory for the finest grid is from parameters +#pragma omp parallel for num_threads(sNThreads) + for (int iphi = 0; iphi < mParamGrid.NPhiVertices; ++iphi) { + for (int ir = 0; ir < mParamGrid.NRVertices; ++ir) { + for (int iz = 0; iz < mParamGrid.NZVertices; ++iz) { + tvChargeFMG[0](ir, iz, iphi) = matricesCharge(iz, ir, iphi); + tvArrayV[0](ir, iz, iphi) = matricesV(iz, ir, iphi); } - tvCharge[index] = tvChargeFMG[index]; } - iOne *= 2; // doubling - jOne *= 2; // doubling - kOne *= 2; } + copyParallel(tvCharge[0], tvChargeFMG[0], sNThreads, MINPOINTSPARALLEL); - // Case full multi grid (FMG) - if (MGParameters::cycleType == CycleType::FCycle) { - // Restrict the charge to coarser grid - iOne = 2; - jOne = 2; - kOne = 2; - int otPhiSlice = mParamGrid.NPhiVertices; - - // 1) Restrict Charge and Boundary to coarser grid - for (int count = 2; count <= nLoop; ++count) { - // tnRRow,tnZColumn in new grid - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - tPhiSlice = kOne == 1 ? mParamGrid.NPhiVertices : mParamGrid.NPhiVertices / kOne; - tPhiSlice = tPhiSlice < nnPhi ? nnPhi : tPhiSlice; - - LOGP(detail, "{}", fmt::format("PoissonMultiGrid3D: Restrict3D, tnRRow={}, tnZColumn={}, newPhiSlice={}, oldPhiSlice={}", tnRRow, tnZColumn, tPhiSlice, otPhiSlice)); - restrict3D(tvChargeFMG[count - 1], tvChargeFMG[count - 2], tnRRow, tnZColumn, tPhiSlice, otPhiSlice); - // copy boundary values of V - restrictBoundary3D(tvArrayV[count - 1], tvArrayV[count - 2], tnRRow, tnZColumn, tPhiSlice, otPhiSlice); - otPhiSlice = tPhiSlice; - - iOne *= 2; // doubling - jOne *= 2; // doubling - kOne *= 2; - } + // full multi grid (FMG) + // 1) Restrict the charge and the boundary of the potential to the coarser grids + for (int index = 1; index < nLoop; ++index) { + LOGP(detail, "{}", fmt::format("PoissonMultiGrid3D: restriction to tnRRow={}, tnZColumn={}, newPhiSlice={}, oldPhiSlice={}", levels[index].nr, levels[index].nz, levels[index].nphi, levels[index - 1].nphi)); + restrictLevel(tvChargeFMG[index], tvChargeFMG[index - 1], levels[index], levels[index - 1]); + restrictBoundaryLevel(tvArrayV[index], tvArrayV[index - 1], levels[index], levels[index - 1]); + } - // Relax on the coarsest grid - // FMG - // 2) Relax on the coarsest grid - // move to the coarsest + 1 - iOne /= 2; - jOne /= 2; - kOne /= 2; - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - tPhiSlice = kOne == 1 ? mParamGrid.NPhiVertices : mParamGrid.NPhiVertices / kOne; - tPhiSlice = tPhiSlice < nnPhi ? nnPhi : tPhiSlice; - otPhiSlice = tPhiSlice; - - const DataT h = gridSpacingR * iOne; + // 2) Relax on the coarsest grid + { + const MGLevel& level = levels[nLoop - 1]; + const DataT h = gridSpacingR * level.rzFactor; const DataT h2 = h * h; - const DataT gridSizePhiInv = tPhiSlice * getGridSizePhiInv(); // h_{phi} + const DataT gridSizePhiInv = level.nphi * getGridSizePhiInv(); // h_{phi} const DataT tempRatioPhi = h2 * gridSizePhiInv * gridSizePhiInv; // ratio_{phi} = gridSize_{r} / gridSize_{phi} - const DataT tempRatioZ = ratioZ * iOne * iOne / (jOne * jOne); - - calcCoefficients(1, tnRRow - 1, h, tempRatioZ, tempRatioPhi, coefficient1, coefficient2, coefficient3, coefficient4); - - // 3) Relax on the coarsest grid - relax3D(tvArrayV[nLoop - 1], tvChargeFMG[nLoop - 1], tnRRow, tnZColumn, tPhiSlice, symmetry, h2, tempRatioZ, coefficient1, coefficient2, coefficient3, coefficient4); - - // 4) V Cycle from coarsest to finest - for (int count = nLoop - 2; count >= 0; --count) { - // move to finer grid - std::fill(std::begin(coefficient1), std::end(coefficient1), 0); - std::fill(std::begin(coefficient2), std::end(coefficient2), 0); - std::fill(std::begin(coefficient3), std::end(coefficient3), 0); - std::fill(std::begin(coefficient4), std::end(coefficient4), 0); - std::fill(std::begin(inverseCoefficient4), std::end(inverseCoefficient4), 0); - - iOne /= 2; - jOne /= 2; - kOne /= 2; - - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - tPhiSlice = kOne == 1 ? mParamGrid.NPhiVertices : mParamGrid.NPhiVertices / kOne; - tPhiSlice = tPhiSlice < nnPhi ? nnPhi : tPhiSlice; - - // 4) a) interpolate from 2h --> h grid - interp3D(tvArrayV[count], tvArrayV[count + 1], tnRRow, tnZColumn, tPhiSlice, otPhiSlice); - - // Copy the relax charge to the tvCharge - if (count > 0) { - tvCharge[count] = tvChargeFMG[count]; - } + calcCoefficients(1, level.nr - 1, h, ratioZ, tempRatioPhi, coefficient1, coefficient2, coefficient3, coefficient4); + relaxLevel(level, tvArrayV[nLoop - 1], tvChargeFMG[nLoop - 1], h2, ratioZ, coefficient1, coefficient2, coefficient3, coefficient4); + } - using timer = std::chrono::high_resolution_clock; - auto start = timer::now(); - for (int mgCycle = 0; mgCycle < MGParameters::nMGCycle; ++mgCycle) { - // copy to store previous potential - tvPrevArrayV[count] = tvArrayV[count]; + // 3) V Cycle from coarsest to finest + for (int count = nLoop - 2; count >= 0; --count) { + // move to finer grid + std::fill(std::begin(coefficient1), std::end(coefficient1), 0); + std::fill(std::begin(coefficient2), std::end(coefficient2), 0); + std::fill(std::begin(coefficient3), std::end(coefficient3), 0); + std::fill(std::begin(coefficient4), std::end(coefficient4), 0); + std::fill(std::begin(inverseCoefficient4), std::end(inverseCoefficient4), 0); - vCycle3D(symmetry, count + 1, nLoop, MGParameters::nPre, MGParameters::nPost, ratioZ, tvArrayV, tvCharge, tvResidue, coefficient1, coefficient2, coefficient3, coefficient4, inverseCoefficient4); + // a) interpolate from 2h --> h grid + interpolateLevel(tvArrayV[count], tvArrayV[count + 1], levels[count], levels[count + 1]); - // converge error - const DataT convergenceError = getConvergenceError(tvArrayV[count], tvPrevArrayV[count]); - // if already converge just break move to finer grid - if (convergenceError <= sConvergenceError) { - LOGP(detail, "Cycle converged. Continue to next cycle..."); - break; - } - if (count <= 1 && !(mgCycle % 10)) { - auto stop = timer::now(); - std::chrono::duration time = stop - start; - const float totalTime = time.count(); - const float timePerCycle = totalTime / (mgCycle + 1); - const float remaining = timePerCycle * (MGParameters::nMGCycle - mgCycle); - LOGP(detail, "Cycle {} out of {} for current V cycle {}. Processed time {}s with {}s per cycle. Max remaining time for current cycle {}s. Convergence {} > {}", mgCycle, MGParameters::nMGCycle, count, time.count(), timePerCycle, remaining, convergenceError, sConvergenceError); - } - if (mgCycle == (MGParameters::nMGCycle - 1)) { - LOGP(warning, "Cycle {} did not convergence! Current convergence error is larger than expected convergence error: {} > {}", mgCycle, convergenceError, sConvergenceError); - } - } - // keep old slice information - otPhiSlice = tPhiSlice; + // Copy the relax charge to the tvCharge + if (count > 0) { + copyParallel(tvCharge[count], tvChargeFMG[count], sNThreads, MINPOINTSPARALLEL); } - } else if (MGParameters::cycleType == CycleType::VCycle) { - // V-cycle - int gridFrom = 1; - int gridTo = nLoop; + using timer = std::chrono::high_resolution_clock; + auto start = timer::now(); for (int mgCycle = 0; mgCycle < MGParameters::nMGCycle; ++mgCycle) { // copy to store previous potential - tvPrevArrayV[0] = tvArrayV[0]; - - // Do V Cycle from the coarsest to finest grid - vCycle3D(symmetry, gridFrom, gridTo, MGParameters::nPre, MGParameters::nPost, ratioZ, tvArrayV, tvCharge, tvResidue, coefficient1, coefficient2, coefficient3, coefficient4, inverseCoefficient4); + copyParallel(tvPrevArrayV[count], tvArrayV[count], sNThreads, MINPOINTSPARALLEL); - // convergence error - const DataT convergenceError = getConvergenceError(tvArrayV[0], tvPrevArrayV[0]); + vCycle3D(count + 1, nLoop, MGParameters::nPre, MGParameters::nPost, ratioZ, levels, tvArrayV, tvCharge, tvResidue, coefficient1, coefficient2, coefficient3, coefficient4, inverseCoefficient4); - // if error already achieved then stop mg iteration + // converge error + const DataT convergenceError = getConvergenceError(tvArrayV[count], tvPrevArrayV[count]); + // if already converge just break move to finer grid if (convergenceError <= sConvergenceError) { + LOGP(detail, "Cycle converged after {} V cycles on level {}. Continue to next cycle...", mgCycle + 1, count); break; } + if (count <= 1 && !(mgCycle % 10)) { + auto stop = timer::now(); + std::chrono::duration time = stop - start; + const float totalTime = time.count(); + const float timePerCycle = totalTime / (mgCycle + 1); + const float remaining = timePerCycle * (MGParameters::nMGCycle - mgCycle); + LOGP(detail, "Cycle {} out of {} for current V cycle {}. Processed time {}s with {}s per cycle. Max remaining time for current cycle {}s. Convergence {} > {}", mgCycle, MGParameters::nMGCycle, count, time.count(), timePerCycle, remaining, convergenceError, sConvergenceError); + } + if (mgCycle == (MGParameters::nMGCycle - 1)) { + LOGP(warning, "Cycle {} on level {} did not convergence! Current convergence error is larger than expected convergence error: {} > {}", mgCycle, count, convergenceError, sConvergenceError); + } } } // fill output +#pragma omp parallel for num_threads(sNThreads) for (int iphi = 0; iphi < mParamGrid.NPhiVertices; ++iphi) { for (int ir = 0; ir < mParamGrid.NRVertices; ++ir) { for (int iz = 0; iz < mParamGrid.NZVertices; ++iz) { @@ -586,447 +237,152 @@ void PoissonSolver::poissonMultiGrid3D(DataContainer& matricesV, const Da } template -void PoissonSolver::wCycle2D(const int gridFrom, const int gridTo, const int gamma, const int nPre, const int nPost, const DataT gridSizeR, const DataT ratio, - std::vector& tvArrayV, std::vector& tvCharge, std::vector& tvResidue) -{ - unsigned int iOne = 1 << (gridFrom - 1); - unsigned int jOne = 1 << (gridFrom - 1); - - int tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - int tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - - std::vector coefficient1(mParamGrid.NRVertices); - std::vector coefficient2(mParamGrid.NZVertices); - - // 1) Go to coarsest level - for (int count = gridFrom; count <= gridTo - 2; ++count) { - const int index = count - 1; - const DataT h = gridSizeR * iOne; - const DataT h2 = h * h; - const DataT ih2 = 1 / h2; - const DataT tempRatio = ratio * iOne * iOne / (jOne * jOne); - const DataT tempFourth = 1 / (2 + 2 * tempRatio); - const DataT inverseTempFourth = 1 / tempFourth; - calcCoefficients2D(1, tnRRow - 1, h, coefficient1, coefficient2); - Vector matricesCurrentV = tvArrayV[index]; - Vector matricesCurrentCharge = tvCharge[index]; - Vector residue = tvResidue[index]; - - // 1) Pre-Smoothing: Gauss-Seidel Relaxation or Jacobi - for (int jPre = 1; jPre <= nPre; ++jPre) { - relax2D(matricesCurrentV, matricesCurrentCharge, tnRRow, tnZColumn, h2, tempFourth, tempRatio, coefficient1, coefficient2); - } - - // 2) Residue calculation - residue2D(residue, matricesCurrentV, matricesCurrentCharge, tnRRow, tnZColumn, ih2, inverseTempFourth, tempRatio, coefficient1, coefficient2); - - iOne *= 2; - jOne *= 2; - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - - matricesCurrentCharge = tvCharge[count]; - matricesCurrentV = tvArrayV[count]; - - // 3) Restriction - restrict2D(matricesCurrentCharge, residue, tnRRow, tnZColumn, 0); - } - - // Do V cycle from: gridTo-1 to gridTo gamma times - for (int iGamma = 0; iGamma < gamma; ++iGamma) { - vCycle2D(gridTo - 1, gridTo, nPre, nPost, gridSizeR, ratio, tvArrayV, tvCharge, tvResidue); - } - - // Go to finest grid - for (int count = gridTo - 2; count >= gridFrom; --count) { - iOne /= 2; - jOne /= 2; - - const DataT h = gridSizeR * iOne; - const DataT h2 = h * h; - const DataT tempRatio = ratio * iOne * iOne / (jOne * jOne); - const DataT tempFourth = 1 / (2 + 2 * tempRatio); - - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - const Vector matricesCurrentCharge = tvCharge[count - 1]; - Vector matricesCurrentV = tvArrayV[count - 1]; - const Vector matricesCurrentVC = tvArrayV[count]; - - // 6) Interpolation/Prolongation - addInterp2D(matricesCurrentV, matricesCurrentVC, tnRRow, tnZColumn, 0); - - calcCoefficients2D(1, tnRRow - 1, h, coefficient1, coefficient2); - - // 7) Post-Smoothing: Gauss-Seidel Relaxation - for (Int_t jPost = 1; jPost <= nPost; ++jPost) { - relax2D(matricesCurrentV, matricesCurrentCharge, tnRRow, tnZColumn, h2, tempFourth, tempRatio, coefficient1, coefficient2); - } // end post smoothing - } -} - -template -void PoissonSolver::vCycle2D(const int gridFrom, const int gridTo, const int nPre, const int nPost, const DataT gridSizeR, const DataT ratio, std::vector& tvArrayV, - std::vector& tvCharge, std::vector& tvResidue) -{ - unsigned int iOne = 1 << (gridFrom - 1); - unsigned int jOne = 1 << (gridFrom - 1); - - int tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - int tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - - std::vector coefficient1(mParamGrid.NRVertices); - std::vector coefficient2(mParamGrid.NZVertices); - - // 1) Go to coarsest level - for (int count = gridFrom; count <= gridTo - 1; ++count) { - const int index = count - 1; - const DataT h = gridSizeR * iOne; - const DataT h2 = h * h; - const DataT ih2 = 1 / h2; - const DataT tempRatio = ratio * iOne * iOne / (jOne * jOne); - const DataT tempFourth = 1 / (2 + 2 * tempRatio); - const DataT inverseTempFourth = 1 / tempFourth; - calcCoefficients2D(1, tnRRow - 1, h, coefficient1, coefficient2); - - for (int jPre = 1; jPre <= nPre; ++jPre) { - relax2D(tvArrayV[index], tvCharge[index], tnRRow, tnZColumn, h2, tempFourth, tempRatio, coefficient1, coefficient2); - } - - // 2) Residue calculation - residue2D(tvResidue[index], tvArrayV[index], tvCharge[index], tnRRow, tnZColumn, ih2, inverseTempFourth, tempRatio, coefficient1, coefficient2); - - iOne *= 2; - jOne *= 2; - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - - // 3) Restriction - restrict2D(tvCharge[count], tvResidue[index], tnRRow, tnZColumn, 0); - - // 4) Zeroing coarser V - std::fill(tvArrayV[count].begin(), tvArrayV[count].end(), 0); // is this necessary??? - } - - // 5) coarsest grid - const DataT h = gridSizeR * iOne; - const DataT h2 = h * h; - const DataT tempRatio = ratio * iOne * iOne / (jOne * jOne); - const DataT tempFourth = 1 / (2 + 2 * tempRatio); - calcCoefficients2D(1, tnRRow - 1, h, coefficient1, coefficient2); - - relax2D(tvArrayV[gridTo - 1], tvCharge[gridTo - 1], tnRRow, tnZColumn, h2, tempFourth, tempRatio, coefficient1, coefficient2); - - // Go to finest grid - for (int count = gridTo - 1; count >= gridFrom; count--) { - const int index = count - 1; - iOne /= 2; - jOne /= 2; - - const DataT hTmp = gridSizeR * iOne; - const DataT h2Tmp = hTmp * hTmp; - const DataT tempRatioTmp = ratio * iOne * iOne / (jOne * jOne); - const DataT tempFourthTmp = 1 / (2 + 2 * tempRatioTmp); - - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - - // 6) Interpolation/Prolongation - addInterp2D(tvArrayV[index], tvArrayV[count], tnRRow, tnZColumn, 0); - - calcCoefficients2D(1, tnRRow - 1, hTmp, coefficient1, coefficient2); - - // 7) Post-Smoothing: Gauss-Seidel Relaxation - for (int jPost = 1; jPost <= nPost; ++jPost) { - relax2D(tvArrayV[index], tvCharge[index], tnRRow, tnZColumn, h2Tmp, tempFourthTmp, tempRatioTmp, coefficient1, coefficient2); - } // end post smoothing - } -} - -template -void PoissonSolver::vCycle3D2D(const int symmetry, const int gridFrom, const int gridTo, const int nPre, const int nPost, const DataT ratioZ, const DataT ratioPhi, - std::vector& tvArrayV, std::vector& tvCharge, std::vector& tvResidue, std::vector& coefficient1, - std::vector& coefficient2, std::vector& coefficient3, std::vector& coefficient4, std::vector& inverseCoefficient4) const -{ - unsigned int iOne = 1 << (gridFrom - 1); - unsigned int jOne = 1 << (gridFrom - 1); - int tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - int tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - - for (int count = gridFrom; count <= gridTo - 1; ++count) { - const int index = count - 1; - const DataT h = getSpacingR() * iOne; - const DataT h2 = h * h; - const DataT ih2 = 1 / h2; - const int iOne2 = iOne * iOne; - const DataT tempRatioPhi = ratioPhi * iOne2; // Used tobe divided by ( m_one * m_one ) when m_one was != 1 - const DataT tempRatioZ = ratioZ * iOne2 / (jOne * jOne); - - calcCoefficients(1, tnRRow - 1, h, tempRatioZ, tempRatioPhi, coefficient1, coefficient2, coefficient3, coefficient4); - for (unsigned int i = 1; i < tnRRow - 1; ++i) { - inverseCoefficient4[i] = 1 / coefficient4[i]; - } - - // Info("VCycle3D2D","Before Pre-smoothing"); - // 1) Pre-Smoothing: Gauss-Seidel Relaxation or Jacobi - for (int jPre = 1; jPre <= nPre; ++jPre) { - relax3D(tvArrayV[index], tvCharge[index], tnRRow, tnZColumn, mParamGrid.NPhiVertices, symmetry, h2, tempRatioZ, coefficient1, coefficient2, coefficient3, coefficient4); - } // end pre smoothing - - // 2) Residue calculation - residue3D(tvResidue[index], tvArrayV[index], tvCharge[index], tnRRow, tnZColumn, mParamGrid.NPhiVertices, symmetry, ih2, tempRatioZ, coefficient1, coefficient2, coefficient3, inverseCoefficient4); - - iOne *= 2; - jOne *= 2; - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - - // 3) Restriction - restrict3D(tvCharge[count], tvResidue[index], tnRRow, tnZColumn, mParamGrid.NPhiVertices, mParamGrid.NPhiVertices); - - // 4) Zeroing coarser V - std::fill(tvArrayV[count].begin(), tvArrayV[count].end(), 0); - } - - // coarsest grid - const DataT hTmp = getSpacingR() * iOne; - const DataT h2Tmp = hTmp * hTmp; - - const int iOne2Tmp = iOne * iOne; - const DataT tempRatioPhiTmp = ratioPhi * iOne2Tmp; - const DataT tempRatioZTmp = ratioZ * iOne2Tmp / (jOne * jOne); - - calcCoefficients(1, tnRRow - 1, hTmp, tempRatioZTmp, tempRatioPhiTmp, coefficient1, coefficient2, coefficient3, coefficient4); - - // 3) Relax on the coarsest grid - relax3D(tvArrayV[gridTo - 1], tvCharge[gridTo - 1], tnRRow, tnZColumn, mParamGrid.NPhiVertices, symmetry, h2Tmp, tempRatioZTmp, coefficient1, coefficient2, coefficient3, coefficient4); - - // back to fine - for (int count = gridTo - 1; count >= gridFrom; --count) { - const int index = count - 1; - iOne /= 2; - jOne /= 2; - - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - - const DataT h = getSpacingR() * iOne; - const DataT h2 = h * h; - const int iOne2 = iOne * iOne; - const DataT tempRatioPhi = ratioPhi * iOne2; // Used tobe divided by ( m_one * m_one ) when m_one was != 1 - const DataT tempRatioZ = ratioZ * iOne2 / (jOne * jOne); - - // 4) Interpolation/Prolongation - addInterp3D(tvArrayV[index], tvArrayV[count], tnRRow, tnZColumn, mParamGrid.NPhiVertices, mParamGrid.NPhiVertices); - - calcCoefficients(1, tnRRow - 1, h, tempRatioZ, tempRatioPhi, coefficient1, coefficient2, coefficient3, coefficient4); - - // 5) Post-Smoothing: Gauss-Seidel Relaxation - for (int jPost = 1; jPost <= nPost; ++jPost) { - relax3D(tvArrayV[index], tvCharge[index], tnRRow, tnZColumn, mParamGrid.NPhiVertices, symmetry, h2, tempRatioZ, coefficient1, coefficient2, coefficient3, coefficient4); - } // end post smoothing - } -} - -template -void PoissonSolver::vCycle3D(const int symmetry, const int gridFrom, const int gridTo, const int nPre, const int nPost, const DataT ratioZ, std::vector& tvArrayV, +void PoissonSolver::vCycle3D(const int gridFrom, const int gridTo, const int nPre, const int nPost, const DataT ratioZ, const std::vector& levels, std::vector& tvArrayV, std::vector& tvCharge, std::vector& tvResidue, std::vector& coefficient1, std::vector& coefficient2, std::vector& coefficient3, std::vector& coefficient4, std::vector& inverseCoefficient4) const { const DataT gridSpacingR = getSpacingR(); - unsigned int iOne = 1 << (gridFrom - 1); - unsigned int jOne = 1 << (gridFrom - 1); - unsigned int kOne = 1 << (gridFrom - 1); - - int nnPhi = mParamGrid.NPhiVertices; - while (nnPhi % 2 == 0) { - nnPhi /= 2; - } - - int tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - int tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - int tPhiSlice = kOne == 1 ? mParamGrid.NPhiVertices : mParamGrid.NPhiVertices / kOne; - tPhiSlice = tPhiSlice < nnPhi ? nnPhi : tPhiSlice; + // set the coefficients for given level + const auto setCoefficients = [&](const MGLevel& level, DataT& h2) { + const DataT h = gridSpacingR * level.rzFactor; + h2 = h * h; + const DataT tempGridSizePhiInv = level.nphi * getGridSizePhiInv(); + const DataT tempRatioPhi = h2 * tempGridSizePhiInv * tempGridSizePhiInv; // ratio_{phi} = gridSize_{r} / gridSize_{phi} + calcCoefficients(1, level.nr - 1, h, ratioZ, tempRatioPhi, coefficient1, coefficient2, coefficient3, coefficient4); + }; for (int count = gridFrom; count <= gridTo - 1; ++count) { const int index = count - 1; - const int otPhiSlice = tPhiSlice; - const DataT h = gridSpacingR * iOne; - const DataT h2 = h * h; + const MGLevel& level = levels[index]; + DataT h2 = 0; + setCoefficients(level, h2); const DataT ih2 = 1 / h2; - const DataT tempGridSizePhiInv = tPhiSlice * getGridSizePhiInv(); // phi now is multiGrid - const DataT tempRatioPhi = h2 * tempGridSizePhiInv * tempGridSizePhiInv; // ratio_{phi} = gridSize_{r} / gridSize_{phi} - const DataT tempRatioZ = ratioZ * iOne * iOne / (jOne * jOne); - - calcCoefficients(1, tnRRow - 1, h, tempRatioZ, tempRatioPhi, coefficient1, coefficient2, coefficient3, coefficient4); - - for (int i = 1; i < tnRRow - 1; ++i) { + for (int i = 1; i < level.nr - 1; ++i) { inverseCoefficient4[i] = 1 / coefficient4[i]; } - // 1) Pre-Smoothing: Gauss-Seidel Relaxation or Jacobi + // 1) Pre-Smoothing: Gauss-Seidel Relaxation for (int jPre = 1; jPre <= nPre; ++jPre) { - relax3D(tvArrayV[index], tvCharge[index], tnRRow, tnZColumn, tPhiSlice, symmetry, h2, tempRatioZ, coefficient1, coefficient2, coefficient3, coefficient4); + relaxLevel(level, tvArrayV[index], tvCharge[index], h2, ratioZ, coefficient1, coefficient2, coefficient3, coefficient4); } // end pre smoothing // 2) Residue calculation - residue3D(tvResidue[index], tvArrayV[index], tvCharge[index], tnRRow, tnZColumn, tPhiSlice, symmetry, ih2, tempRatioZ, coefficient1, coefficient2, coefficient3, inverseCoefficient4); - - iOne *= 2; - jOne *= 2; - kOne *= 2; - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - tPhiSlice = mParamGrid.NPhiVertices / kOne; - tPhiSlice = tPhiSlice < nnPhi ? nnPhi : tPhiSlice; + residue3D(tvResidue[index], tvArrayV[index], tvCharge[index], level.nr, level.nz, level.nphi, ih2, ratioZ, coefficient1, coefficient2, coefficient3, inverseCoefficient4); // 3) Restriction - restrict3D(tvCharge[count], tvResidue[index], tnRRow, tnZColumn, tPhiSlice, otPhiSlice); + restrictLevel(tvCharge[count], tvResidue[index], levels[count], level); // 4) Zeroing coarser V std::fill(tvArrayV[count].begin(), tvArrayV[count].end(), 0); } // coarsest grid - const DataT hTmp = gridSpacingR * iOne; - const DataT h2Tmp = hTmp * hTmp; - const DataT tempGridSizePhiInvTmp = tPhiSlice * getGridSizePhiInv(); // phi now is multiGrid - const DataT tempRatioPhiTmp = h2Tmp * tempGridSizePhiInvTmp * tempGridSizePhiInvTmp; // ratio_{phi} = gridSize_{r} / gridSize_{phi} - const DataT tempRatioZTmp = ratioZ * iOne * iOne / (jOne * jOne); - - calcCoefficients(1, tnRRow - 1, hTmp, tempRatioZTmp, tempRatioPhiTmp, coefficient1, coefficient2, coefficient3, coefficient4); + { + const MGLevel& level = levels[gridTo - 1]; + DataT h2 = 0; + setCoefficients(level, h2); + relaxLevel(level, tvArrayV[gridTo - 1], tvCharge[gridTo - 1], h2, ratioZ, coefficient1, coefficient2, coefficient3, coefficient4); + } - // 3) Relax on the coarsest grid - relax3D(tvArrayV[gridTo - 1], tvCharge[gridTo - 1], tnRRow, tnZColumn, tPhiSlice, symmetry, h2Tmp, tempRatioZTmp, coefficient1, coefficient2, coefficient3, coefficient4); // back to fine for (int count = gridTo - 1; count >= gridFrom; --count) { const int index = count - 1; - const int otPhiSlice = tPhiSlice; - iOne /= 2; - jOne /= 2; - kOne /= 2; - - tnRRow = iOne == 1 ? mParamGrid.NRVertices : mParamGrid.NRVertices / iOne + 1; - tnZColumn = jOne == 1 ? mParamGrid.NZVertices : mParamGrid.NZVertices / jOne + 1; - tPhiSlice = kOne == 1 ? mParamGrid.NPhiVertices : mParamGrid.NPhiVertices / kOne; - tPhiSlice = tPhiSlice < nnPhi ? nnPhi : tPhiSlice; - - const DataT h = gridSpacingR * iOne; - const DataT h2 = h * h; - const DataT tempGridSizePhiInv = tPhiSlice * getGridSizePhiInv(); - const DataT tempRatioPhi = h2 * tempGridSizePhiInv * tempGridSizePhiInv; // ratio_{phi} = gridSize_{r} / gridSize_{phi} - const DataT tempRatioZ = ratioZ * iOne * iOne / (jOne * jOne); + const MGLevel& level = levels[index]; // 4) Interpolation/Prolongation - addInterp3D(tvArrayV[index], tvArrayV[count], tnRRow, tnZColumn, tPhiSlice, otPhiSlice); + interpolateLevel(tvArrayV[index], tvArrayV[count], level, levels[count]); - calcCoefficients(1, tnRRow - 1, h, tempRatioZ, tempRatioPhi, coefficient1, coefficient2, coefficient3, coefficient4); + DataT h2 = 0; + setCoefficients(level, h2); // 5) Post-Smoothing: Gauss-Seidel Relaxation for (int jPost = 1; jPost <= nPost; ++jPost) { - relax3D(tvArrayV[index], tvCharge[index], tnRRow, tnZColumn, tPhiSlice, symmetry, h2, tempRatioZ, coefficient1, coefficient2, coefficient3, coefficient4); + relaxLevel(level, tvArrayV[index], tvCharge[index], h2, ratioZ, coefficient1, coefficient2, coefficient3, coefficient4); } } } template -void PoissonSolver::residue2D(Vector& residue, const Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const DataT ih2, const DataT inverseTempFourth, - const DataT tempRatio, std::vector& coefficient1, std::vector& coefficient2) +void PoissonSolver::restrictLevel(Vector& coarse, const Vector& fine, const MGLevel& levelCoarse, const MGLevel& levelFine) const { - const int iPhi = 0; -#pragma omp parallel for num_threads(sNThreads) - for (int i = 1; i < tnRRow - 1; ++i) { - for (int j = 1; j < tnZColumn - 1; ++j) { - residue(i, j, iPhi) = ih2 * (coefficient1[i] * matricesCurrentV(i + 1, j, iPhi) + coefficient2[i] * matricesCurrentV(i - 1, j, iPhi) + tempRatio * (matricesCurrentV(i, j + 1, iPhi) + matricesCurrentV(i, j - 1, iPhi)) - inverseTempFourth * matricesCurrentV(i, j, iPhi)) + matricesCurrentCharge(i, j, iPhi); - } // end cols - } // end nRRow + restrict3D(coarse, fine, levelCoarse.nr, levelCoarse.nz, levelCoarse.nphi, levelFine.nphi); +} - // Boundary points. - for (int i = 0; i < tnRRow; ++i) { - residue(i, 0, iPhi) = residue(i, tnZColumn - 1, iPhi) = 0.0; - } +template +void PoissonSolver::restrictBoundaryLevel(Vector& coarse, const Vector& fine, const MGLevel& levelCoarse, const MGLevel& levelFine) const +{ + restrictBoundary3D(coarse, fine, levelCoarse.nr, levelCoarse.nz, levelCoarse.nphi, levelFine.nphi); +} - for (int j = 0; j < tnZColumn; ++j) { - residue(0, j, iPhi) = residue(tnRRow - 1, j, iPhi) = 0.0; - } +template +template +void PoissonSolver::interpolateLevel(Vector& fine, const Vector& coarse, const MGLevel& levelFine, const MGLevel& levelCoarse) const +{ + interpolate3D(fine, coarse, levelFine.nr, levelFine.nz, levelFine.nphi, levelCoarse.nphi); } template -void PoissonSolver::residue3D(Vector& residue, const Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const int tnPhi, const int symmetry, +void PoissonSolver::residue3D(Vector& residue, const Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const int tnPhi, const DataT ih2, const DataT tempRatioZ, const std::vector& coefficient1, const std::vector& coefficient2, const std::vector& coefficient3, const std::vector& inverseCoefficient4) const { -#pragma omp parallel for num_threads(sNThreads) // parallising this loop is possible - but using more than 2 cores makes it slower - + // strides of the storage: index = i + nr * (j + nz * m) + const int strideZ = matricesCurrentV.getNr(); + const int stridePhi = strideZ * matricesCurrentV.getNz(); + const DataT* pot = matricesCurrentV.data().data(); + const DataT* charge = matricesCurrentCharge.data().data(); + DataT* res = residue.data().data(); + const bool runParallel = (sNThreads > 1) && (static_cast(tnRRow) * tnZColumn * tnPhi > MINPOINTSPARALLEL); + +#pragma omp parallel for num_threads(sNThreads) if (runParallel) for (int m = 0; m < tnPhi; ++m) { - int mp1 = m + 1; - int signPlus = 1; - int mm1 = m - 1; - int signMinus = 1; - - // Reflection symmetry in phi (e.g. symmetry at sector boundaries, or half sectors, etc.) - if (symmetry == 1) { - if (mp1 > tnPhi - 1) { - mp1 = tnPhi - 2; - } - if (mm1 < 0) { - mm1 = 1; - } - } - // Anti-symmetry in phi - else if (symmetry == -1) { - if (mp1 > tnPhi - 1) { - mp1 = tnPhi - 2; - signPlus = -1; - } - if (mm1 < 0) { - mm1 = 1; - signMinus = -1; - } - } else { // No Symmetries in phi, no boundaries, the calculation is continuous across all phi - if (mp1 > tnPhi - 1) { - mp1 = m + 1 - tnPhi; - } - if (mm1 < 0) { - mm1 = m - 1 + tnPhi; - } - } + int mp1, mm1; + getPhiNeighbours(m, tnPhi, mp1, mm1); + const DataT* potM = pot + m * stridePhi; + const DataT* potP = pot + mp1 * stridePhi; + const DataT* potMm = pot + mm1 * stridePhi; + const DataT* chargeM = charge + m * stridePhi; + DataT* resM = res + m * stridePhi; for (int j = 1; j < tnZColumn - 1; ++j) { + const int row = j * strideZ; for (int i = 1; i < tnRRow - 1; ++i) { - residue(i, j, m) = ih2 * (coefficient2[i] * matricesCurrentV(i - 1, j, m) + tempRatioZ * (matricesCurrentV(i, j - 1, m) + matricesCurrentV(i, j + 1, m)) + coefficient1[i] * matricesCurrentV(i + 1, j, m) + - coefficient3[i] * (signPlus * matricesCurrentV(i, j, mp1) + signMinus * matricesCurrentV(i, j, mm1)) - inverseCoefficient4[i] * matricesCurrentV(i, j, m)) + - matricesCurrentCharge(i, j, m); + const int idx = row + i; + resM[idx] = ih2 * (coefficient2[i] * potM[idx - 1] + tempRatioZ * (potM[idx - strideZ] + potM[idx + strideZ]) + coefficient1[i] * potM[idx + 1] + + coefficient3[i] * (potP[idx] + potMm[idx]) - inverseCoefficient4[i] * potM[idx]) + + chargeM[idx]; } // end cols } // end mParamGrid.NRVertices } } template -void PoissonSolver::interp3D(Vector& matricesCurrentV, const Vector& matricesCurrentVC, const int tnRRow, const int tnZColumn, const int newPhiSlice, const int oldPhiSlice) const +void PoissonSolver::getPhiNeighbours(const int m, const int nPhi, int& mp1, int& mm1) +{ + // periodic boundary in phi + mp1 = (m + 1 > nPhi - 1) ? (m + 1 - nPhi) : (m + 1); + mm1 = (m - 1 < 0) ? (m - 1 + nPhi) : (m - 1); +} + +template +template +void PoissonSolver::interpolate3D(Vector& matricesCurrentV, const Vector& matricesCurrentVC, const int tnRRow, const int tnZColumn, const int newPhiSlice, const int oldPhiSlice) const { - // Do restrict 2 D for each slice if (newPhiSlice == 2 * oldPhiSlice) { - for (int m = 0; m < newPhiSlice; m += 2) { - // assuming no symmetry - int mm = m / 2; - int mmPlus = mm + 1; - int mp1 = m + 1; - - // round - if (mmPlus > oldPhiSlice - 1) { - mmPlus = mm + 1 - oldPhiSlice; - } - if (mp1 > newPhiSlice - 1) { - mp1 = m + 1 - newPhiSlice; - } + // interpolation also in phi: each iteration writes only to the fine slices m and m + 1 +#pragma omp parallel for num_threads(sNThreads) if ((sNThreads > 1) && (static_cast(tnRRow) * tnZColumn * newPhiSlice > MINPOINTSPARALLEL)) + for (int mm = 0; mm < oldPhiSlice; ++mm) { + const int m = 2 * mm; + const int mp1 = m + 1; // newPhiSlice is even: m + 1 < newPhiSlice + const int mmPlus = (mm + 1 > oldPhiSlice - 1) ? (mm + 1 - oldPhiSlice) : (mm + 1); // periodic boundary in phi for (int j = 2; j < tnZColumn - 1; j += 2) { for (int i = 2; i < tnRRow - 1; i += 2) { const int iHalf = i / 2; const int jHalf = j / 2; - matricesCurrentV(i, j, m) = matricesCurrentVC(iHalf, jHalf, mm); + setOrAdd(matricesCurrentV(i, j, m), matricesCurrentVC(iHalf, jHalf, mm)); // point on corner lines at phi direction - matricesCurrentV(i, j, mp1) = (matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf, mmPlus)) / 2; + setOrAdd(matricesCurrentV(i, j, mp1), (matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf, mmPlus)) / 2); } } @@ -1034,9 +390,9 @@ void PoissonSolver::interp3D(Vector& matricesCurrentV, const Vector& matr for (int i = 2; i < tnRRow - 1; i += 2) { const int iHalf = i / 2; const int jHalf = j / 2; - matricesCurrentV(i, j, m) = (matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf + 1, mm)) / 2; + setOrAdd(matricesCurrentV(i, j, m), (matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf + 1, mm)) / 2); // point on corner lines at phi direction - matricesCurrentV(i, j, mp1) = (matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf + 1, mm) + matricesCurrentVC(iHalf, jHalf, mmPlus) + matricesCurrentVC(iHalf, jHalf + 1, mmPlus)) / 4; + setOrAdd(matricesCurrentV(i, j, mp1), (matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf + 1, mm) + matricesCurrentVC(iHalf, jHalf, mmPlus) + matricesCurrentVC(iHalf, jHalf + 1, mmPlus)) / 4); } } @@ -1044,9 +400,9 @@ void PoissonSolver::interp3D(Vector& matricesCurrentV, const Vector& matr for (int i = 1; i < tnRRow - 1; i += 2) { const int iHalf = i / 2; const int jHalf = j / 2; - matricesCurrentV(i, j, m) = (matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf + 1, jHalf, mm)) / 2; + setOrAdd(matricesCurrentV(i, j, m), (matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf + 1, jHalf, mm)) / 2); // point on line at phi direction - matricesCurrentV(i, j, mp1) = ((matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf, mmPlus)) + (matricesCurrentVC(iHalf + 1, jHalf, mmPlus) + matricesCurrentVC(iHalf + 1, jHalf, mm))) / 4; + setOrAdd(matricesCurrentV(i, j, mp1), ((matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf, mmPlus)) + (matricesCurrentVC(iHalf + 1, jHalf, mmPlus) + matricesCurrentVC(iHalf + 1, jHalf, mm))) / 4); } } @@ -1054,30 +410,30 @@ void PoissonSolver::interp3D(Vector& matricesCurrentV, const Vector& matr for (int i = 1; i < tnRRow - 1; i += 2) { const int iHalf = i / 2; const int jHalf = j / 2; - matricesCurrentV(i, j, m) = ((matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf + 1, mm)) + (matricesCurrentVC(iHalf + 1, jHalf, mm) + matricesCurrentVC(iHalf + 1, jHalf + 1, mm))) / 4; + setOrAdd(matricesCurrentV(i, j, m), ((matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf + 1, mm)) + (matricesCurrentVC(iHalf + 1, jHalf, mm) + matricesCurrentVC(iHalf + 1, jHalf + 1, mm))) / 4); // point at the center at phi direction - matricesCurrentV(i, j, mp1) = ((matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf + 1, mm) + matricesCurrentVC(iHalf, jHalf, mmPlus) + matricesCurrentVC(iHalf, jHalf + 1, mmPlus)) + - (matricesCurrentVC(iHalf + 1, jHalf, mm) + matricesCurrentVC(iHalf + 1, jHalf + 1, mm) + matricesCurrentVC(iHalf + 1, jHalf, mmPlus) + matricesCurrentVC(iHalf + 1, jHalf + 1, mmPlus))) / - 8; + setOrAdd(matricesCurrentV(i, j, mp1), ((matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf + 1, mm) + matricesCurrentVC(iHalf, jHalf, mmPlus) + matricesCurrentVC(iHalf, jHalf + 1, mmPlus)) + + (matricesCurrentVC(iHalf + 1, jHalf, mm) + matricesCurrentVC(iHalf + 1, jHalf + 1, mm) + matricesCurrentVC(iHalf + 1, jHalf, mmPlus) + matricesCurrentVC(iHalf + 1, jHalf + 1, mmPlus))) / + 8); } } } - } else { -#pragma omp parallel for num_threads(sNThreads) // no change + // same number of phi slices: interpolation in r and z for each phi slice +#pragma omp parallel for num_threads(sNThreads) if ((sNThreads > 1) && (static_cast(tnRRow) * tnZColumn * newPhiSlice > MINPOINTSPARALLEL)) for (int m = 0; m < newPhiSlice; ++m) { - interp2D(matricesCurrentV, matricesCurrentVC, tnRRow, tnZColumn, m); + interpolate2D(matricesCurrentV, matricesCurrentVC, tnRRow, tnZColumn, m); } } } template -void PoissonSolver::interp2D(Vector& matricesCurrentV, const Vector& matricesCurrentVC, const int tnRRow, const int tnZColumn, const int iphi) const +template +void PoissonSolver::interpolate2D(Vector& matricesCurrentV, const Vector& matricesCurrentVC, const int tnRRow, const int tnZColumn, const int iphi) const { for (int j = 2; j < tnZColumn - 1; j += 2) { for (int i = 2; i < tnRRow - 1; i += 2) { - const int jHalf = j / 2; - matricesCurrentV(i, j, iphi) = matricesCurrentVC(i / 2, jHalf, iphi); + setOrAdd(matricesCurrentV(i, j, iphi), matricesCurrentVC(i / 2, j / 2, iphi)); } } @@ -1085,7 +441,7 @@ void PoissonSolver::interp2D(Vector& matricesCurrentV, const Vector& matr for (int i = 2; i < tnRRow - 1; i += 2) { const int iHalf = i / 2; const int jHalf = j / 2; - matricesCurrentV(i, j, iphi) = (matricesCurrentVC(iHalf, jHalf, iphi) + matricesCurrentVC(iHalf, jHalf + 1, iphi)) / 2; + setOrAdd(matricesCurrentV(i, j, iphi), (matricesCurrentVC(iHalf, jHalf, iphi) + matricesCurrentVC(iHalf, jHalf + 1, iphi)) / 2); } } @@ -1093,314 +449,229 @@ void PoissonSolver::interp2D(Vector& matricesCurrentV, const Vector& matr for (int i = 1; i < tnRRow - 1; i += 2) { const int iHalf = i / 2; const int jHalf = j / 2; - matricesCurrentV(i, j, iphi) = (matricesCurrentVC(iHalf, jHalf, iphi) + matricesCurrentVC(iHalf + 1, jHalf, iphi)) / 2; + setOrAdd(matricesCurrentV(i, j, iphi), (matricesCurrentVC(iHalf, jHalf, iphi) + matricesCurrentVC(iHalf + 1, jHalf, iphi)) / 2); } } - // only if full - if (MGParameters::gtType == GridTransferType::Full) { - for (int j = 1; j < tnZColumn - 1; j += 2) { - for (int i = 1; i < tnRRow - 1; i += 2) { - const int iHalf = i / 2; - const int jHalf = j / 2; - matricesCurrentV(i, j, iphi) = (matricesCurrentVC(iHalf, jHalf, iphi) + matricesCurrentVC(iHalf, jHalf + 1, iphi) + matricesCurrentVC(iHalf + 1, jHalf, iphi) + matricesCurrentVC(iHalf + 1, jHalf + 1, iphi)) / 4; - } + for (int j = 1; j < tnZColumn - 1; j += 2) { + for (int i = 1; i < tnRRow - 1; i += 2) { + const int iHalf = i / 2; + const int jHalf = j / 2; + setOrAdd(matricesCurrentV(i, j, iphi), (matricesCurrentVC(iHalf, jHalf, iphi) + matricesCurrentVC(iHalf, jHalf + 1, iphi) + matricesCurrentVC(iHalf + 1, jHalf, iphi) + matricesCurrentVC(iHalf + 1, jHalf + 1, iphi)) / 4); } } } template -void PoissonSolver::addInterp3D(Vector& matricesCurrentV, const Vector& matricesCurrentVC, const int tnRRow, const int tnZColumn, const int newPhiSlice, const int oldPhiSlice) const +void PoissonSolver::relax3D(Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const int iPhi, const DataT h2, + const DataT tempRatioZ, const std::vector& coefficient1, const std::vector& coefficient2, const std::vector& coefficient3, const std::vector& coefficient4) const { - // Do restrict 2 D for each slice - if (newPhiSlice == 2 * oldPhiSlice) { - for (int m = 0; m < newPhiSlice; m += 2) { - // assuming no symmetry - int mm = m / 2; - int mmPlus = mm + 1; - int mp1 = m + 1; - - // round - if (mmPlus > (oldPhiSlice)-1) { - mmPlus = mm + 1 - (oldPhiSlice); - } - if (mp1 > (newPhiSlice)-1) { - mp1 = m + 1 - (newPhiSlice); - } - - for (int j = 2; j < tnZColumn - 1; j += 2) { - for (int i = 2; i < tnRRow - 1; i += 2) { - const int iHalf = i / 2; - const int jHalf = j / 2; - matricesCurrentV(i, j, m) += matricesCurrentVC(iHalf, jHalf, mm); - // point on corner lines at phi direction - matricesCurrentV(i, j, mp1) += (matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf, mmPlus)) / 2; - } - } - - for (int j = 1; j < tnZColumn - 1; j += 2) { - for (int i = 2; i < tnRRow - 1; i += 2) { - const int iHalf = i / 2; - const int jHalf = j / 2; - matricesCurrentV(i, j, m) += (matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf + 1, mm)) / 2; - // point on corner lines at phi direction - matricesCurrentV(i, j, mp1) += (matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf + 1, mm) + matricesCurrentVC(iHalf, jHalf, mmPlus) + matricesCurrentVC(iHalf, jHalf + 1, mmPlus)) / 4; - } - } - - for (int j = 2; j < tnZColumn - 1; j += 2) { - for (int i = 1; i < tnRRow - 1; i += 2) { - const int iHalf = i / 2; - const int jHalf = j / 2; - matricesCurrentV(i, j, m) += (matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf + 1, jHalf, mm)) / 2; - // point on line at phi direction - matricesCurrentV(i, j, mp1) += ((matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf, mmPlus)) + (matricesCurrentVC(iHalf + 1, jHalf, mmPlus) + matricesCurrentVC(iHalf + 1, jHalf, mm))) / 4; - } - } - - for (int j = 1; j < tnZColumn - 1; j += 2) { - for (int i = 1; i < tnRRow - 1; i += 2) { - const int iHalf = i / 2; - const int jHalf = j / 2; - matricesCurrentV(i, j, m) += ((matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf + 1, mm)) + (matricesCurrentVC(iHalf + 1, jHalf, mm) + matricesCurrentVC(iHalf + 1, jHalf + 1, mm))) / 4; - // point at the center at phi direction - matricesCurrentV(i, j, mp1) += ((matricesCurrentVC(iHalf, jHalf, mm) + matricesCurrentVC(iHalf, jHalf + 1, mm) + matricesCurrentVC(iHalf, jHalf, mmPlus) + matricesCurrentVC(iHalf, jHalf + 1, mmPlus)) + - (matricesCurrentVC(iHalf + 1, jHalf, mm) + matricesCurrentVC(iHalf + 1, jHalf + 1, mm) + matricesCurrentVC(iHalf + 1, jHalf, mmPlus) + matricesCurrentVC(iHalf + 1, jHalf + 1, mmPlus))) / - 8; - } - } - } - - } else { -#pragma omp parallel for num_threads(sNThreads) // no change - for (int m = 0; m < newPhiSlice; m++) { - addInterp2D(matricesCurrentV, matricesCurrentVC, tnRRow, tnZColumn, m); - } - } + // Gauss-Seidel (Red Black) + // strides of the storage: index = i + nr * (j + nz * m) + const int nr = matricesCurrentV.getNr(); + const int strideZ = nr; + const int stridePhi = nr * matricesCurrentV.getNz(); + const DataT* charge = matricesCurrentCharge.data().data(); + DataT* pot = matricesCurrentV.data().data(); + + // relax all points of one colour in phi slice m + auto relaxSlice = [&](const int msw, const int m) { + const int jsw = ((msw + m) % 2) ? 1 : 2; + int mp1, mm1; + getPhiNeighbours(m, iPhi, mp1, mm1); + DataT* potM = pot + m * stridePhi; + const DataT* potP = pot + mp1 * stridePhi; + const DataT* potMm = pot + mm1 * stridePhi; + const DataT* chargeM = charge + m * stridePhi; + int isw = jsw; + for (int j = 1; j < tnZColumn - 1; ++j, isw = 3 - isw) { + const int row = j * strideZ; + for (int i = isw; i < tnRRow - 1; i += 2) { + const int idx = row + i; + potM[idx] = (coefficient2[i] * potM[idx - 1] + tempRatioZ * (potM[idx - strideZ] + potM[idx + strideZ]) + coefficient1[i] * potM[idx + 1] + coefficient3[i] * (potP[idx] + potMm[idx]) + (h2 * chargeM[idx])) * coefficient4[i]; + } // end cols + } // end mParamGrid.NRVertices + }; + + // In one pass only the points with one parity of (i + j + m) are updated. All direct neighbours of these points have the other parity, + // which makes the updates within one pass independent of each other and the phi slices can be processed in parallel. + // The only exception is the periodic case with an odd number of phi slices: the first and the last slice are neighbours with the same parity. + // To get exactly the same result as with the sequential loop (m = 0, 1, ..., iPhi - 1), the last slice is processed after all other slices. + const bool runParallel = (sNThreads > 1) && (static_cast(tnRRow) * tnZColumn * iPhi > MINPOINTSPARALLEL); + for (int iPass = 1; iPass <= 2; ++iPass) { + const int msw = (iPass % 2) ? 1 : 2; +#pragma omp parallel for num_threads(sNThreads) if (runParallel) + for (int m = 0; m < iPhi - 1; ++m) { + relaxSlice(msw, m); + } + relaxSlice(msw, iPhi - 1); + } // end sweep } template -void PoissonSolver::addInterp2D(Vector& matricesCurrentV, const Vector& matricesCurrentVC, const int tnRRow, const int tnZColumn, const int tnPhi) const +void PoissonSolver::relaxLevel(const MGLevel& level, Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const DataT h2, const DataT tempRatioZ, + const std::vector& coefficient1, const std::vector& coefficient2, const std::vector& coefficient3, const std::vector& coefficient4) const { - for (int j = 2; j < tnZColumn - 1; j += 2) { - for (int i = 2; i < tnRRow - 1; i += 2) { - matricesCurrentV(i, j, tnPhi) = matricesCurrentV(i, j, tnPhi) + matricesCurrentVC(i / 2, j / 2, tnPhi); - } - } - - for (int j = 1; j < tnZColumn - 1; j += 2) { - for (int i = 2; i < tnRRow - 1; i += 2) { - const int iHalf = i / 2; - const int jHalf = j / 2; - matricesCurrentV(i, j, tnPhi) = matricesCurrentV(i, j, tnPhi) + (matricesCurrentVC(iHalf, jHalf, tnPhi) + matricesCurrentVC(iHalf, jHalf + 1, tnPhi)) / 2; - } - } - - for (int j = 2; j < tnZColumn - 1; j += 2) { - for (int i = 1; i < tnRRow - 1; i += 2) { - const int iHalf = i / 2; - const int jHalf = j / 2; - matricesCurrentV(i, j, tnPhi) = matricesCurrentV(i, j, tnPhi) + (matricesCurrentVC(iHalf, jHalf, tnPhi) + matricesCurrentVC(iHalf + 1, jHalf, tnPhi)) / 2; - } - } - - // only if full - if (MGParameters::gtType == GridTransferType::Full) { - for (int j = 1; j < tnZColumn - 1; j += 2) { - for (int i = 1; i < tnRRow - 1; i += 2) { - const int iHalf = i / 2; - const int jHalf = j / 2; - matricesCurrentV(i, j, tnPhi) = matricesCurrentV(i, j, tnPhi) + (matricesCurrentVC(iHalf, jHalf, tnPhi) + matricesCurrentVC(iHalf, jHalf + 1, tnPhi) + matricesCurrentVC(iHalf + 1, jHalf, tnPhi) + matricesCurrentVC(iHalf + 1, jHalf + 1, tnPhi)) / 4; - } - } + if (level.lineRelaxPhi) { + relaxLinePhi(matricesCurrentV, matricesCurrentCharge, level.nr, level.nz, level.nphi, h2, tempRatioZ, coefficient1, coefficient2, coefficient3, coefficient4); + } else { + relax3D(matricesCurrentV, matricesCurrentCharge, level.nr, level.nz, level.nphi, h2, tempRatioZ, coefficient1, coefficient2, coefficient3, coefficient4); } } template -void PoissonSolver::relax3D(Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const int iPhi, const int symmetry, const DataT h2, - const DataT tempRatioZ, const std::vector& coefficient1, const std::vector& coefficient2, const std::vector& coefficient3, const std::vector& coefficient4) const +void PoissonSolver::relaxLinePhi(Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const int iPhi, const DataT h2, const DataT tempRatioZ, + const std::vector& coefficient1, const std::vector& coefficient2, const std::vector& coefficient3, const std::vector& coefficient4) const { - // Gauss-Seidel (Read Black} - if (MGParameters::relaxType == RelaxType::GaussSeidel) { - // for each slice - for (int iPass = 1; iPass <= 2; ++iPass) { - const int msw = (iPass % 2) ? 1 : 2; - for (int m = 0; m < iPhi; ++m) { - const int jsw = ((msw + m) % 2) ? 1 : 2; - int mp1 = m + 1; - int signPlus = 1; - int mm1 = m - 1; - int signMinus = 1; - // Reflection symmetry in phi (e.g. symmetry at sector boundaries, or half sectors, etc.) - if (symmetry == 1) { - if (mp1 > iPhi - 1) { - mp1 = iPhi - 2; - } - if (mm1 < 0) { - mm1 = 1; + // For each line (i, j) the equations a_i * V_m - b_i * (V_{m-1} + V_{m+1}) = rhs_m with periodic boundary in phi are solved, with + // a_i = 1 / coefficient4[i] = 2 * (1 + tempRatioZ + coefficient3[i]), b_i = coefficient3[i] and rhs_m containing the neighbours in r and z and the charge. + // The periodic tridiagonal system is solved with the Thomas algorithm and the Sherman-Morrison formula (see Numerical Recipes, cyclic tridiagonal systems). + // The matrix depends only on i: the factorisation is calculated once for all lines. Storage of the factorisation: index = i + nr * m + const int nr = matricesCurrentV.getNr(); + const int strideZ = nr; + const int stridePhi = nr * matricesCurrentV.getNz(); + const int n = iPhi; + std::vector cPrime(static_cast(nr) * n); // modified super diagonal of the Thomas algorithm + std::vector invDen(static_cast(nr) * n); // inverse of the modified diagonal of the Thomas algorithm + std::vector zSM(static_cast(nr) * n); // solution of T z = u for the Sherman-Morrison correction + std::vector factDenInv(nr), bOverA(nr), bCoeff(nr); // 1 / (1 + z_0 + b/a * z_{n-1}), b/a, b + const bool runParallel = (sNThreads > 1) && (static_cast(tnRRow) * tnZColumn * iPhi > MINPOINTSPARALLEL); + + // the factorisations for different i are independent +#pragma omp parallel for num_threads(sNThreads) if (runParallel) + for (int i = 1; i < tnRRow - 1; ++i) { + const DataT a = 1 / coefficient4[i]; + const DataT b = coefficient3[i]; + const DataT gamma = -a; + bCoeff[i] = b; + bOverA[i] = b / a; + // modified diagonal: corners -b of the periodic matrix are moved to the rank one correction u * v^T with u = (gamma, 0, ..., 0, -b), v = (1, 0, ..., 0, -b / gamma) + const auto diag = [&](const int m) { return (m == 0) ? (a - gamma) : ((m == n - 1) ? (a - b * b / gamma) : a); }; + DataT den = diag(0); + invDen[i] = 1 / den; + cPrime[i] = -b / den; + for (int m = 1; m < n; ++m) { + den = diag(m) + b * cPrime[i + nr * (m - 1)]; + invDen[i + nr * m] = 1 / den; + cPrime[i + nr * m] = -b / den; + } + // solve T z = u + DataT d = gamma * invDen[i]; + zSM[i] = d; + for (int m = 1; m < n; ++m) { + const DataT u = (m == n - 1) ? -b : 0; + d = (u + b * d) * invDen[i + nr * m]; + zSM[i + nr * m] = d; + } + for (int m = n - 2; m >= 0; --m) { + zSM[i + nr * m] -= cPrime[i + nr * m] * zSM[i + nr * (m + 1)]; + } + factDenInv[i] = 1 / (1 + zSM[i] + bOverA[i] * zSM[i + nr * (n - 1)]); + } + + const DataT* charge = matricesCurrentCharge.data().data(); + DataT* pot = matricesCurrentV.data().data(); + + // the lines are distributed to the threads in blocks of consecutive i of one row j. Several blocks per row are used to get enough tasks for a good load balance + const int nRows = tnZColumn - 2; + const int nInner = tnRRow - 2; // number of inner vertices in r + int nBlocks = 1; + if (runParallel) { + const int tasksPerThread = 16; + const int minLinesPerBlock = 2; + nBlocks = std::clamp((tasksPerThread * sNThreads + nRows - 1) / nRows, 1, std::max(1, nInner / (2 * minLinesPerBlock))); + } + const int blockSize = 2 * ((nInner + 2 * nBlocks - 1) / (2 * nBlocks)); // even number of i per block + const int nTasks = nRows * nBlocks; + +#pragma omp parallel num_threads(sNThreads) if (runParallel) + { + std::vector work(static_cast(blockSize) * n); // forward elimination and solution for the lines of one block: index = (i - iLow) + blockSize * m + + // the lines with even and odd (i + j) are processed one after the other: all neighbours in r and z of a line have the other parity, the lines of one parity are independent + for (int parity = 0; parity < 2; ++parity) { +#pragma omp for schedule(static) + for (int task = 0; task < nTasks; ++task) { + const int j = 1 + task / nBlocks; + const int iLow = 1 + (task % nBlocks) * blockSize; + const int iEnd = std::min(iLow + blockSize, tnRRow - 1); + const int iStart = ((iLow + j) % 2 == parity) ? iLow : iLow + 1; // lines with (i + j) % 2 == parity + const int row = j * strideZ; + DataT* workBlock = work.data(); + // forward elimination + for (int m = 0; m < n; ++m) { + const DataT* potM = pot + m * stridePhi; + const DataT* chargeM = charge + m * stridePhi; + DataT* workM = workBlock + blockSize * m; + const DataT* workPrev = workBlock + blockSize * std::max(m - 1, 0); // not used for m = 0 + for (int i = iStart; i < iEnd; i += 2) { + const int idx = row + i; + const int iw = i - iLow; + const DataT rhs = coefficient2[i] * potM[idx - 1] + coefficient1[i] * potM[idx + 1] + tempRatioZ * (potM[idx - strideZ] + potM[idx + strideZ]) + h2 * chargeM[idx]; + workM[iw] = (m == 0) ? rhs * invDen[i] : (rhs + bCoeff[i] * workPrev[iw]) * invDen[i + nr * m]; } } - // Anti-symmetry in phi - else if (symmetry == -1) { - if (mp1 > iPhi - 1) { - mp1 = iPhi - 2; - signPlus = -1; - } - if (mm1 < 0) { - mm1 = 1; - signMinus = -1; - } - } else { // No Symmetries in phi, no boundaries, the calculation is continuous across all phi - if (mp1 > iPhi - 1) { - mp1 = m + 1 - iPhi; + // back substitution + for (int m = n - 2; m >= 0; --m) { + DataT* workM = workBlock + blockSize * m; + const DataT* workNext = workBlock + blockSize * (m + 1); + for (int i = iStart; i < iEnd; i += 2) { + workM[i - iLow] -= cPrime[i + nr * m] * workNext[i - iLow]; } - if (mm1 < 0) { - mm1 = m - 1 + iPhi; - } - } - int isw = jsw; - for (int j = 1; j < tnZColumn - 1; ++j, isw = 3 - isw) { - for (int i = isw; i < tnRRow - 1; i += 2) { - (matricesCurrentV)(i, j, m) = (coefficient2[i] * (matricesCurrentV)(i - 1, j, m) + tempRatioZ * ((matricesCurrentV)(i, j - 1, m) + (matricesCurrentV)(i, j + 1, m)) + coefficient1[i] * (matricesCurrentV)(i + 1, j, m) + coefficient3[i] * (signPlus * (matricesCurrentV)(i, j, mp1) + signMinus * (matricesCurrentV)(i, j, mm1)) + (h2 * (matricesCurrentCharge)(i, j, m))) * coefficient4[i]; - } // end cols - } // end mParamGrid.NRVertices - } // end phi - } // end sweep - } else if (MGParameters::relaxType == RelaxType::Jacobi) { - // for each slice - for (int m = 0; m < iPhi; ++m) { - int mp1 = m + 1; - int signPlus = 1; - int mm1 = m - 1; - int signMinus = 1; - - // Reflection symmetry in phi (e.g. symmetry at sector boundaries, or half sectors, etc.) - if (symmetry == 1) { - if (mp1 > iPhi - 1) { - mp1 = iPhi - 2; - } - if (mm1 < 0) { - mm1 = 1; } - } - // Anti-symmetry in phi - else if (symmetry == -1) { - if (mp1 > iPhi - 1) { - mp1 = iPhi - 2; - signPlus = -1; - } - if (mm1 < 0) { - mm1 = 1; - signMinus = -1; - } - } else { // No Symmetries in phi, no boundaries, the calculation is continuous across all phi - if (mp1 > iPhi - 1) { - mp1 = m + 1 - iPhi; - } - if (mm1 < 0) { - mm1 = m - 1 + iPhi; + // Sherman-Morrison correction and store the solution + const DataT* work0 = workBlock; + const DataT* workLast = workBlock + blockSize * (n - 1); + for (int m = 0; m < n; ++m) { + DataT* potM = pot + m * stridePhi; + const DataT* workM = workBlock + blockSize * m; + for (int i = iStart; i < iEnd; i += 2) { + const DataT fact = (work0[i - iLow] + bOverA[i] * workLast[i - iLow]) * factDenInv[i]; + potM[row + i] = workM[i - iLow] - fact * zSM[i + nr * m]; + } } } - // Jacobian - for (int j = 1; j < tnZColumn - 1; ++j) { - for (int i = 1; i < tnRRow - 1; ++i) { - (matricesCurrentV)(i, j, m) = (coefficient2[i] * (matricesCurrentV)(i - 1, j, m) + tempRatioZ * ((matricesCurrentV)(i, j - 1, m) + (matricesCurrentV)(i, j + 1, m)) + coefficient1[i] * (matricesCurrentV)(i + 1, j, m) + coefficient3[i] * (signPlus * (matricesCurrentV)(i, j, mp1) + signMinus * (matricesCurrentV)(i, j, mm1)) + (h2 * (matricesCurrentCharge)(i, j, m))) * coefficient4[i]; - } // end cols - } // end mParamGrid.NRVertices - } // end phi - } else { - // Case weighted Jacobi - // TODO - } -} - -template -void PoissonSolver::relax2D(Vector& matricesCurrentV, const Vector& matricesCurrentCharge, const int tnRRow, const int tnZColumn, const DataT h2, const DataT tempFourth, const DataT tempRatio, - std::vector& coefficient1, std::vector& coefficient2) -{ - // Gauss-Seidel - const int iPhi = 0; - if (MGParameters::relaxType == RelaxType::GaussSeidel) { - int jsw = 1; - for (int iPass = 1; iPass <= 2; ++iPass, jsw = 3 - jsw) { - int isw = jsw; - for (int j = 1; j < tnZColumn - 1; ++j, isw = 3 - isw) { - for (int i = isw; i < tnRRow - 1; i += 2) { - matricesCurrentV(i, j, iPhi) = tempFourth * (coefficient1[i] * matricesCurrentV(i + 1, j, iPhi) + coefficient2[i] * matricesCurrentV(i - 1, j, iPhi) + - tempRatio * (matricesCurrentV(i, j + 1, iPhi) + matricesCurrentV(i, j - 1, iPhi)) + (h2 * matricesCurrentCharge(i, j, iPhi))); - } // end cols - } // end mParamGrid.NRVertices - } // end pass red-black - } else if (MGParameters::relaxType == RelaxType::Jacobi) { - for (int j = 1; j < tnZColumn - 1; ++j) { - for (int i = 1; i < tnRRow - 1; ++i) { - matricesCurrentV(i, j, iPhi) = tempFourth * (coefficient1[i] * matricesCurrentV(i + 1, j, iPhi) + coefficient2[i] * matricesCurrentV(i - 1, j, iPhi) + - tempRatio * (matricesCurrentV(i, j + 1, iPhi) + matricesCurrentV(i, j - 1, iPhi)) + (h2 * matricesCurrentCharge(i, j, iPhi))); - } // end cols - } // end mParamGrid.NRVertices - } else if (MGParameters::relaxType == RelaxType::WeightedJacobi) { - // Weighted Jacobi - // TODO + } } } template void PoissonSolver::restrictBoundary3D(Vector& matricesCurrentCharge, const Vector& residue, const int tnRRow, const int tnZColumn, const int newPhiSlice, const int oldPhiSlice) const { - // in case of full 3d and the mParamGrid.NPhiVertices is also coarsening - if (2 * newPhiSlice == oldPhiSlice) { - for (int m = 0, mm = 0; m < newPhiSlice; ++m, mm += 2) { - // for boundary - for (int j = 0, jj = 0; j < tnZColumn; ++j, jj += 2) { - matricesCurrentCharge(0, j, m) = residue(0, jj, mm); - matricesCurrentCharge(tnRRow - 1, j, m) = residue((tnRRow - 1) * 2, jj, mm); - } - - // for boundary - for (int i = 0, ii = 0; i < tnRRow; ++i, ii += 2) { - matricesCurrentCharge(i, 0, m) = residue(ii, 0, mm); - matricesCurrentCharge(i, tnZColumn - 1, m) = residue(ii, (tnZColumn - 1) * 2, mm); - } - } // end phis - } else { - for (int m = 0; m < newPhiSlice; ++m) { - restrictBoundary2D(matricesCurrentCharge, residue, tnRRow, tnZColumn, m); - } + // the phi direction is coarsened if the number of phi slices is halved + const int phiFactor = (2 * newPhiSlice == oldPhiSlice) ? 2 : 1; + for (int m = 0; m < newPhiSlice; ++m) { + restrictBoundary2D(matricesCurrentCharge, residue, tnRRow, tnZColumn, m, phiFactor * m); } } template -void PoissonSolver::restrictBoundary2D(Vector& matricesCurrentCharge, const Vector& residue, const int tnRRow, const int tnZColumn, const int tnPhi) const +void PoissonSolver::restrictBoundary2D(Vector& matricesCurrentCharge, const Vector& residue, const int tnRRow, const int tnZColumn, const int phiCoarse, const int phiFine) const { - // for boundary + // boundary in r for (int j = 0, jj = 0; j < tnZColumn; ++j, jj += 2) { - matricesCurrentCharge(0, j, tnPhi) = residue(0, jj, tnPhi); - matricesCurrentCharge(tnRRow - 1, j, tnPhi) = residue((tnRRow - 1) * 2, jj, tnPhi); + matricesCurrentCharge(0, j, phiCoarse) = residue(0, jj, phiFine); + matricesCurrentCharge(tnRRow - 1, j, phiCoarse) = residue((tnRRow - 1) * 2, jj, phiFine); } - // for boundary + // boundary in z for (int i = 0, ii = 0; i < tnRRow; ++i, ii += 2) { - matricesCurrentCharge(i, 0, tnPhi) = residue(ii, 0, tnPhi); - matricesCurrentCharge(i, tnZColumn - 1, tnPhi) = residue(ii, (tnZColumn - 1) * 2, tnPhi); + matricesCurrentCharge(i, 0, phiCoarse) = residue(ii, 0, phiFine); + matricesCurrentCharge(i, tnZColumn - 1, phiCoarse) = residue(ii, (tnZColumn - 1) * 2, phiFine); } } template void PoissonSolver::restrict3D(Vector& matricesCurrentCharge, const Vector& residue, const int tnRRow, const int tnZColumn, const int newPhiSlice, const int oldPhiSlice) const { + const bool runParallel = (sNThreads > 1) && (static_cast(tnRRow) * tnZColumn * newPhiSlice > MINPOINTSPARALLEL / 4); if (2 * newPhiSlice == oldPhiSlice) { - int mm = 0; - for (int m = 0; m < newPhiSlice; m++, mm += 2) { - // assuming no symmetry - int mp1 = mm + 1; - int mm1 = mm - 1; - - if (mp1 > (oldPhiSlice)-1) { - mp1 = mm + 1 - (oldPhiSlice); - } - if (mm1 < 0) { - mm1 = mm - 1 + (oldPhiSlice); - } + // each iteration writes only to the coarse slice m +#pragma omp parallel for num_threads(sNThreads) if (runParallel) + for (int m = 0; m < newPhiSlice; ++m) { + const int mm = 2 * m; + int mp1, mm1; + getPhiNeighbours(mm, oldPhiSlice, mp1, mm1); for (int i = 1, ii = 2; i < tnRRow - 1; ++i, ii += 2) { for (int j = 1, jj = 2; j < tnZColumn - 1; ++j, jj += 2) { @@ -1422,21 +693,11 @@ void PoissonSolver::restrict3D(Vector& matricesCurrentCharge, const Vecto matricesCurrentCharge(i, j, m) = residue(ii, jj, mm) / 8 + s1 / 16 + s2 / 32 + s3 / 64; } // end cols } // end mParamGrid.NRVertices - - // for boundary - for (int j = 0, jj = 0; j < tnZColumn; ++j, jj += 2) { - matricesCurrentCharge(0, j, m) = residue(0, jj, mm); - matricesCurrentCharge(tnRRow - 1, j, m) = residue((tnRRow - 1) * 2, jj, mm); - } - - // for boundary - for (int i = 0, ii = 0; i < tnRRow; ++i, ii += 2) { - matricesCurrentCharge(i, 0, m) = residue(ii, 0, mm); - matricesCurrentCharge(i, tnZColumn - 1, m) = residue(ii, (tnZColumn - 1) * 2, mm); - } + restrictBoundary2D(matricesCurrentCharge, residue, tnRRow, tnZColumn, m, mm); } // end phis } else { +#pragma omp parallel for num_threads(sNThreads) if (runParallel) for (int m = 0; m < newPhiSlice; ++m) { restrict2D(matricesCurrentCharge, residue, tnRRow, tnZColumn, m); } @@ -1452,45 +713,47 @@ void PoissonSolver::restrict2D(Vector& matricesCurrentCharge, const Vecto const int iim1 = ii - 1; const int jjp1 = jj + 1; const int jjm1 = jj - 1; - if (MGParameters::gtType == GridTransferType::Half) { - // half - matricesCurrentCharge(i, j, iphi) = residue(ii, jj, iphi) / 2 + (residue(iip1, jj, iphi) + residue(iim1, jj, iphi) + residue(ii, jjp1, iphi) + residue(ii, jjm1, iphi)) / 8; - } else if (MGParameters::gtType == GridTransferType::Full) { - matricesCurrentCharge(i, j, iphi) = residue(ii, jj, iphi) / 4 + (residue(iip1, jj, iphi) + residue(iim1, jj, iphi) + residue(ii, jjp1, iphi) + residue(ii, jjm1, iphi)) / 8 + - (residue(iip1, jjp1, iphi) + residue(iim1, jjp1, iphi) + residue(iip1, jjm1, iphi) + residue(iim1, jjm1, iphi)) / 16; - } + matricesCurrentCharge(i, j, iphi) = residue(ii, jj, iphi) / 4 + (residue(iip1, jj, iphi) + residue(iim1, jj, iphi) + residue(ii, jjp1, iphi) + residue(ii, jjm1, iphi)) / 8 + + (residue(iip1, jjp1, iphi) + residue(iim1, jjp1, iphi) + residue(iip1, jjm1, iphi) + residue(iim1, jjm1, iphi)) / 16; } // end cols } // end mParamGrid.NRVertices - // boundary - // for boundary - for (int j = 0, jj = 0; j < tnZColumn; ++j, jj += 2) { - matricesCurrentCharge(0, j, iphi) = residue(0, jj, iphi); - matricesCurrentCharge(tnRRow - 1, j, iphi) = residue((tnRRow - 1) * 2, jj, iphi); - } - // for boundary - for (int i = 0, ii = 0; i < tnRRow; ++i, ii += 2) { - matricesCurrentCharge(i, 0, iphi) = residue(ii, 0, iphi); - matricesCurrentCharge(i, tnZColumn - 1, iphi) = residue(ii, (tnZColumn - 1) * 2, iphi); - } + restrictBoundary2D(matricesCurrentCharge, residue, tnRRow, tnZColumn, iphi, iphi); } template -DataT PoissonSolver::getConvergenceError(const Vector& matricesCurrentV, Vector& prevArrayV) const +DataT PoissonSolver::getConvergenceError(const Vector& matricesCurrentV, const Vector& prevArrayV) const { - std::vector errorArr(prevArrayV.getNphi()); - - // subtract the two matrices - std::transform(prevArrayV.begin(), prevArrayV.end(), matricesCurrentV.begin(), prevArrayV.begin(), std::minus()); - -#pragma omp parallel for num_threads(sNThreads) // parallising this loop is possible - but using more than 2 cores makes it slower - - for (unsigned int m = 0; m < prevArrayV.getNphi(); ++m) { - // square each entry in the vector and sum them up - const auto phiStep = prevArrayV.getNr() * prevArrayV.getNz(); // number of points in one phi slice - const auto start = prevArrayV.begin() + m * phiStep; - const auto end = start + phiStep; - errorArr[m] = std::inner_product(start, end, start, DataT(0)); // inner product "Sum (matrix[a]*matrix[a])" - } - // return largest error + const int nPhi = prevArrayV.getNphi(); + const size_t phiStep = static_cast(prevArrayV.getNr()) * prevArrayV.getNz(); // number of points in one phi slice + std::vector errorArr(nPhi); + std::vector normArr(nPhi); + const DataT* prev = prevArrayV.data().data(); + const DataT* curr = matricesCurrentV.data().data(); + +#pragma omp parallel for num_threads(sNThreads) if ((sNThreads > 1) && (phiStep * nPhi > MINPOINTSPARALLEL)) + for (int m = 0; m < nPhi; ++m) { + // sum of the squared differences and of the squared potential in the phi slice + const DataT* prevM = prev + m * phiStep; + const DataT* currM = curr + m * phiStep; + DataT sum = 0; + DataT sumNorm = 0; + for (size_t k = 0; k < phiStep; ++k) { + const DataT diff = prevM[k] - currM[k]; + sum = sum + diff * diff; + sumNorm = sumNorm + currM[k] * currM[k]; + } + errorArr[m] = sum; + normArr[m] = sumNorm; + } + + if (MGParameters::relativeConvergence) { + // relative change of the potential: sqrt(sum dV^2 / sum V^2) + const DataT sumDiff = std::accumulate(errorArr.begin(), errorArr.end(), DataT(0)); + const DataT sumNorm = std::accumulate(normArr.begin(), normArr.end(), DataT(0)); + return (sumNorm > 0) ? std::sqrt(sumDiff / sumNorm) : std::sqrt(sumDiff); + } + + // largest sum of dV^2 of the phi slices return *std::max_element(std::begin(errorArr), std::end(errorArr)); } @@ -1498,7 +761,7 @@ template void PoissonSolver::calcCoefficients(unsigned int from, unsigned int to, const DataT h, const DataT tempRatioZ, const DataT tempRatioPhi, std::vector& coefficient1, std::vector& coefficient2, std::vector& coefficient3, std::vector& coefficient4) const { for (unsigned int i = from; i < to; ++i) { - const DataT radiusInv = 1 / (TPCParameters::IFCRADIUS + i * h); + const DataT radiusInv = 1 / (mGrid3D.getGridMinR() + i * h); const DataT hRadiusTmp = h * radiusInv / 2; coefficient1[i] = 1 + hRadiusTmp; coefficient2[i] = 1 - hRadiusTmp; @@ -1508,13 +771,43 @@ void PoissonSolver::calcCoefficients(unsigned int from, unsigned int to, } template -void PoissonSolver::calcCoefficients2D(unsigned int from, unsigned int to, const DataT h, std::vector& coefficient1, std::vector& coefficient2) const +DataT PoissonSolver::getRelCouplingPhi(const MGLevel& level, const DataT radius) const { - for (int i = from; i < to; ++i) { - DataT radiusInvHalf = h / (2 * (TPCParameters::IFCRADIUS + i * h)); - coefficient1[i] = 1 + radiusInvHalf; - coefficient2[i] = 1 - radiusInvHalf; - } + const DataT spacingRZ = std::min(getSpacingR(), getSpacingZ()) * level.rzFactor; + const DataT spacingPhiInv = level.nphi * getGridSizePhiInv(); + const DataT rel = spacingRZ * spacingPhiInv / radius; + return rel * rel; +} + +template +std::vector::MGLevel> PoissonSolver::getMultiGridLevels(const int nLevelsRZ) const +{ + std::vector levels{MGLevel{mParamGrid.NRVertices, mParamGrid.NZVertices, mParamGrid.NPhiVertices, 1}}; + const DataT radiusMin = mGrid3D.getGridMinR(); + const int maxRZFactor = 1 << (nLevelsRZ - 1); + while (levels.back().rzFactor < maxRZFactor) { + const MGLevel fine = levels.back(); + + // The point-wise relaxation does not smooth the error well in a direction which is only weakly coupled compared to the other directions + // and coarsening in such a direction leads to a slow convergence of the multi grid (number of V cycles increases). + // Coarsen r and z. Coarsen phi only if the coupling in phi (largest at the inner radius) is not weak compared to the coupling in r/z: (h_{r,z} / (r_{min} * h_{phi}))^2 + bool coarsenPhi = (fine.nphi % 2 == 0); + if (coarsenPhi && (MGParameters::phiCoarseningThreshold > 0)) { + coarsenPhi = getRelCouplingPhi(fine, radiusMin) >= MGParameters::phiCoarseningThreshold; + } + const int rzFactor = 2 * fine.rzFactor; + levels.push_back(MGLevel{mParamGrid.NRVertices / rzFactor + 1, mParamGrid.NZVertices / rzFactor + 1, coarsenPhi ? fine.nphi / 2 : fine.nphi, rzFactor}); + } + + // line relaxation in phi on the levels where phi is strongly coupled + std::string levelInfo; + for (auto& level : levels) { + // the periodic tridiagonal solver needs at least 3 phi slices (for 1 or 2 slices the neighbours in phi coincide, the point-wise relaxation is used) + level.lineRelaxPhi = (MGParameters::phiLineRelaxThreshold > 0) && (level.nphi >= 3) && (getRelCouplingPhi(level, radiusMin) >= MGParameters::phiLineRelaxThreshold); + levelInfo += fmt::format(" {}x{}x{}{}", level.nr, level.nz, level.nphi, level.lineRelaxPhi ? "(line)" : ""); + } + LOGP(detail, "Multi grid levels (r x z x phi):{}", levelInfo); + return levels; } template diff --git a/Detectors/TPC/spacecharge/src/SpaceCharge.cxx b/Detectors/TPC/spacecharge/src/SpaceCharge.cxx index bcf75c9df3419..b2ef7c1a0d7c3 100644 --- a/Detectors/TPC/spacecharge/src/SpaceCharge.cxx +++ b/Detectors/TPC/spacecharge/src/SpaceCharge.cxx @@ -41,7 +41,9 @@ #include "TCanvas.h" #include "TROOT.h" #include "TStopwatch.h" -#include "THnSparse.h" +#include +#include +#include #include "TRandom.h" #include @@ -57,6 +59,22 @@ templateClassImp(o2::tpc::SpaceCharge); using namespace o2::tpc; +namespace +{ +/// print a warning if the global distortions/corrections contain non-finite values (the drift of the electrons left the grid, e.g. for unphysically large fields) +template +void warnNonFinite(const DataContainer3D& dR, const DataContainer3D& dRPhi, const DataContainer3D& dZ, const std::string_view name, const o2::tpc::Side side) +{ + size_t nNonFinite = 0; + for (const auto* data : {&dR, &dRPhi, &dZ}) { + nNonFinite += std::count_if(data->getData().begin(), data->getData().end(), [](const DataT val) { return !std::isfinite(val); }); + } + if (nNonFinite) { + LOGP(warning, "{} for side {}: {} non-finite values (the electron drift left the grid)", name, (side == o2::tpc::Side::A) ? "A" : "C", nNonFinite); + } +} +} // namespace + template SpaceCharge::SpaceCharge(const int bfield, const unsigned short nZVertices, const unsigned short nRVertices, const unsigned short nPhiVertices, const bool initBuffers) : mParamGrid{nRVertices, nZVertices, nPhiVertices} { @@ -76,28 +94,25 @@ SpaceCharge::SpaceCharge() template void SpaceCharge::initAllBuffers() { + std::vector containers; for (int iside = 0; iside < SIDES; ++iside) { const Side side = (iside == 0) ? Side::A : Side::C; - initContainer(mLocalDistdR[side], true); - initContainer(mLocalDistdZ[side], true); - initContainer(mLocalDistdRPhi[side], true); - initContainer(mLocalVecDistdR[side], true); - initContainer(mLocalVecDistdZ[side], true); - initContainer(mLocalVecDistdRPhi[side], true); - initContainer(mLocalCorrdR[side], true); - initContainer(mLocalCorrdZ[side], true); - initContainer(mLocalCorrdRPhi[side], true); - initContainer(mGlobalDistdR[side], true); - initContainer(mGlobalDistdZ[side], true); - initContainer(mGlobalDistdRPhi[side], true); - initContainer(mGlobalCorrdR[side], true); - initContainer(mGlobalCorrdZ[side], true); - initContainer(mGlobalCorrdRPhi[side], true); - initContainer(mDensity[side], true); - initContainer(mPotential[side], true); - initContainer(mElectricFieldEr[side], true); - initContainer(mElectricFieldEz[side], true); - initContainer(mElectricFieldEphi[side], true); + containers.insert(containers.end(), {&mLocalDistdR[side], &mLocalDistdZ[side], &mLocalDistdRPhi[side], &mLocalVecDistdR[side], &mLocalVecDistdZ[side], &mLocalVecDistdRPhi[side], + &mLocalCorrdR[side], &mLocalCorrdZ[side], &mLocalCorrdRPhi[side], &mGlobalDistdR[side], &mGlobalDistdZ[side], &mGlobalDistdRPhi[side], + &mGlobalCorrdR[side], &mGlobalCorrdZ[side], &mGlobalCorrdRPhi[side], &mDensity[side], &mPotential[side], + &mElectricFieldEr[side], &mElectricFieldEz[side], &mElectricFieldEphi[side]}); + } + initContainers(containers); +} + +template +void SpaceCharge::initContainers(const std::vector& containers) +{ + // the memory of a new container is zeroed by a single thread. For large grids the first access to the memory (page faults) dominates, + // which is done concurrently for different containers +#pragma omp parallel for num_threads(sNThreads) schedule(dynamic, 1) + for (size_t i = 0; i < containers.size(); ++i) { + initContainer(*containers[i], true); } } @@ -121,6 +136,20 @@ void SpaceCharge::calculateDistortionsCorrections(const o2::tpc::Side sid const std::array sideName{"A", "C"}; LOGP(info, "====== starting calculation of distortions and corrections for Side {} ======", sideName[side]); + + // allocate all containers which are filled during the calculation at once (concurrently) + std::vector containers{&mPotential[side], &mElectricFieldEr[side], &mElectricFieldEz[side], &mElectricFieldEphi[side], + &mLocalCorrdR[side], &mLocalCorrdZ[side], &mLocalCorrdRPhi[side], &mGlobalCorrdR[side], &mGlobalCorrdZ[side], &mGlobalCorrdRPhi[side]}; + if (getGlobalDistType() != SC::GlobalDistType::None) { + containers.insert(containers.end(), {&mGlobalDistdR[side], &mGlobalDistdZ[side], &mGlobalDistdRPhi[side]}); + } + if (getGlobalDistType() == SC::GlobalDistType::Standard) { + containers.insert(containers.end(), {&mLocalDistdR[side], &mLocalDistdZ[side], &mLocalDistdRPhi[side]}); + } + if (calcVectors) { + containers.insert(containers.end(), {&mLocalVecDistdR[side], &mLocalVecDistdZ[side], &mLocalVecDistdRPhi[side]}); + } + initContainers(containers); LOGP(info, "Using {} threads", getNThreads()); if (getGlobalDistCorrMethod() == SC::GlobalDistCorrMethod::LocalDistCorr) { @@ -599,22 +628,23 @@ void SpaceCharge::setPotentialBoundaryFromFormula(const AnalyticalFields< } template -void SpaceCharge::poissonSolver(const Side side, const DataT stoppingConvergence, const int symmetry) +void SpaceCharge::poissonSolver(const Side side, const DataT stoppingConvergence) { initContainer(mDensity[side], true); initContainer(mPotential[side], true); PoissonSolver::setConvergenceError(stoppingConvergence); + PoissonSolver::setNThreads(sNThreads); PoissonSolver poissonSolver(mGrid3D[0]); - poissonSolver.poissonSolver3D(mPotential[side], mDensity[side], symmetry); + poissonSolver.poissonSolver3D(mPotential[side], mDensity[side]); } template -void SpaceCharge::poissonSolver(const DataT stoppingConvergence, const int symmetry) +void SpaceCharge::poissonSolver(const DataT stoppingConvergence) { -#pragma omp parallel for num_threads(2) + // the sides are processed one after the other: the calculation for each side is parallelised (nested OpenMP regions are not active by default, so a parallel loop over the two sides would use only two threads in total) for (int iside = 0; iside < FNSIDES; ++iside) { const o2::tpc::Side side = (iside == 0) ? Side::A : Side::C; - poissonSolver(side, stoppingConvergence, symmetry); + poissonSolver(side, stoppingConvergence); } } @@ -650,32 +680,9 @@ void SpaceCharge::calcEField(const Side side) initContainer(mElectricFieldEphi[side], true); #pragma omp parallel for num_threads(sNThreads) for (size_t iPhi = 0; iPhi < mParamGrid.NPhiVertices; ++iPhi) { - const int symmetry = 0; - size_t tmpPlus = iPhi + 1; - int signPlus = 1; - int tmpMinus = static_cast(iPhi - 1); - int signMinus = 1; - if (symmetry == 1 || symmetry == -1) { // Reflection symmetry in phi (e.g. symmetry at sector boundaries, or half sectors, etc.) - if (tmpPlus > mParamGrid.NPhiVertices - 1) { - if (symmetry == -1) { - signPlus = -1; - } - tmpPlus = mParamGrid.NPhiVertices - 2; - } - if (tmpMinus < 0) { - tmpMinus = 1; // SHOULD IT BE =0? - if (symmetry == -1) { - signMinus = -1; - } - } - } else { // No Symmetries in phi, no boundaries, the calculations is continuous across all phi - if (tmpPlus > mParamGrid.NPhiVertices - 1) { - tmpPlus = iPhi + 1 - mParamGrid.NPhiVertices; - } - if (tmpMinus < 0) { - tmpMinus = static_cast(iPhi - 1 + mParamGrid.NPhiVertices); - } - } + // neighbouring phi slices (periodic boundary in phi) + const size_t tmpPlus = (iPhi + 1 > mParamGrid.NPhiVertices - 1) ? (iPhi + 1 - mParamGrid.NPhiVertices) : (iPhi + 1); + const size_t tmpMinus = (iPhi == 0) ? (mParamGrid.NPhiVertices - 1) : (iPhi - 1); // for non-boundary V for (size_t iR = 1; iR < mParamGrid.NRVertices - 1; iR++) { @@ -683,7 +690,7 @@ void SpaceCharge::calcEField(const Side side) for (size_t iZ = 1; iZ < mParamGrid.NZVertices - 1; iZ++) { mElectricFieldEr[side](iZ, iR, iPhi) = -1 * (mPotential[side](iZ, iR + 1, iPhi) - mPotential[side](iZ, iR - 1, iPhi)) * static_cast(0.5) * getInvSpacingR(side); // r direction mElectricFieldEz[side](iZ, iR, iPhi) = -1 * (mPotential[side](iZ + 1, iR, iPhi) - mPotential[side](iZ - 1, iR, iPhi)) * static_cast(0.5) * getInvSpacingZ(side); // z direction - mElectricFieldEphi[side](iZ, iR, iPhi) = -1 * (signPlus * mPotential[side](iZ, iR, tmpPlus) - signMinus * mPotential[side](iZ, iR, tmpMinus)) * static_cast(0.5) * getInvSpacingPhi(side) / radius; // phi direction + mElectricFieldEphi[side](iZ, iR, iPhi) = -1 * (mPotential[side](iZ, iR, tmpPlus) - mPotential[side](iZ, iR, tmpMinus)) * static_cast(0.5) * getInvSpacingPhi(side) / radius; // phi direction } } @@ -697,7 +704,7 @@ void SpaceCharge::calcEField(const Side side) const DataT radius = getRVertex(iR, side); for (size_t iZ = 1; iZ < mParamGrid.NZVertices - 1; iZ++) { mElectricFieldEz[side](iZ, iR, iPhi) = -1 * (mPotential[side](iZ + 1, iR, iPhi) - mPotential[side](iZ - 1, iR, iPhi)) * static_cast(0.5) * getInvSpacingZ(side); // z direction - mElectricFieldEphi[side](iZ, iR, iPhi) = -1 * (signPlus * mPotential[side](iZ, iR, tmpPlus) - signMinus * mPotential[side](iZ, iR, tmpMinus)) * static_cast(0.5) * getInvSpacingPhi(side) / radius; // phi direction + mElectricFieldEphi[side](iZ, iR, iPhi) = -1 * (mPotential[side](iZ, iR, tmpPlus) - mPotential[side](iZ, iR, tmpMinus)) * static_cast(0.5) * getInvSpacingPhi(side) / radius; // phi direction } } @@ -711,7 +718,7 @@ void SpaceCharge::calcEField(const Side side) const DataT radius = getRVertex(iR, side); for (size_t iZ = 0; iZ < mParamGrid.NZVertices; iZ += mParamGrid.NZVertices - 1) { mElectricFieldEr[side](iZ, iR, iPhi) = -1 * (mPotential[side](iZ, iR + 1, iPhi) - mPotential[side](iZ, iR - 1, iPhi)) * static_cast(0.5) * getInvSpacingR(side); // r direction - mElectricFieldEphi[side](iZ, iR, iPhi) = -1 * (signPlus * mPotential[side](iZ, iR, tmpPlus) - signMinus * mPotential[side](iZ, iR, tmpMinus)) * static_cast(0.5) * getInvSpacingPhi(side) / radius; // phi direction + mElectricFieldEphi[side](iZ, iR, iPhi) = -1 * (mPotential[side](iZ, iR, tmpPlus) - mPotential[side](iZ, iR, tmpMinus)) * static_cast(0.5) * getInvSpacingPhi(side) / radius; // phi direction } } @@ -719,7 +726,7 @@ void SpaceCharge::calcEField(const Side side) for (size_t iR = 0; iR < mParamGrid.NRVertices; iR += mParamGrid.NRVertices - 1) { const DataT radius = getRVertex(iR, side); for (size_t iZ = 0; iZ < mParamGrid.NZVertices; iZ += mParamGrid.NZVertices - 1) { - mElectricFieldEphi[side](iZ, iR, iPhi) = -1 * (signPlus * mPotential[side](iZ, iR, tmpPlus) - signMinus * mPotential[side](iZ, iR, tmpMinus)) * static_cast(0.5) * getInvSpacingPhi(side) / radius; // phi direction + mElectricFieldEphi[side](iZ, iR, iPhi) = -1 * (mPotential[side](iZ, iR, tmpPlus) - mPotential[side](iZ, iR, tmpMinus)) * static_cast(0.5) * getInvSpacingPhi(side) / radius; // phi direction } } } @@ -744,7 +751,7 @@ void SpaceCharge::calcGlobalDistWithGlobalCorrIterative(const Side side, template void SpaceCharge::calcGlobalDistWithGlobalCorrIterative(const SpaceCharge* scSCale, float scale, const int maxIter, const DataT approachZ, const DataT approachR, const DataT approachPhi, const DataT diffCorr) { -#pragma omp parallel for num_threads(sNThreads) + // the sides are processed one after the other: the calculation for each side is parallelised (nested OpenMP regions are not active by default, so a parallel loop over the two sides would use only two threads in total) for (int iside = 0; iside < FNSIDES; ++iside) { const o2::tpc::Side side = (iside == 0) ? Side::A : Side::C; calcGlobalDistWithGlobalCorrIterative(side, scSCale, scale, maxIter, approachZ, approachR, approachPhi, diffCorr); @@ -760,7 +767,7 @@ void SpaceCharge::calcGlobalCorrWithGlobalDistIterative(const Side side, template void SpaceCharge::calcGlobalCorrWithGlobalDistIterative(const SpaceCharge* scSCale, float scale, const int maxIter, const DataT approachZ, const DataT approachR, const DataT approachPhi, const DataT diffCorr) { -#pragma omp parallel for num_threads(sNThreads) + // the sides are processed one after the other: the calculation for each side is parallelised (nested OpenMP regions are not active by default, so a parallel loop over the two sides would use only two threads in total) for (int iside = 0; iside < FNSIDES; ++iside) { const o2::tpc::Side side = (iside == 0) ? Side::A : Side::C; calcGlobalCorrWithGlobalDistIterative(side, scSCale, scale, maxIter, approachZ, approachR, approachPhi, diffCorr); @@ -781,7 +788,9 @@ void SpaceCharge::calcGlobalDistCorrIterative(const DistCorrInterpolator< initContainer(mGlobalCorrdRPhi[side], true); } - const auto& scSCaleInterpolator = (type == Type::Distortions) ? scSCale->mInterpolatorGlobalCorr[side] : scSCale->mInterpolatorGlobalDist[side]; + // interpolator of the object used for scaling (only used if scSCale is set and scale != 0) + const bool useScaling = scSCale && (scale != 0); + const DistCorrInterpolator* scSCaleInterpolator = useScaling ? ((type == Type::Distortions) ? &scSCale->mInterpolatorGlobalCorr[side] : &scSCale->mInterpolatorGlobalDist[side]) : nullptr; #pragma omp parallel for num_threads(sNThreads) for (unsigned int iPhi = 0; iPhi < mParamGrid.NPhiVertices; ++iPhi) { @@ -824,25 +833,34 @@ void SpaceCharge::calcGlobalDistCorrIterative(const DistCorrInterpolator< } // interpolate global correction at new point and calculate position of global correction - corrdR = globCorr.evaldR(zCurrPos, rCurrPos, phiCurrPos); - if (scSCale && scale != 0) { - corrdR += scale * scSCaleInterpolator.evaldR(zCurrPos, rCurrPos, phiCurrPos); + DataT globCorrdZ, globCorrdR, globCorrdRPhi; + globCorr.evalDistCorr(zCurrPos, rCurrPos, phiCurrPos, globCorrdZ, globCorrdR, globCorrdRPhi); + DataT scaleCorrdZ = 0; + DataT scaleCorrdR = 0; + DataT scaleCorrdRPhi = 0; + if (useScaling) { + scSCaleInterpolator->evalDistCorr(zCurrPos, rCurrPos, phiCurrPos, scaleCorrdZ, scaleCorrdR, scaleCorrdRPhi); + } + + corrdR = globCorrdR; + if (useScaling) { + corrdR += scale * scaleCorrdR; } const DataT rNewPos = rCurrPos + corrdR; DataT corrPhi = 0; - if (scSCale && scale != 0) { - corrPhi = scale * scSCaleInterpolator.evaldRPhi(zCurrPos, rCurrPos, phiCurrPos); + if (useScaling) { + corrPhi = scale * scaleCorrdRPhi; } - corrPhi += globCorr.evaldRPhi(zCurrPos, rCurrPos, phiCurrPos); + corrPhi += globCorrdRPhi; corrPhi /= rCurrPos; corrdRPhi = corrPhi * rNewPos; // normalize to new r coordinate const DataT phiNewPos = phiCurrPos + corrPhi; - corrdZ = globCorr.evaldZ(zCurrPos, rCurrPos, phiCurrPos); - if (scSCale && scale != 0) { - corrdZ += scale * scSCaleInterpolator.evaldZ(zCurrPos, rCurrPos, phiCurrPos); + corrdZ = globCorrdZ; + if (useScaling) { + corrdZ += scale * scaleCorrdZ; } const DataT zNewPos = zCurrPos + corrdZ; @@ -889,6 +907,11 @@ void SpaceCharge::calcGlobalDistCorrIterative(const DistCorrInterpolator< } } } + if (type == Type::Distortions) { + warnNonFinite(mGlobalDistdR[side], mGlobalDistdRPhi[side], mGlobalDistdZ[side], "global distortions", side); + } else { + warnNonFinite(mGlobalCorrdR[side], mGlobalCorrdRPhi[side], mGlobalCorrdZ[side], "global corrections", side); + } } template @@ -900,7 +923,7 @@ void SpaceCharge::calcGlobalDistWithGlobalCorrIterativeLinearCartesian(co template void SpaceCharge::calcGlobalDistWithGlobalCorrIterativeLinearCartesian(const SpaceCharge* scSCale, float scale, const int maxIter, const DataT approachX, const DataT approachY, const DataT approachZ, const DataT diffCorr) { -#pragma omp parallel for num_threads(sNThreads) + // the sides are processed one after the other: the calculation for each side is parallelised (nested OpenMP regions are not active by default, so a parallel loop over the two sides would use only two threads in total) for (int iside = 0; iside < FNSIDES; ++iside) { const o2::tpc::Side side = (iside == 0) ? Side::A : Side::C; calcGlobalDistWithGlobalCorrIterativeLinearCartesian(side, scSCale, scale, maxIter, approachX, approachY, approachZ, diffCorr); @@ -916,7 +939,7 @@ void SpaceCharge::calcGlobalCorrWithGlobalDistIterativeLinearCartesian(co template void SpaceCharge::calcGlobalCorrWithGlobalDistIterativeLinearCartesian(const SpaceCharge* scSCale, float scale, const int maxIter, const DataT approachX, const DataT approachY, const DataT approachZ, const DataT diffCorr) { -#pragma omp parallel for num_threads(sNThreads) + // the sides are processed one after the other: the calculation for each side is parallelised (nested OpenMP regions are not active by default, so a parallel loop over the two sides would use only two threads in total) for (int iside = 0; iside < FNSIDES; ++iside) { const o2::tpc::Side side = (iside == 0) ? Side::A : Side::C; calcGlobalCorrWithGlobalDistIterativeLinearCartesian(side, scSCale, scale, maxIter, approachX, approachY, approachZ, diffCorr); @@ -937,8 +960,6 @@ void SpaceCharge::calcGlobalDistCorrIterativeLinearCartesian(const DistCo initContainer(mGlobalCorrdRPhi[side], true); } - const auto& scSCaleInterpolator = (type == Type::Distortions) ? scSCale->mInterpolatorGlobalCorr[side] : scSCale->mInterpolatorGlobalDist[side]; - #pragma omp parallel for num_threads(sNThreads) for (unsigned int iPhi = 0; iPhi < mParamGrid.NPhiVertices; ++iPhi) { const DataT phi = getPhiVertex(iPhi, side); @@ -983,7 +1004,7 @@ void SpaceCharge::calcGlobalDistCorrIterativeLinearCartesian(const DistCo DataT corrXScale = 0; DataT corrYScale = 0; DataT corrZScale = 0; - (type == Type::Distortions) ? scSCale->getCorrections(xCurrPos, yCurrPos, zCurrPos, side, corrXScale, corrYScale, corrZScale) : getDistortions(xCurrPos, yCurrPos, zCurrPos, side, corrX, corrY, corrZ); + (type == Type::Distortions) ? scSCale->getCorrections(xCurrPos, yCurrPos, zCurrPos, side, corrXScale, corrYScale, corrZScale) : scSCale->getDistortions(xCurrPos, yCurrPos, zCurrPos, side, corrXScale, corrYScale, corrZScale); corrX += scale * corrXScale; corrY += scale * corrYScale; corrZ += scale * corrZScale; @@ -1021,11 +1042,11 @@ void SpaceCharge::calcGlobalDistCorrIterativeLinearCartesian(const DistCo const DataT radiusNew = getRadiusFromCartesian(xNew, yNew); const DataT corrdR = -(radiusNew - getRadiusFromCartesian(lastX, lastY)); - float phiNew = getPhiFromCartesian(xNew, yNew); - o2::math_utils::bringTo02PiGen(phiNew); + DataT phiNew = getPhiFromCartesian(xNew, yNew); + o2::math_utils::detail::bringTo02PiGen(phiNew); - float phiLast = getPhiFromCartesian(lastX, lastY); - o2::math_utils::bringTo02PiGen(phiLast); + DataT phiLast = getPhiFromCartesian(lastX, lastY); + o2::math_utils::detail::bringTo02PiGen(phiLast); DataT deltaPhi = (phiNew - phiLast); // handle edge cases @@ -1135,8 +1156,36 @@ void SpaceCharge::fillChargeFromCalDet(const std::vector>& fillChargeDensityFromHisto(hConverted); } +namespace +{ +/// call func with an accessor (iPhi, iR, iZ) -> bin content of the histogram. Direct access to the bin array for TH3F and TH3D +/// (exact class: derived classes such as TProfile3D compute the bin content differently and use GetBinContent()) +template +void withBinContentAccessor(const TH3& h, Func&& func) +{ + const int nx = h.GetNbinsX() + 2; + const int ny = h.GetNbinsY() + 2; + if (h.IsA() == TH3F::Class()) { + const float* data = static_cast(h).GetArray(); + func([data, nx, ny](int ix, int iy, int iz) { return static_cast(data[ix + nx * (iy + ny * iz)]); }); + } else if (h.IsA() == TH3D::Class()) { + const double* data = static_cast(h).GetArray(); + func([data, nx, ny](int ix, int iy, int iz) { return data[ix + nx * (iy + ny * iz)]; }); + } else { + func([&h](int ix, int iy, int iz) { return h.GetBinContent(ix, iy, iz); }); + } +} +} // namespace + template void SpaceCharge::fillChargeDensityFromHisto(const TH3& hOrig) +{ + withBinContentAccessor(hOrig, [&](const auto& getBinContent) { fillChargeDensityFromHisto(*hOrig.GetXaxis(), *hOrig.GetYaxis(), *hOrig.GetZaxis(), getBinContent); }); +} + +template +template +void SpaceCharge::fillChargeDensityFromHisto(const TAxis& axisPhiOrig, const TAxis& axisROrig, const TAxis& axisZOrig, const BinContent& getBinContentOrig) { const unsigned short nBinsZNew = mParamGrid.NZVertices; const unsigned short nBinsRNew = mParamGrid.NRVertices; @@ -1146,55 +1195,46 @@ void SpaceCharge::fillChargeDensityFromHisto(const TH3& hOrig) initContainer(mDensity[Side::C], true); const int nBinsZNewTwo = 2 * nBinsZNew; - const auto phiLow = hOrig.GetXaxis()->GetBinLowEdge(1); - const auto phiUp = hOrig.GetXaxis()->GetBinUpEdge(hOrig.GetNbinsX()); - const auto rLow = hOrig.GetYaxis()->GetBinLowEdge(1); - const auto rUp = hOrig.GetYaxis()->GetBinUpEdge(hOrig.GetNbinsY()); - const auto zLow = hOrig.GetZaxis()->GetBinLowEdge(1); - const auto zUp = hOrig.GetZaxis()->GetBinUpEdge(hOrig.GetNbinsZ()); - - const int dim = 3; - int bins[dim]{nBinsPhiNew, nBinsRNew, nBinsZNewTwo}; - double xmin[dim]{phiLow, rLow, zLow}; - double xmax[dim]{phiUp, rUp, zUp}; - const THnSparseF hRebin("hTmp", "hTmp", dim, bins, xmin, xmax); + const TAxis axisPhiNew(nBinsPhiNew, axisPhiOrig.GetBinLowEdge(1), axisPhiOrig.GetBinUpEdge(axisPhiOrig.GetNbins())); + const TAxis axisRNew(nBinsRNew, axisROrig.GetBinLowEdge(1), axisROrig.GetBinUpEdge(axisROrig.GetNbins())); + const TAxis axisZNew(nBinsZNewTwo, axisZOrig.GetBinLowEdge(1), axisZOrig.GetBinUpEdge(axisZOrig.GetNbins())); #pragma omp parallel for num_threads(sNThreads) for (int iBinPhi = 1; iBinPhi <= nBinsPhiNew; ++iBinPhi) { - const auto phiLowEdge = hRebin.GetAxis(0)->GetBinLowEdge(iBinPhi); - const auto phiUpEdge = hRebin.GetAxis(0)->GetBinUpEdge(iBinPhi); + const auto phiLowEdge = axisPhiNew.GetBinLowEdge(iBinPhi); + const auto phiUpEdge = axisPhiNew.GetBinUpEdge(iBinPhi); - const int phiLowBinOrig = hOrig.GetXaxis()->FindBin(phiLowEdge); - const int phiUpBinOrig = hOrig.GetXaxis()->FindBin(phiUpEdge); + const int phiLowBinOrig = axisPhiOrig.FindBin(phiLowEdge); + const int phiUpBinOrig = axisPhiOrig.FindBin(phiUpEdge); // calculate the weights (area of original bin lies in the new bin / binwidthOrig) of the first and last bins - const auto binWidthPhiOrig = hOrig.GetXaxis()->GetBinWidth(phiLowBinOrig); - const auto lowerBinWeightPhi = std::abs(phiLowEdge - hOrig.GetXaxis()->GetBinUpEdge(phiLowBinOrig)) / binWidthPhiOrig; - const auto upperBinWeightPhi = std::abs(phiUpEdge - hOrig.GetXaxis()->GetBinLowEdge(phiUpBinOrig)) / binWidthPhiOrig; + const auto binWidthPhiOrig = axisPhiOrig.GetBinWidth(phiLowBinOrig); + const auto lowerBinWeightPhi = std::abs(phiLowEdge - axisPhiOrig.GetBinUpEdge(phiLowBinOrig)) / binWidthPhiOrig; + const auto upperBinWeightPhi = std::abs(phiUpEdge - axisPhiOrig.GetBinLowEdge(phiUpBinOrig)) / binWidthPhiOrig; for (int iBinR = 1; iBinR <= nBinsRNew; ++iBinR) { - const auto rLowEdge = hRebin.GetAxis(1)->GetBinLowEdge(iBinR); - const auto rUpEdge = hRebin.GetAxis(1)->GetBinUpEdge(iBinR); + const auto rLowEdge = axisRNew.GetBinLowEdge(iBinR); + const auto rUpEdge = axisRNew.GetBinUpEdge(iBinR); - const int rLowBinOrig = hOrig.GetYaxis()->FindBin(rLowEdge); - const int rUpBinOrig = hOrig.GetYaxis()->FindBin(rUpEdge); + const int rLowBinOrig = axisROrig.FindBin(rLowEdge); + const int rUpBinOrig = axisROrig.FindBin(rUpEdge); // calculate the weights (area of original bin lies in the new bin / binwidthOrig) of the first and last bins - const auto binWidthROrig = hOrig.GetYaxis()->GetBinWidth(rLowBinOrig); - const auto lowerBinWeightR = std::abs(rLowEdge - hOrig.GetYaxis()->GetBinUpEdge(rLowBinOrig)) / binWidthROrig; - const auto upperBinWeightR = std::abs(rUpEdge - hOrig.GetYaxis()->GetBinLowEdge(rUpBinOrig)) / binWidthROrig; + const auto binWidthROrig = axisROrig.GetBinWidth(rLowBinOrig); + const auto lowerBinWeightR = std::abs(rLowEdge - axisROrig.GetBinUpEdge(rLowBinOrig)) / binWidthROrig; + const auto upperBinWeightR = std::abs(rUpEdge - axisROrig.GetBinLowEdge(rUpBinOrig)) / binWidthROrig; for (int iBinZ = 1; iBinZ <= nBinsZNewTwo; ++iBinZ) { - const auto zLowEdge = hRebin.GetAxis(2)->GetBinLowEdge(iBinZ); - const auto zUpEdge = hRebin.GetAxis(2)->GetBinUpEdge(iBinZ); - const auto zCenter = hRebin.GetAxis(2)->GetBinCenter(iBinZ); + const auto zLowEdge = axisZNew.GetBinLowEdge(iBinZ); + const auto zUpEdge = axisZNew.GetBinUpEdge(iBinZ); + const auto zCenter = axisZNew.GetBinCenter(iBinZ); - int zLowBinOrig = hOrig.GetZaxis()->FindBin(zLowEdge); - int zUpBinOrig = hOrig.GetZaxis()->FindBin(zUpEdge); + int zLowBinOrig = axisZOrig.FindBin(zLowEdge); + int zUpBinOrig = axisZOrig.FindBin(zUpEdge); const int currside = getSide(zCenter); // set the side of the current z-bin // get the side of the lowest and uppest bin from the orig histo - const int sideLowOrig = getSide(hOrig.GetZaxis()->GetBinCenter(zLowBinOrig)); - const int sideUpOrig = getSide(hOrig.GetZaxis()->GetBinCenter(zUpBinOrig)); + const int sideLowOrig = getSide(axisZOrig.GetBinCenter(zLowBinOrig)); + const int sideUpOrig = getSide(axisZOrig.GetBinCenter(zUpBinOrig)); // make bounds/side check of the zLowBinOrig and zUpBinOrig bins. They must be on the same side as the currside!!! if (currside != sideLowOrig && zLowBinOrig != zUpBinOrig) { @@ -1207,7 +1247,7 @@ void SpaceCharge::fillChargeDensityFromHisto(const TH3& hOrig) zLowBinOrig = zUpBinOrig; notequal = false; } - const int sideTmp = getSide(hOrig.GetZaxis()->GetBinCenter(zLowBinOrig)); + const int sideTmp = getSide(axisZOrig.GetBinCenter(zLowBinOrig)); if (sideTmp == currside) { notequal = false; } @@ -1224,16 +1264,16 @@ void SpaceCharge::fillChargeDensityFromHisto(const TH3& hOrig) zUpBinOrig = zLowBinOrig; notequal = false; } - const int sideTmp = getSide(hOrig.GetZaxis()->GetBinCenter(zUpBinOrig)); + const int sideTmp = getSide(axisZOrig.GetBinCenter(zUpBinOrig)); if (sideTmp == currside) { notequal = false; } } while (notequal); } - const auto binWidthZOrig = hOrig.GetZaxis()->GetBinWidth(zLowBinOrig); - const auto lowerBinWeightZ = std::abs(zLowEdge - hOrig.GetZaxis()->GetBinUpEdge(zLowBinOrig)) / binWidthZOrig; - const auto upperBinWeightZ = std::abs(zUpEdge - hOrig.GetZaxis()->GetBinLowEdge(zUpBinOrig)) / binWidthZOrig; + const auto binWidthZOrig = axisZOrig.GetBinWidth(zLowBinOrig); + const auto lowerBinWeightZ = std::abs(zLowEdge - axisZOrig.GetBinUpEdge(zLowBinOrig)) / binWidthZOrig; + const auto upperBinWeightZ = std::abs(zUpEdge - axisZOrig.GetBinLowEdge(zUpBinOrig)) / binWidthZOrig; // get the mean value of the original histogram of the found bin range DataT sum = 0; @@ -1261,7 +1301,7 @@ void SpaceCharge::fillChargeDensityFromHisto(const TH3& hOrig) } else if (iZ == zUpBinOrig) { weightZ = upperBinWeightZ; } - const auto val = hOrig.GetBinContent(iPhi, iR, iZ); + const auto val = getBinContentOrig(iPhi, iR, iZ); // if(val==0){ // what to do now??? // } @@ -1580,11 +1620,13 @@ void SpaceCharge::calcGlobalDistortions(const Fields& formulaStruct, cons // if one uses local distortions the interpolated value for the last bin has to be scaled. // This has to be done because of the interpolated value is defined for a drift length of one z bin, but in the last bin the distance to the readout can be smaller than one z bin. const bool checkReached = side == Side::A ? z1Tmp >= getZMax(side) : z1Tmp <= getZMax(side); + DataT z1Drift = z1Tmp; // end of the drift step without distortion if (formulaStruct.getID() == 2 && checkReached) { const DataT fac = std::abs((getZMax(side) - z0Tmp) * getInvSpacingZ(side)); ddR *= fac; ddZ *= fac; ddPhi *= fac; + z1Drift = getZMax(side); // the local distortions are scaled to the drift up to the readout } // add local distortions to global distortions @@ -1592,13 +1634,13 @@ void SpaceCharge::calcGlobalDistortions(const Fields& formulaStruct, cons dPhiDist += ddPhi; dzDist += ddZ; - // set loop to exit if the readout is reached and approximate distortion of 'missing' (one never ends exactly on the readout: z1Tmp + ddZ != ZMAX) drift distance. - // approximation is done by the current calculated values of the distortions and scaled linear to the 'missing' distance. + // set loop to exit if the readout is reached and approximate distortion of 'missing' (one never ends exactly on the readout: z1Drift + ddZ != ZMAX) drift distance. + // approximation is done by the current calculated values of the distortions and scaled linear to the 'missing' distance (negative if the readout was passed). if (checkReached) { - const DataT endPoint = z1Tmp + ddZ; + const DataT endPoint = z1Drift + ddZ; const DataT deltaZ = getZMax(side) - endPoint; // distance from last point to read out const DataT diff = endPoint - z0Tmp; - const DataT fac = diff != 0 ? std::abs(deltaZ / diff) : 0; // approximate the distortions for the 'missing' distance deltaZ + const DataT fac = diff != 0 ? deltaZ / diff : 0; // approximate the distortions for the 'missing' distance deltaZ drDist += ddR * fac; dPhiDist += ddPhi * fac; dzDist += ddZ * fac; @@ -1613,6 +1655,7 @@ void SpaceCharge::calcGlobalDistortions(const Fields& formulaStruct, cons } } } + warnNonFinite(mGlobalDistdR[side], mGlobalDistdRPhi[side], mGlobalDistdZ[side], "global distortions", side); } template @@ -1729,8 +1772,8 @@ void SpaceCharge::calcGlobalCorrections(const Formulas& formulaStruct, co if (type == 1) { // just replace with last valid number (assumption: values at iR==mParamGrid.NRVertices / 2 are valid) mGlobalCorrdR[side](iZ, iR, iPhi) = mGlobalCorrdR[side](iZ, iRUp, iPhi); - mGlobalCorrdRPhi[side](iZ, iR, iPhi) = mGlobalCorrdR[side](iZ, iRUp, iPhi); - mGlobalCorrdZ[side](iZ, iR, iPhi) = mGlobalCorrdR[side](iZ, iRUp, iPhi); + mGlobalCorrdRPhi[side](iZ, iR, iPhi) = mGlobalCorrdRPhi[side](iZ, iRUp, iPhi); + mGlobalCorrdZ[side](iZ, iR, iPhi) = mGlobalCorrdZ[side](iZ, iRUp, iPhi); } else if (type == 2) { // extrapolate values const size_t iRUpTwo = iR + 2; @@ -1748,8 +1791,8 @@ void SpaceCharge::calcGlobalCorrections(const Formulas& formulaStruct, co if (type == 1) { // just replace with last valid number (assumption: values at iR==mParamGrid.NRVertices / 2 are valid) mGlobalCorrdR[side](iZ, iR, iPhi) = mGlobalCorrdR[side](iZ, iRUp, iPhi); - mGlobalCorrdRPhi[side](iZ, iR, iPhi) = mGlobalCorrdR[side](iZ, iRUp, iPhi); - mGlobalCorrdZ[side](iZ, iR, iPhi) = mGlobalCorrdR[side](iZ, iRUp, iPhi); + mGlobalCorrdRPhi[side](iZ, iR, iPhi) = mGlobalCorrdRPhi[side](iZ, iRUp, iPhi); + mGlobalCorrdZ[side](iZ, iR, iPhi) = mGlobalCorrdZ[side](iZ, iRUp, iPhi); } else if (type == 2) { // extrapolate values const size_t iRUpTwo = iR - 2; @@ -1763,7 +1806,7 @@ void SpaceCharge::calcGlobalCorrections(const Formulas& formulaStruct, co } } } - // set flag that global corrections are set to true + warnNonFinite(mGlobalCorrdR[side], mGlobalCorrdRPhi[side], mGlobalCorrdZ[side], "global corrections", side); auto stop = timer::now(); std::chrono::duration time = stop - start; const float totalTime = time.count(); @@ -1781,8 +1824,8 @@ void SpaceCharge::correctElectron(GlobalPosition3D& point) // get the distortions for input coordinate getCorrections(point.X(), point.Y(), point.Z(), side, corrX, corrY, corrZ); - // set distorted coordinates - point.SetXYZ(point.X() + corrX, point.Y() + corrY, point.Y() + corrY); + // set corrected coordinates + point.SetXYZ(point.X() + corrX, point.Y() + corrY, point.Z() + corrZ); } template @@ -1875,17 +1918,13 @@ std::vector SpaceCharge::getPotentialCyl(const std::vector& template void SpaceCharge::getElectricFieldsCyl(const DataT z, const DataT r, const DataT phi, const Side side, DataT& eZ, DataT& eR, DataT& ePhi) const { - eZ = mInterpolatorEField[side].evalFieldZ(z, r, phi); - eR = mInterpolatorEField[side].evalFieldR(z, r, phi); - ePhi = mInterpolatorEField[side].evalFieldPhi(z, r, phi); + mInterpolatorEField[side].evalFields(z, r, phi, eZ, eR, ePhi); } template void SpaceCharge::getLocalCorrectionsCyl(const DataT z, const DataT r, const DataT phi, const Side side, DataT& lcorrZ, DataT& lcorrR, DataT& lcorrRPhi) const { - lcorrZ = mInterpolatorLocalCorr[side].evaldZ(z, r, phi); - lcorrR = mInterpolatorLocalCorr[side].evaldR(z, r, phi); - lcorrRPhi = mInterpolatorLocalCorr[side].evaldRPhi(z, r, phi); + mInterpolatorLocalCorr[side].evalDistCorr(z, r, phi, lcorrZ, lcorrR, lcorrRPhi); } template @@ -1904,9 +1943,7 @@ void SpaceCharge::getLocalCorrectionsCyl(const std::vector& z, con template void SpaceCharge::getCorrectionsCyl(const DataT z, const DataT r, const DataT phi, const Side side, DataT& corrZ, DataT& corrR, DataT& corrRPhi) const { - corrZ = mInterpolatorGlobalCorr[side].evaldZ(z, r, phi); - corrR = mInterpolatorGlobalCorr[side].evaldR(z, r, phi); - corrRPhi = mInterpolatorGlobalCorr[side].evaldRPhi(z, r, phi); + mInterpolatorGlobalCorr[side].evalDistCorr(z, r, phi, corrZ, corrR, corrRPhi); } template @@ -1949,9 +1986,7 @@ void SpaceCharge::getCorrections(const DataT x, const DataT y, const Data template void SpaceCharge::getLocalDistortionsCyl(const DataT z, const DataT r, const DataT phi, const Side side, DataT& ldistZ, DataT& ldistR, DataT& ldistRPhi) const { - ldistZ = mInterpolatorLocalDist[side].evaldZ(z, r, phi); - ldistR = mInterpolatorLocalDist[side].evaldR(z, r, phi); - ldistRPhi = mInterpolatorLocalDist[side].evaldRPhi(z, r, phi); + mInterpolatorLocalDist[side].evalDistCorr(z, r, phi, ldistZ, ldistR, ldistRPhi); } template @@ -1970,9 +2005,7 @@ void SpaceCharge::getLocalDistortionsCyl(const std::vector& z, con template void SpaceCharge::getLocalDistortionVectorCyl(const DataT z, const DataT r, const DataT phi, const Side side, DataT& lvecdistZ, DataT& lvecdistR, DataT& lvecdistRPhi) const { - lvecdistZ = mInterpolatorLocalVecDist[side].evaldZ(z, r, phi); - lvecdistR = mInterpolatorLocalVecDist[side].evaldR(z, r, phi); - lvecdistRPhi = mInterpolatorLocalVecDist[side].evaldRPhi(z, r, phi); + mInterpolatorLocalVecDist[side].evalDistCorr(z, r, phi, lvecdistZ, lvecdistR, lvecdistRPhi); } template @@ -1991,9 +2024,10 @@ void SpaceCharge::getLocalDistortionVectorCyl(const std::vector& z template void SpaceCharge::getLocalCorrectionVectorCyl(const DataT z, const DataT r, const DataT phi, const Side side, DataT& lveccorrZ, DataT& lveccorrR, DataT& lveccorrRPhi) const { - lveccorrZ = -mInterpolatorLocalVecDist[side].evaldZ(z, r, phi); - lveccorrR = -mInterpolatorLocalVecDist[side].evaldR(z, r, phi); - lveccorrRPhi = -mInterpolatorLocalVecDist[side].evaldRPhi(z, r, phi); + mInterpolatorLocalVecDist[side].evalDistCorr(z, r, phi, lveccorrZ, lveccorrR, lveccorrRPhi); + lveccorrZ = -lveccorrZ; + lveccorrR = -lveccorrR; + lveccorrRPhi = -lveccorrRPhi; } template @@ -2012,9 +2046,7 @@ void SpaceCharge::getLocalCorrectionVectorCyl(const std::vector& z template void SpaceCharge::getDistortionsCyl(const DataT z, const DataT r, const DataT phi, const Side side, DataT& distZ, DataT& distR, DataT& distRPhi) const { - distZ = mInterpolatorGlobalDist[side].evaldZ(z, r, phi); - distR = mInterpolatorGlobalDist[side].evaldR(z, r, phi); - distRPhi = mInterpolatorGlobalDist[side].evaldRPhi(z, r, phi); + mInterpolatorGlobalDist[side].evalDistCorr(z, r, phi, distZ, distR, distRPhi); } template @@ -2156,13 +2188,11 @@ template void SpaceCharge::integrateEFieldsTrapezoidal(const DataT p1r, const DataT p1phi, const DataT p1z, const DataT p2z, DataT& localIntErOverEz, DataT& localIntEPhiOverEz, DataT& localIntDeltaEz, const Fields& formulaStruct, const DataT ezField, const Side side) const { //========trapezoidal rule see: https://en.wikipedia.org/wiki/Trapezoidal_rule ============== - const DataT fielder0 = formulaStruct.evalFieldR(p1z, p1r, p1phi); - const DataT fieldez0 = formulaStruct.evalFieldZ(p1z, p1r, p1phi); - const DataT fieldephi0 = formulaStruct.evalFieldPhi(p1z, p1r, p1phi); + DataT fieldez0, fielder0, fieldephi0; + formulaStruct.evalFields(p1z, p1r, p1phi, fieldez0, fielder0, fieldephi0); - const DataT fielder1 = formulaStruct.evalFieldR(p2z, p1r, p1phi); - const DataT fieldez1 = formulaStruct.evalFieldZ(p2z, p1r, p1phi); - const DataT fieldephi1 = formulaStruct.evalFieldPhi(p2z, p1r, p1phi); + DataT fieldez1, fielder1, fieldephi1; + formulaStruct.evalFields(p2z, p1r, p1phi, fieldez1, fielder1, fieldephi1); const DataT eZ0 = isCloseToZero(ezField, fieldez0) ? 0 : 1. / (ezField + fieldez0); const DataT eZ1 = isCloseToZero(ezField, fieldez1) ? 0 : 1. / (ezField + fieldez1); @@ -2178,20 +2208,19 @@ template void SpaceCharge::integrateEFieldsSimpson(const DataT p1r, const DataT p1phi, const DataT p1z, const DataT p2z, DataT& localIntErOverEz, DataT& localIntEPhiOverEz, DataT& localIntDeltaEz, const Fields& formulaStruct, const DataT ezField, const Side side) const { //==========simpsons rule see: https://en.wikipedia.org/wiki/Simpson%27s_rule ============================= - const DataT fielder0 = formulaStruct.evalFieldR(p1z, p1r, p1phi); - const DataT fieldez0 = formulaStruct.evalFieldZ(p1z, p1r, p1phi); - const DataT fieldephi0 = formulaStruct.evalFieldPhi(p1z, p1r, p1phi); + DataT fieldez0, fielder0, fieldephi0; + formulaStruct.evalFields(p1z, p1r, p1phi, fieldez0, fielder0, fieldephi0); - const DataT fielder1 = formulaStruct.evalFieldR(p2z, p1r, p1phi); - const DataT fieldez1 = formulaStruct.evalFieldZ(p2z, p1r, p1phi); - const DataT fieldephi1 = formulaStruct.evalFieldPhi(p2z, p1r, p1phi); + DataT fieldez1, fielder1, fieldephi1; + formulaStruct.evalFields(p2z, p1r, p1phi, fieldez1, fielder1, fieldephi1); const DataT deltaX = p2z - p1z; const DataT xk2N = (p2z - static_cast(0.5) * deltaX); - const DataT ezField2 = formulaStruct.evalFieldZ(xk2N, p1r, p1phi); + DataT ezField2, erField2, ephiField2; + formulaStruct.evalFields(xk2N, p1r, p1phi, ezField2, erField2, ephiField2); const DataT ezField2Denominator = isCloseToZero(ezField, ezField2) ? 0 : 1. / (ezField + ezField2); - const DataT fieldSum2ErOverEz = formulaStruct.evalFieldR(xk2N, p1r, p1phi) * ezField2Denominator; - const DataT fieldSum2EphiOverEz = formulaStruct.evalFieldPhi(xk2N, p1r, p1phi) * ezField2Denominator; + const DataT fieldSum2ErOverEz = erField2 * ezField2Denominator; + const DataT fieldSum2EphiOverEz = ephiField2 * ezField2Denominator; const DataT eZ0 = isCloseToZero(ezField, fieldez0) ? 0 : 1. / (ezField + fieldez0); const DataT eZ1 = isCloseToZero(ezField, fieldez1) ? 0 : 1. / (ezField + fieldez1); @@ -2204,20 +2233,20 @@ void SpaceCharge::integrateEFieldsSimpson(const DataT p1r, const DataT p1 template template -void SpaceCharge::integrateEFieldsSimpsonIterative(const DataT p1r, const DataT p2r, const DataT p1phi, const DataT p2phi, const DataT p1z, const DataT p2z, DataT& localIntErOverEz, DataT& localIntEPhiOverEz, DataT& localIntDeltaEz, const Fields& formulaStruct, const DataT ezField, const Side side) const +void SpaceCharge::integrateEFieldsSimpsonIterative(const DataT p1r, const DataT p2r, const DataT p1phi, const DataT p2phi, const DataT p1z, const DataT p2z, const DataT (&fieldsStart)[3], DataT& localIntErOverEz, DataT& localIntEPhiOverEz, DataT& localIntDeltaEz, const Fields& formulaStruct, const DataT ezField, const Side side) const { //==========simpsons rule see: https://en.wikipedia.org/wiki/Simpson%27s_rule ============================= // const Side side = formulaStruct.getSide(); // const DataT ezField = getEzField(side); const DataT p2phiSave = regulatePhi(p2phi, side); - const DataT fielder0 = formulaStruct.evalFieldR(p1z, p1r, p1phi); - const DataT fieldez0 = formulaStruct.evalFieldZ(p1z, p1r, p1phi); - const DataT fieldephi0 = formulaStruct.evalFieldPhi(p1z, p1r, p1phi); + // fields at the start point (the same for all iterations, evaluated by the caller) + const DataT fieldez0 = fieldsStart[0]; + const DataT fielder0 = fieldsStart[1]; + const DataT fieldephi0 = fieldsStart[2]; - const DataT fielder1 = formulaStruct.evalFieldR(p2z, p2r, p2phiSave); - const DataT fieldez1 = formulaStruct.evalFieldZ(p2z, p2r, p2phiSave); - const DataT fieldephi1 = formulaStruct.evalFieldPhi(p2z, p2r, p2phiSave); + DataT fieldez1, fielder1, fieldephi1; + formulaStruct.evalFields(p2z, p2r, p2phiSave, fieldez1, fielder1, fieldephi1); const DataT eZ0Inv = isCloseToZero(ezField, fieldez0) ? 0 : 1. / (ezField + fieldez0); const DataT eZ1Inv = isCloseToZero(ezField, fieldez1) ? 0 : 1. / (ezField + fieldez1); @@ -2226,10 +2255,9 @@ void SpaceCharge::integrateEFieldsSimpsonIterative(const DataT p1r, const const DataT pHalfPhiSave = regulatePhi(0.5 * (p1phi + p2phi), side); // needs to be regulated since p2phi is not regulated const DataT pHalfR = 0.5 * (p1r + p2r); - const DataT ezField2 = formulaStruct.evalFieldZ(pHalfZ, pHalfR, pHalfPhiSave); + DataT ezField2, fieldSum2ErOverEz, fieldSum2EphiOverEz; + formulaStruct.evalFields(pHalfZ, pHalfR, pHalfPhiSave, ezField2, fieldSum2ErOverEz, fieldSum2EphiOverEz); const DataT eZHalfInv = (isCloseToZero(ezField, ezField2) | isCloseToZero(ezField, fieldez0) | isCloseToZero(ezField, fieldez1)) ? 0 : 1. / (ezField + ezField2); - const DataT fieldSum2ErOverEz = formulaStruct.evalFieldR(pHalfZ, pHalfR, pHalfPhiSave); - const DataT fieldSum2EphiOverEz = formulaStruct.evalFieldPhi(pHalfZ, pHalfR, pHalfPhiSave); const DataT deltaXSimpsonSixth = (p2z - p1z) / 6; localIntErOverEz = deltaXSimpsonSixth * (4 * fieldSum2ErOverEz * eZHalfInv + fielder0 * eZ0Inv + fielder1 * eZ1Inv); @@ -2580,19 +2608,29 @@ void SpaceCharge::calcDistCorr(const DataT p1r, const DataT p1phi, const // there are differentnumerical integration strategys implements. for details see each function. switch (sNumericalIntegrationStrategy) { - case IntegrationStrategy::SimpsonIterative: // iterative simpson integration (should be more precise at least for the analytical E-Field case but takes alot more time than normal simpson integration) + case IntegrationStrategy::SimpsonIterative: { // iterative simpson integration (should be more precise at least for the analytical E-Field case but takes alot more time than normal simpson integration) + // the fields at the start point do not change between the iterations + DataT fieldsStartE[3]{}; + DataT fieldsStartB[3]{}; + if (mSimEDistortions) { + formulaStruct.evalFields(p1z, p1r, p1phi, fieldsStartE[0], fieldsStartE[1], fieldsStartE[2]); + } + if (mSimExBMisalignment) { + mBField.evalFields(p1z, p1r, p1phi, fieldsStartB[0], fieldsStartB[1], fieldsStartB[2]); + } for (int i = 0; i < sSimpsonNIteratives; ++i) { // TODO define a convergence criterion to abort the algorithm earlier for speed up. const DataT tmpZ = localDistCorr ? (p2z + ddZ) : regulateZ(p2z + ddZ, formulaStruct.getSide()); // dont regulate for local distortions/corrections! (to get same result as using electric field at last/first bin) if (mSimEDistortions) { - integrateEFieldsSimpsonIterative(p1r, p1r + ddR + ddRExB, p1phi, p1phi + ddPhi + ddPhiExB, p1z, tmpZ, localIntErOverEz, localIntEPhiOverEz, localIntDeltaEz, formulaStruct, getEzField(side), side); + integrateEFieldsSimpsonIterative(p1r, p1r + ddR + ddRExB, p1phi, p1phi + ddPhi + ddPhiExB, p1z, tmpZ, fieldsStartE, localIntErOverEz, localIntEPhiOverEz, localIntDeltaEz, formulaStruct, getEzField(side), side); langevinCylindricalE(ddR, ddPhi, ddZ, (p1r + 0.5 * (ddR + ddRExB)), localIntErOverEz, localIntEPhiOverEz, localIntDeltaEz); // using the mean radius '(p1r + 0.5 * ddR)' for calculation of distortions/corections } if (mSimExBMisalignment) { - integrateEFieldsSimpsonIterative(p1r, p1r + ddR + ddRExB, p1phi, p1phi + ddPhi + ddPhiExB, p1z, tmpZ, localIntBrOverBz, localIntBPhiOverBz, localIntDeltaBz, mBField, 0, side); + integrateEFieldsSimpsonIterative(p1r, p1r + ddR + ddRExB, p1phi, p1phi + ddPhi + ddPhiExB, p1z, tmpZ, fieldsStartB, localIntBrOverBz, localIntBPhiOverBz, localIntDeltaBz, mBField, 0, side); langevinCylindricalB(ddRExB, ddPhiExB, (p1r + 0.5 * (ddR + ddRExB)), localIntBrOverBz, localIntBPhiOverBz); } } break; + } case IntegrationStrategy::Simpson: // simpson integration if (mSimEDistortions) { integrateEFieldsSimpson(p1r, p1phi, p1z, p2z, localIntErOverEz, localIntEPhiOverEz, localIntDeltaEz, formulaStruct, getEzField(side), side); @@ -2815,36 +2853,26 @@ void SpaceCharge::fillChargeDensityFromHisto(const char* file, const char if (!hSCC) { LOGP(error, "Histogram {} not found", nameC); } + if (!hSCA || !hSCC) { + return; + } fillChargeDensityFromHisto(*hSCA, *hSCC); } template void SpaceCharge::fillChargeDensityFromHisto(const TH3& hisSCDensity3D_A, const TH3& hisSCDensity3D_C) { - const int nPhiBinsTmp = hisSCDensity3D_A.GetXaxis()->GetNbins(); - const int nRBinsTmp = hisSCDensity3D_A.GetYaxis()->GetNbins(); + // the C-side histogram is mapped to z in [-zMax, 0] and the A-side histogram to z in [0, zMax] (z bin number of the C-side + // histogram = z bin number of the merged axis). The rebinning reads both histograms directly without creating a merged copy. const int nZBins = hisSCDensity3D_A.GetZaxis()->GetNbins(); - const auto phiLow = hisSCDensity3D_A.GetXaxis()->GetBinLowEdge(1); - const auto phiUp = hisSCDensity3D_A.GetXaxis()->GetBinUpEdge(nPhiBinsTmp); - const auto rLow = hisSCDensity3D_A.GetYaxis()->GetBinLowEdge(1); - const auto rUp = hisSCDensity3D_A.GetYaxis()->GetBinUpEdge(nRBinsTmp); const auto zUp = hisSCDensity3D_A.GetZaxis()->GetBinUpEdge(nZBins); - - TH3F hisSCMerged("hisMerged", "hisMerged", nPhiBinsTmp, phiLow, phiUp, nRBinsTmp, rLow, rUp, 2 * nZBins, -zUp, zUp); - - for (int iside = 0; iside < FNSIDES; ++iside) { - const auto& hSC = (iside == 0) ? hisSCDensity3D_A : hisSCDensity3D_C; -#pragma omp parallel for num_threads(sNThreads) - for (int iz = 1; iz <= nZBins; ++iz) { - const int izTmp = (iside == 0) ? (nZBins + iz) : iz; - for (int ir = 1; ir <= nRBinsTmp; ++ir) { - for (int iphi = 1; iphi <= nPhiBinsTmp; ++iphi) { - hisSCMerged.SetBinContent(iphi, ir, izTmp, hSC.GetBinContent(iphi, ir, iz)); - } - } - } - } - fillChargeDensityFromHisto(hisSCMerged); + const TAxis axisZMerged(2 * nZBins, -zUp, zUp); + withBinContentAccessor(hisSCDensity3D_A, [&](const auto& getA) { + withBinContentAccessor(hisSCDensity3D_C, [&](const auto& getC) { + const auto getBinContent = [&getA, &getC, nZBins](int iPhi, int iR, int iZ) { return (iZ > nZBins) ? getA(iPhi, iR, iZ - nZBins) : getC(iPhi, iR, iZ); }; + fillChargeDensityFromHisto(*hisSCDensity3D_A.GetXaxis(), *hisSCDensity3D_A.GetYaxis(), axisZMerged, getBinContent); + }); + }); } template @@ -3366,7 +3394,7 @@ void SpaceCharge::scaleChargeDensityStack(const float scalingFactor, cons for (unsigned int iR = 0; iR < mParamGrid.NRVertices; ++iR) { const DataT radius = getRVertex(iR, side); for (unsigned int iPhi = iPhiFirst; iPhi < iPhiLast; ++iPhi) { - const DataT phi = getPhiVertex(iR, side); + const DataT phi = getPhiVertex(iPhi, side); const GlobalPosition3D pos(getXFromPolar(radius, phi), getYFromPolar(radius, phi), ((side == Side::A) ? 10 : -10)); const auto& mapper = o2::tpc::Mapper::instance(); const o2::tpc::DigitPos digiPadPos = mapper.findDigitPosFromGlobalPosition(pos); diff --git a/Detectors/TPC/spacecharge/src/SpaceChargeIO.cxx b/Detectors/TPC/spacecharge/src/SpaceChargeIO.cxx index f7750e7d89ac6..17541c60860b8 100644 --- a/Detectors/TPC/spacecharge/src/SpaceChargeIO.cxx +++ b/Detectors/TPC/spacecharge/src/SpaceChargeIO.cxx @@ -28,6 +28,8 @@ #include "fmt/core.h" #include "TFile.h" #include "TTree.h" +#include "TSystem.h" +#include "TString.h" #include "TStopwatch.h" #include "ROOT/RDataFrame.hxx" #include @@ -95,6 +97,15 @@ int DataContainer3D::writeToFile(std::string_view file, std::string_view ROOT::RDF::RSnapshotOptions opt; opt.fMode = option; opt.fOverwriteIfExists = true; // overwrite if already exists + // ZSTD: same file size as the default ZLIB (the data are not well compressible) but ~3 times faster reading + opt.fCompressionAlgorithm = ROOT::RCompressionSetting::EAlgorithm::kZSTD; + opt.fCompressionLevel = 1; + TString mode(option.data()); + mode.ToUpper(); + if ((mode == "UPDATE") && gSystem->AccessPathName(file.data())) { + // a file opened in UPDATE mode keeps its compression setting: create a missing file with the same compression as for RECREATE + TFile fNew(file.data(), "RECREATE", "", ROOT::CompressionSettings(opt.fCompressionAlgorithm, opt.fCompressionLevel)); + } TStopwatch timer; // note: first call has some overhead (~2s) diff --git a/Detectors/TPC/spacecharge/src/TPCSpacechargeLinkDef.h b/Detectors/TPC/spacecharge/src/TPCSpacechargeLinkDef.h index b1bb33dc1aeb7..c27d4fb6e7a08 100644 --- a/Detectors/TPC/spacecharge/src/TPCSpacechargeLinkDef.h +++ b/Detectors/TPC/spacecharge/src/TPCSpacechargeLinkDef.h @@ -30,6 +30,8 @@ #pragma link C++ class o2::tpc::DataContainer3D < double> + ; #pragma link C++ class o2::tpc::PoissonSolver < double> + ; #pragma link C++ class o2::tpc::SpaceCharge < double> + ; +// set up the transient grid properties from the number of vertices read from the file +#pragma read sourceClass = "o2::tpc::SpaceCharge" targetClass = "o2::tpc::SpaceCharge" version = "[1-]" source = "" target = "" code = "{ newObj->initAfterReadingFromFile(); }" #pragma link C++ class o2::tpc::NumericalFields < double> + ; #pragma link C++ class o2::tpc::DistCorrInterpolator < double> + ; #pragma link C++ class o2::tpc::GridProperties < double> + ; @@ -41,6 +43,8 @@ #pragma link C++ class o2::tpc::DataContainer3D < float> + ; #pragma link C++ class o2::tpc::PoissonSolver < float> + ; #pragma link C++ class o2::tpc::SpaceCharge < float> + ; +// set up the transient grid properties from the number of vertices read from the file +#pragma read sourceClass = "o2::tpc::SpaceCharge" targetClass = "o2::tpc::SpaceCharge" version = "[1-]" source = "" target = "" code = "{ newObj->initAfterReadingFromFile(); }" #pragma link C++ class o2::tpc::NumericalFields < float> + ; #pragma link C++ class o2::tpc::DistCorrInterpolator < float> + ; #pragma link C++ class o2::tpc::GridProperties < float> + ; diff --git a/Detectors/TPC/spacecharge/src/TriCubic.cxx b/Detectors/TPC/spacecharge/src/TriCubic.cxx index a58ba0cc74f2f..3cef47eba60ba 100644 --- a/Detectors/TPC/spacecharge/src/TriCubic.cxx +++ b/Detectors/TPC/spacecharge/src/TriCubic.cxx @@ -16,7 +16,9 @@ #include "TPCSpaceCharge/TriCubic.h" #include "TPCSpaceCharge/DataContainer3D.h" -#include "TPCSpaceCharge/Vector.h" +#include +#include +#include using namespace o2::tpc; @@ -26,6 +28,7 @@ TriCubicInterpolator::TriCubicInterpolator(TriCubicInterpolator&& { if (this != &other) { mExtrapolationType = other.mExtrapolationType; + mExtraPolateValues = other.mExtraPolateValues; // do not change address of old pointers! other.mGridData = nullptr; other.mGridProperties = nullptr; @@ -37,6 +40,7 @@ TriCubicInterpolator& TriCubicInterpolator::operator=(TriCubicInte { if (this != &other) { mExtrapolationType = other.mExtrapolationType; + mExtraPolateValues = other.mExtraPolateValues; // do not change address of old pointers! other.mGridData = nullptr; other.mGridProperties = nullptr; @@ -72,1009 +76,214 @@ DataT TriCubicInterpolator::parabolExtrapolation(const DataT valk, const return val; } +namespace +{ +/// calculate the 4 weights of the 1D cubic (Catmull-Rom) interpolation for the relative position t in the cell template -DataT TriCubicInterpolator::interpolateSparse(const DataT z, const DataT r, const DataT phi) const +inline void getWeights(const DataT t, DataT (&weights)[4]) { - // check if data is empty - if (!mGridData->getNDataPoints()) { - return 0; - } - const Vector coordinates{{z, r, phi}}; // vector holding the coordinates - Vector posRel{(coordinates - mGridProperties->getGridMin()) * mGridProperties->getInvSpacing()}; // needed for the grid index - posRel[FPHI] = mGridProperties->clampToGridCircularRel(posRel[FPHI], FPHI); - Vector posRelN{posRel}; - posRel[FZ] = mGridProperties->clampToGridRel(posRel[FZ], FZ); - posRel[FR] = mGridProperties->clampToGridRel(posRel[FR], FR); - if (!mExtraPolateValues) { - posRelN[FZ] = posRel[FZ]; - posRelN[FR] = posRel[FR]; - } - - const int nPoints = 4; - std::array, 16> cVals; - const Vector index{floor_vec(posRel)}; - setValues(index[FZ], index[FR], index[FPHI], cVals); - - const Vector vals0{posRelN - index}; - const Vector vals1{vals0 * vals0}; - const Vector vals2{vals0 * vals1}; - - const Vector vecValX{{1, vals0[FZ], vals1[FZ], vals2[FZ]}}; - const Vector vecValY{{1, vals0[FR], vals1[FR], vals2[FR]}}; - const Vector vecValZ{{1, vals0[FPHI], vals1[FPHI], vals2[FPHI]}}; - - const static std::array, nPoints>, nPoints> matrixA{{{0, -0.5, 1, -0.5}, - {1, 0, -2.5, 1.5}, - {0, 0.5, 2., -1.5}, - {0, 0, -0.5, 0.5}}}; - - const Vector vecValXMult{matrixA * vecValX}; - const Vector vecValYMult{matrixA * vecValY}; - const Vector vecValZMult{matrixA * vecValZ}; + const DataT t2 = t * t; + const DataT t3 = t2 * t; + weights[0] = DataT(0.5) * (-t + 2 * t2 - t3); + weights[1] = DataT(0.5) * (2 - 5 * t2 + 3 * t3); + weights[2] = DataT(0.5) * (t + 4 * t2 - 3 * t3); + weights[3] = DataT(0.5) * (t3 - t2); +} - DataT result{}; - int ind = 0; - for (int slice = 0; slice < nPoints; ++slice) { - const Vector vecA{vecValZMult[slice] * vecValYMult}; - for (int row = 0; row < nPoints; ++row) { - result += sum(vecA[row] * vecValXMult * cVals[ind++]); +/// sum_{slice, row, iz} weightsPhi[slice] * weightsR[row] * weightsZ[iz] * vals[sliceOffset[slice] + row * rowStride + iz] +/// \param vals values of the grid +/// \param sliceOffset offset of the 4 phi slices +/// \param rowStride offset between the rows in r direction (the 4 values in z direction are consecutive) +template +inline DataT interpolateCell(const DataT* vals, const int (&sliceOffset)[4], const int rowStride, const DataT (&weightsZ)[4], const DataT (&weightsR)[4], const DataT (&weightsPhi)[4]) +{ + DataT result = 0; + for (int slice = 0; slice < 4; ++slice) { + DataT sumR = 0; + for (int row = 0; row < 4; ++row) { + const DataT* val = vals + sliceOffset[slice] + row * rowStride; + sumR += weightsR[row] * (weightsZ[0] * val[0] + weightsZ[1] * val[1] + weightsZ[2] * val[2] + weightsZ[3] * val[3]); } + result += weightsPhi[slice] * sumR; } return result; } -// for perdiodic boundary condition +/// interpolateCell() for three data sets on the same grid in one pass (same order of the operations as interpolateCell() for each data set) template -void TriCubicInterpolator::getDataIndexCircularArray(const int index0, const int dim, int arr[]) const +inline void interpolateCell3(const DataT* vals0, const DataT* vals1, const DataT* vals2, const int (&sliceOffset)[4], const int rowStride, const DataT (&weightsZ)[4], const DataT (&weightsR)[4], const DataT (&weightsPhi)[4], DataT& result0, DataT& result1, DataT& result2) { - const int delta_min1 = getRegulatedDelta(index0, -1, dim, mGridProperties->getN(dim) - 1); - const int delta_plus1 = getRegulatedDelta(index0, +1, dim, 1 - mGridProperties->getN(dim)); - const int delta_plus2 = getRegulatedDelta(index0, +2, dim, 2 - mGridProperties->getN(dim)); - - arr[0] = mGridProperties->getDeltaDataIndex(delta_min1, dim); - arr[1] = mGridProperties->getDeltaDataIndex(delta_plus1, dim); - arr[2] = mGridProperties->getDeltaDataIndex(delta_plus2, dim); + result0 = 0; + result1 = 0; + result2 = 0; + for (int slice = 0; slice < 4; ++slice) { + DataT sumR0 = 0; + DataT sumR1 = 0; + DataT sumR2 = 0; + for (int row = 0; row < 4; ++row) { + const int offset = sliceOffset[slice] + row * rowStride; + const DataT* val0 = vals0 + offset; + const DataT* val1 = vals1 + offset; + const DataT* val2 = vals2 + offset; + sumR0 += weightsR[row] * (weightsZ[0] * val0[0] + weightsZ[1] * val0[1] + weightsZ[2] * val0[2] + weightsZ[3] * val0[3]); + sumR1 += weightsR[row] * (weightsZ[0] * val1[0] + weightsZ[1] * val1[1] + weightsZ[2] * val1[2] + weightsZ[3] * val1[3]); + sumR2 += weightsR[row] * (weightsZ[0] * val2[0] + weightsZ[1] * val2[1] + weightsZ[2] * val2[2] + weightsZ[3] * val2[3]); + } + result0 += weightsPhi[slice] * sumR0; + result1 += weightsPhi[slice] * sumR1; + result2 += weightsPhi[slice] * sumR2; + } } +} // namespace template -typename TriCubicInterpolator::GridPos TriCubicInterpolator::findPos(const int iz, const int ir, const int iphi) const +TriCubicCell TriCubicInterpolator::getCell(const DataT z, const DataT r, const DataT phi) const { - GridPos pos = GridPos::None; - if (isInInnerVolume(iz, ir, iphi, pos)) { - return pos; + TriCubicCell cell; + + // relative position in the grid (in units of the grid spacing) + DataT posRel[FDim]{(z - mGridProperties->getGridMinZ()) * mGridProperties->getInvSpacingZ(), (r - mGridProperties->getGridMinR()) * mGridProperties->getInvSpacingR(), (phi - mGridProperties->getGridMinPhi()) * mGridProperties->getInvSpacingPhi()}; + if (!std::isfinite(posRel[FZ]) || !std::isfinite(posRel[FR]) || !std::isfinite(posRel[FPHI])) { + // query position is not finite: use a valid cell to avoid an access outside of the grid, the interpolated value is NaN + const DataT nan = std::numeric_limits::quiet_NaN(); + getWeights(nan, cell.weightsZ); + getWeights(nan, cell.weightsR); + getWeights(nan, cell.weightsPhi); + return cell; } - - if (findEdge(iz, ir, iphi, pos)) { - return pos; - } - - if (findLine(iz, ir, iphi, pos)) { - return pos; + posRel[FPHI] = mGridProperties->clampToGridCircularRel(posRel[FPHI], FPHI); + DataT posRelN[FDim]{posRel[FZ], posRel[FR], posRel[FPHI]}; + posRel[FZ] = mGridProperties->clampToGridRel(posRel[FZ], FZ); + posRel[FR] = mGridProperties->clampToGridRel(posRel[FR], FR); + if (!mExtraPolateValues) { + // restrict the query position to the grid. The index of the cell is at most the second last vertex (see clampToGridRel()), but the position has to be clamped to the last vertex + posRelN[FZ] = std::min(std::max(posRelN[FZ], DataT(0)), mGridProperties->getMaxIndexZ()); + posRelN[FR] = std::min(std::max(posRelN[FR], DataT(0)), mGridProperties->getMaxIndexR()); } - if (findSide(iz, ir, iphi, pos)) { - return pos; + // index of the cell: the relative positions are >= 0 after the clamping, so the conversion to int is equal to floor() + cell.indexZ = static_cast(posRel[FZ]); + cell.indexR = static_cast(posRel[FR]); + cell.indexPhi = static_cast(posRel[FPHI]); + + // weights of the interpolation in each dimension + getWeights(posRelN[FZ] - cell.indexZ, cell.weightsZ); + getWeights(posRelN[FR] - cell.indexR, cell.weightsR); + getWeights(posRelN[FPHI] - cell.indexPhi, cell.weightsPhi); + + // all 64 points used for the interpolation are on the grid if the cell does not touch the first or last vertex in z and r (the phi direction is periodic) + cell.inGrid = (cell.indexZ >= 1) && (cell.indexZ <= static_cast(mGridProperties->getN(FZ)) - 3) && (cell.indexR >= 1) && (cell.indexR <= static_cast(mGridProperties->getN(FR)) - 3); + if (cell.inGrid) { + // all 64 values are inside the grid (the phi direction is periodic): the values are directly taken from the grid. + // The values in z direction are consecutive in memory (DataContainer3D::getDataIndex()): start at (iz - 1, ir - 1, iphi) + int deltaPhi[3]{}; + getDataIndexCircularArray(cell.indexPhi, FPHI, deltaPhi); + cell.sliceOffset[0] = deltaPhi[0]; + cell.sliceOffset[1] = 0; + cell.sliceOffset[2] = deltaPhi[1]; + cell.sliceOffset[3] = deltaPhi[2]; + cell.rowStride = mGridProperties->getDeltaDataIndex(1, FR); + cell.dataIndex = mGridData->getDataIndex(cell.indexZ, cell.indexR, cell.indexPhi) - cell.rowStride - 1; } - return GridPos::None; + return cell; } template -bool TriCubicInterpolator::findEdge(const int iz, const int ir, const int iphi, GridPos& posType) const +DataT TriCubicInterpolator::interpolateData(const DataContainer& data, const TriCubicCell& cell) const { - const int iR = 2; - if (iz == 0 && ir == 0) { - if (iphi == 0) { - posType = GridPos::Edge0; - return true; - } else if (iphi == mGridData->getNPhi() - iR) { - posType = GridPos::Edge4; - return true; - } - } else if (iz == mGridData->getNZ() - iR && ir == 0) { - if (iphi == 0) { - posType = GridPos::Edge1; - return true; - } else if (iphi == mGridData->getNPhi() - iR) { - posType = GridPos::Edge5; - return true; - } - } else if (iz == 0 && ir == mGridData->getNR() - iR) { - if (iphi == 0) { - posType = GridPos::Edge2; - return true; - } else if (iphi == mGridData->getNPhi() - iR) { - posType = GridPos::Edge6; - return true; - } - } else if (iz == mGridData->getNZ() - iR && ir == mGridData->getNR() - iR) { - if (iphi == 0) { - posType = GridPos::Edge3; - return true; - } else if (iphi == mGridData->getNPhi() - iR) { - posType = GridPos::Edge7; - return true; - } + // check if data is empty + if (!data.getNDataPoints()) { + return 0; } - return false; -} -template -bool TriCubicInterpolator::findLine(const int iz, const int ir, const int iphi, GridPos& posType) const -{ - const int iR = 2; - // check line - if (ir == 0) { - if (iphi == 0) { - posType = GridPos::LineA; - return true; - } else if (iphi == mGridData->getNPhi() - iR) { - posType = GridPos::LineE; - return true; - } - if (iz == 0) { - posType = GridPos::LineI; - return true; - } else if (iz == mGridData->getNZ() - iR) { - posType = GridPos::LineJ; - return true; - } - } else if (ir == mGridData->getNR() - iR) { - if (iphi == 0) { - posType = GridPos::LineB; - return true; - } else if (iphi == mGridData->getNPhi() - iR) { - posType = GridPos::LineF; - return true; - } - if (iz == 0) { - posType = GridPos::LineK; - return true; - } else if (iz == mGridData->getNZ() - iR) { - posType = GridPos::LineL; - return true; - } - } else if (iz == 0) { - if (iphi == 0) { - posType = GridPos::LineC; - return true; - } else if (iphi == mGridData->getNPhi() - iR) { - posType = GridPos::LineG; - return true; - } - } else if (iz == mGridData->getNZ() - iR) { - if (iphi == 0) { - posType = GridPos::LineD; - return true; - } else if (iphi == mGridData->getNPhi() - iR) { - posType = GridPos::LineH; - return true; - } + if (cell.inGrid) { + return interpolateCell(data.getData().data() + cell.dataIndex, cell.sliceOffset, cell.rowStride, cell.weightsZ, cell.weightsR, cell.weightsPhi); } - return false; -} -template -bool TriCubicInterpolator::findSide(const int iz, const int ir, const int iphi, GridPos& posType) const -{ - if (isSideRight(iz, FZ)) { - posType = GridPos::SideXRight; - return true; - } else if (isSideLeft(iz)) { - posType = GridPos::SideXLeft; - return true; - } - if (isSideRight(ir, FR)) { - posType = GridPos::SideYRight; - return true; - } else if (isSideLeft(ir)) { - posType = GridPos::SideYLeft; - return true; - } - if (isSideRight(iphi, FPHI)) { - posType = GridPos::SideZRight; - return true; - } else if (isSideLeft(iphi)) { - posType = GridPos::SideZLeft; - return true; - } - return false; + // missing values outside of the grid are extrapolated + DataT cVals[16][4]; + setValues(data, cell.indexZ, cell.indexR, cell.indexPhi, cVals); + static constexpr int sliceOffset[4]{0, 16, 32, 48}; + return interpolateCell(&cVals[0][0], sliceOffset, 4, cell.weightsZ, cell.weightsR, cell.weightsPhi); } template -bool TriCubicInterpolator::isInInnerVolume(const int iz, const int ir, const int iphi, GridPos& posType) const +DataT TriCubicInterpolator::interpolateSparse(const DataT z, const DataT r, const DataT phi) const { - if (iz >= 1 && iz < static_cast(mGridData->getNZ() - 2) && ir >= 1 && ir < static_cast(mGridData->getNR() - 2) && iphi >= 1 && iphi < static_cast(mGridData->getNPhi() - 2)) { - posType = GridPos::InnerVolume; - return true; + // check if data is empty + if (!mGridData->getNDataPoints()) { + return 0; } - return false; + return interpolateData(*mGridData, getCell(z, r, phi)); } template -bool TriCubicInterpolator::isSideRight(const int ind, const int dim) const +void TriCubicInterpolator::interpolate(const DataT z, const DataT r, const DataT phi, const TriCubicInterpolator& other1, const TriCubicInterpolator& other2, DataT& val0, DataT& val1, DataT& val2) const { - if (ind == static_cast(mGridProperties->getN(dim) - 2)) { - return true; + const size_t nPoints = mGridData->getNDataPoints(); + if (!nPoints || (other1.mGridData->getNDataPoints() != nPoints) || (other2.mGridData->getNDataPoints() != nPoints)) { + // empty data or data with different grid: separate interpolation + val0 = interpolateSparse(z, r, phi); + val1 = other1.interpolateSparse(z, r, phi); + val2 = other2.interpolateSparse(z, r, phi); + return; } - return false; + const TriCubicCell cell = getCell(z, r, phi); + if (cell.inGrid) { + // all values inside of the grid: interpolate the three data sets in one pass + interpolateCell3(mGridData->getData().data() + cell.dataIndex, other1.mGridData->getData().data() + cell.dataIndex, other2.mGridData->getData().data() + cell.dataIndex, cell.sliceOffset, cell.rowStride, cell.weightsZ, cell.weightsR, cell.weightsPhi, val0, val1, val2); + return; + } + val0 = interpolateData(*mGridData, cell); + val1 = interpolateData(*other1.mGridData, cell); + val2 = interpolateData(*other2.mGridData, cell); } +// for perdiodic boundary condition template -bool TriCubicInterpolator::isSideLeft(const int ind) const +void TriCubicInterpolator::getDataIndexCircularArray(const int index0, const int dim, int arr[]) const { - if (ind == 0) { - return true; - } - return false; + const int delta_min1 = getRegulatedDelta(index0, -1, dim, mGridProperties->getN(dim) - 1); + const int delta_plus1 = getRegulatedDelta(index0, +1, dim, 1 - mGridProperties->getN(dim)); + const int delta_plus2 = getRegulatedDelta(index0, +2, dim, 2 - mGridProperties->getN(dim)); + + arr[0] = mGridProperties->getDeltaDataIndex(delta_min1, dim); + arr[1] = mGridProperties->getDeltaDataIndex(delta_plus1, dim); + arr[2] = mGridProperties->getDeltaDataIndex(delta_plus2, dim); } template -void TriCubicInterpolator::setValues(const int iz, const int ir, const int iphi, std::array, 16>& cVals) const +void TriCubicInterpolator::setValues(const DataContainer& data, const int iz, const int ir, const int iphi, DataT (&cVals)[16][4]) const { - const GridPos location = findPos(iz, ir, iphi); - const int ii_x_y_z = mGridData->getDataIndex(iz, ir, iphi); - cVals[5][1] = (*mGridData)[ii_x_y_z]; + const int nZ = mGridProperties->getN(FZ); + const int nR = mGridProperties->getN(FR); - int deltaZ[3]{mGridProperties->getDeltaDataIndex(-1, 0), mGridProperties->getDeltaDataIndex(1, 0), mGridProperties->getDeltaDataIndex(2, 0)}; - int deltaR[3]{mGridProperties->getDeltaDataIndex(-1, 1), mGridProperties->getDeltaDataIndex(1, 1), mGridProperties->getDeltaDataIndex(2, 1)}; + // offsets of the data: the phi direction is periodic int deltaPhi[3]{}; getDataIndexCircularArray(iphi, FPHI, deltaPhi); - - const int i0 = 0; - const int i1 = 1; - const int i2 = 2; - - switch (location) { - case GridPos::InnerVolume: - case GridPos::SideZRight: - case GridPos::SideZLeft: - default: { - const int ind[4][4][4]{ - {{ii_x_y_z + deltaPhi[i0] + deltaR[i0] + deltaZ[i0], ind[0][0][0] - deltaZ[i0], ind[0][0][1] - deltaZ[i0], ind[0][0][2] - deltaZ[i0]}, - {ind[0][0][0] - deltaR[i0], ind[0][1][0] - deltaZ[i0], ind[0][1][1] - deltaZ[i0], ind[0][1][2] - deltaZ[i0]}, - {ind[0][1][0] - deltaR[i0], ind[0][2][0] - deltaZ[i0], ind[0][2][1] - deltaZ[i0], ind[0][2][2] - deltaZ[i0]}, - {ind[0][2][0] - deltaR[i0], ind[0][3][0] - deltaZ[i0], ind[0][3][1] - deltaZ[i0], ind[0][3][2] - deltaZ[i0]}}, - {{ii_x_y_z + deltaR[i0] + deltaZ[i0], ind[1][0][0] - deltaZ[i0], ind[1][0][1] - deltaZ[i0], ind[1][0][2] - deltaZ[i0]}, - {ind[1][0][0] - deltaR[i0], ind[1][1][0] - deltaZ[i0], ind[1][1][1] - deltaZ[i0], ind[1][1][2] - deltaZ[i0]}, - {ind[1][1][0] - deltaR[i0], ind[1][2][0] - deltaZ[i0], ind[1][2][1] - deltaZ[i0], ind[1][2][2] - deltaZ[i0]}, - {ind[1][2][0] - deltaR[i0], ind[1][3][0] - deltaZ[i0], ind[1][3][1] - deltaZ[i0], ind[1][3][2] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i1] + deltaR[i0] + deltaZ[i0], ind[2][0][0] - deltaZ[i0], ind[2][0][1] - deltaZ[i0], ind[2][0][2] - deltaZ[i0]}, - {ind[2][0][0] - deltaR[i0], ind[2][1][0] - deltaZ[i0], ind[2][1][1] - deltaZ[i0], ind[2][1][2] - deltaZ[i0]}, - {ind[2][1][0] - deltaR[i0], ind[2][2][0] - deltaZ[i0], ind[2][2][1] - deltaZ[i0], ind[2][2][2] - deltaZ[i0]}, - {ind[2][2][0] - deltaR[i0], ind[2][3][0] - deltaZ[i0], ind[2][3][1] - deltaZ[i0], ind[2][3][2] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i2] + deltaR[i0] + deltaZ[i0], ind[3][0][0] - deltaZ[i0], ind[3][0][1] - deltaZ[i0], ind[3][0][2] - deltaZ[i0]}, - {ind[3][0][0] - deltaR[i0], ind[3][1][0] - deltaZ[i0], ind[3][1][1] - deltaZ[i0], ind[3][1][2] - deltaZ[i0]}, - {ind[3][1][0] - deltaR[i0], ind[3][2][0] - deltaZ[i0], ind[3][2][1] - deltaZ[i0], ind[3][2][2] - deltaZ[i0]}, - {ind[3][2][0] - deltaR[i0], ind[3][3][0] - deltaZ[i0], ind[3][3][1] - deltaZ[i0], ind[3][3][2] - deltaZ[i0]}}}; - - cVals[0][0] = (*mGridData)[ind[0][0][0]]; - cVals[0][1] = (*mGridData)[ind[0][0][1]]; - cVals[0][2] = (*mGridData)[ind[0][0][2]]; - cVals[0][3] = (*mGridData)[ind[0][0][3]]; - cVals[1][0] = (*mGridData)[ind[0][1][0]]; - cVals[1][1] = (*mGridData)[ind[0][1][1]]; - cVals[1][2] = (*mGridData)[ind[0][1][2]]; - cVals[1][3] = (*mGridData)[ind[0][1][3]]; - cVals[2][0] = (*mGridData)[ind[0][2][0]]; - cVals[2][1] = (*mGridData)[ind[0][2][1]]; - cVals[2][2] = (*mGridData)[ind[0][2][2]]; - cVals[2][3] = (*mGridData)[ind[0][2][3]]; - cVals[3][0] = (*mGridData)[ind[0][3][0]]; - cVals[3][1] = (*mGridData)[ind[0][3][1]]; - cVals[3][2] = (*mGridData)[ind[0][3][2]]; - cVals[3][3] = (*mGridData)[ind[0][3][3]]; - cVals[4][0] = (*mGridData)[ind[1][0][0]]; - cVals[4][1] = (*mGridData)[ind[1][0][1]]; - cVals[4][2] = (*mGridData)[ind[1][0][2]]; - cVals[4][3] = (*mGridData)[ind[1][0][3]]; - cVals[5][2] = (*mGridData)[ind[1][1][2]]; - cVals[5][0] = (*mGridData)[ind[1][1][0]]; - cVals[5][3] = (*mGridData)[ind[1][1][3]]; - cVals[6][0] = (*mGridData)[ind[1][2][0]]; - cVals[6][1] = (*mGridData)[ind[1][2][1]]; - cVals[6][2] = (*mGridData)[ind[1][2][2]]; - cVals[6][3] = (*mGridData)[ind[1][2][3]]; - cVals[7][0] = (*mGridData)[ind[1][3][0]]; - cVals[7][1] = (*mGridData)[ind[1][3][1]]; - cVals[7][2] = (*mGridData)[ind[1][3][2]]; - cVals[7][3] = (*mGridData)[ind[1][3][3]]; - cVals[8][0] = (*mGridData)[ind[2][0][0]]; - cVals[8][1] = (*mGridData)[ind[2][0][1]]; - cVals[8][2] = (*mGridData)[ind[2][0][2]]; - cVals[8][3] = (*mGridData)[ind[2][0][3]]; - cVals[9][0] = (*mGridData)[ind[2][1][0]]; - cVals[9][1] = (*mGridData)[ind[2][1][1]]; - cVals[9][2] = (*mGridData)[ind[2][1][2]]; - cVals[9][3] = (*mGridData)[ind[2][1][3]]; - cVals[10][0] = (*mGridData)[ind[2][2][0]]; - cVals[10][1] = (*mGridData)[ind[2][2][1]]; - cVals[10][2] = (*mGridData)[ind[2][2][2]]; - cVals[10][3] = (*mGridData)[ind[2][2][3]]; - cVals[11][0] = (*mGridData)[ind[2][3][0]]; - cVals[11][1] = (*mGridData)[ind[2][3][1]]; - cVals[11][2] = (*mGridData)[ind[2][3][2]]; - cVals[11][3] = (*mGridData)[ind[2][3][3]]; - cVals[12][0] = (*mGridData)[ind[3][0][0]]; - cVals[12][1] = (*mGridData)[ind[3][0][1]]; - cVals[12][2] = (*mGridData)[ind[3][0][2]]; - cVals[12][3] = (*mGridData)[ind[3][0][3]]; - cVals[13][0] = (*mGridData)[ind[3][1][0]]; - cVals[13][1] = (*mGridData)[ind[3][1][1]]; - cVals[13][2] = (*mGridData)[ind[3][1][2]]; - cVals[13][3] = (*mGridData)[ind[3][1][3]]; - cVals[14][0] = (*mGridData)[ind[3][2][0]]; - cVals[14][1] = (*mGridData)[ind[3][2][1]]; - cVals[14][2] = (*mGridData)[ind[3][2][2]]; - cVals[14][3] = (*mGridData)[ind[3][2][3]]; - cVals[15][0] = (*mGridData)[ind[3][3][0]]; - cVals[15][1] = (*mGridData)[ind[3][3][1]]; - cVals[15][2] = (*mGridData)[ind[3][3][2]]; - cVals[15][3] = (*mGridData)[ind[3][3][3]]; - } break; - - case GridPos::SideXRight: - case GridPos::LineD: - case GridPos::LineH: { - const int ind[4][4][3]{ - {{ii_x_y_z + deltaPhi[i0] + deltaR[i0] + deltaZ[i0], ind[0][0][0] - deltaZ[i0], ind[0][0][1] - deltaZ[i0]}, - {ind[0][0][0] - deltaR[i0], ind[0][1][0] - deltaZ[i0], ind[0][1][1] - deltaZ[i0]}, - {ind[0][1][0] - deltaR[i0], ind[0][2][0] - deltaZ[i0], ind[0][2][1] - deltaZ[i0]}, - {ind[0][2][0] - deltaR[i0], ind[0][3][0] - deltaZ[i0], ind[0][3][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaR[i0] + deltaZ[i0], ind[1][0][0] - deltaZ[i0], ind[1][0][1] - deltaZ[i0]}, - {ind[1][0][0] - deltaR[i0], ind[1][1][0] - deltaZ[i0], ind[1][1][1] - deltaZ[i0]}, - {ind[1][1][0] - deltaR[i0], ind[1][2][0] - deltaZ[i0], ind[1][2][1] - deltaZ[i0]}, - {ind[1][2][0] - deltaR[i0], ind[1][3][0] - deltaZ[i0], ind[1][3][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i1] + deltaR[i0] + deltaZ[i0], ind[2][0][0] - deltaZ[i0], ind[2][0][1] - deltaZ[i0]}, - {ind[2][0][0] - deltaR[i0], ind[2][1][0] - deltaZ[i0], ind[2][1][1] - deltaZ[i0]}, - {ind[2][1][0] - deltaR[i0], ind[2][2][0] - deltaZ[i0], ind[2][2][1] - deltaZ[i0]}, - {ind[2][2][0] - deltaR[i0], ind[2][3][0] - deltaZ[i0], ind[2][3][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i2] + deltaR[i0] + deltaZ[i0], ind[3][0][0] - deltaZ[i0], ind[3][0][1] - deltaZ[i0]}, - {ind[3][0][0] - deltaR[i0], ind[3][1][0] - deltaZ[i0], ind[3][1][1] - deltaZ[i0]}, - {ind[3][1][0] - deltaR[i0], ind[3][2][0] - deltaZ[i0], ind[3][2][1] - deltaZ[i0]}, - {ind[3][2][0] - deltaR[i0], ind[3][3][0] - deltaZ[i0], ind[3][3][1] - deltaZ[i0]}}}; - - cVals[0][0] = (*mGridData)[ind[0][0][0]]; - cVals[0][1] = (*mGridData)[ind[0][0][1]]; - cVals[0][2] = (*mGridData)[ind[0][0][2]]; - cVals[0][3] = extrapolation((*mGridData)[ind[0][0][2]], (*mGridData)[ind[0][0][1]], (*mGridData)[ind[0][0][0]]); - cVals[1][0] = (*mGridData)[ind[0][1][0]]; - cVals[1][1] = (*mGridData)[ind[0][1][1]]; - cVals[1][2] = (*mGridData)[ind[0][1][2]]; - cVals[1][3] = extrapolation((*mGridData)[ind[0][1][2]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][1][0]]); - cVals[2][0] = (*mGridData)[ind[0][2][0]]; - cVals[2][1] = (*mGridData)[ind[0][2][1]]; - cVals[2][2] = (*mGridData)[ind[0][2][2]]; - cVals[2][3] = extrapolation((*mGridData)[ind[0][2][2]], (*mGridData)[ind[0][2][1]], (*mGridData)[ind[0][2][0]]); - cVals[3][0] = (*mGridData)[ind[0][3][0]]; - cVals[3][1] = (*mGridData)[ind[0][3][1]]; - cVals[3][2] = (*mGridData)[ind[0][3][2]]; - cVals[3][3] = extrapolation((*mGridData)[ind[0][3][2]], (*mGridData)[ind[0][3][1]], (*mGridData)[ind[0][3][0]]); - cVals[4][0] = (*mGridData)[ind[1][0][0]]; - cVals[4][1] = (*mGridData)[ind[1][0][1]]; - cVals[4][2] = (*mGridData)[ind[1][0][2]]; - cVals[4][3] = extrapolation((*mGridData)[ind[1][0][2]], (*mGridData)[ind[1][0][1]], (*mGridData)[ind[1][0][0]]); - cVals[5][0] = (*mGridData)[ind[1][1][0]]; - cVals[5][2] = (*mGridData)[ind[1][1][2]]; - cVals[5][3] = extrapolation((*mGridData)[ind[1][1][2]], (*mGridData)[ii_x_y_z], (*mGridData)[ind[1][1][0]]); - cVals[6][0] = (*mGridData)[ind[1][2][0]]; - cVals[6][1] = (*mGridData)[ind[1][2][1]]; - cVals[6][2] = (*mGridData)[ind[1][2][2]]; - cVals[6][3] = extrapolation((*mGridData)[ind[1][2][2]], (*mGridData)[ind[1][2][1]], (*mGridData)[ind[1][2][0]]); - cVals[7][0] = (*mGridData)[ind[1][3][0]]; - cVals[7][1] = (*mGridData)[ind[1][3][1]]; - cVals[7][2] = (*mGridData)[ind[1][3][2]]; - cVals[7][3] = extrapolation((*mGridData)[ind[1][3][2]], (*mGridData)[ind[1][3][1]], (*mGridData)[ind[1][3][0]]); - cVals[8][0] = (*mGridData)[ind[2][0][0]]; - cVals[8][1] = (*mGridData)[ind[2][0][1]]; - cVals[8][2] = (*mGridData)[ind[2][0][2]]; - cVals[8][3] = extrapolation((*mGridData)[ind[2][0][2]], (*mGridData)[ind[2][0][1]], (*mGridData)[ind[2][0][0]]); - cVals[9][0] = (*mGridData)[ind[2][1][0]]; - cVals[9][1] = (*mGridData)[ind[2][1][1]]; - cVals[9][2] = (*mGridData)[ind[2][1][2]]; - cVals[9][3] = extrapolation((*mGridData)[ind[2][1][2]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][1][0]]); - cVals[10][0] = (*mGridData)[ind[2][2][0]]; - cVals[10][1] = (*mGridData)[ind[2][2][1]]; - cVals[10][2] = (*mGridData)[ind[2][2][2]]; - cVals[10][3] = extrapolation((*mGridData)[ind[2][2][2]], (*mGridData)[ind[2][2][1]], (*mGridData)[ind[2][2][0]]); - cVals[11][0] = (*mGridData)[ind[2][3][0]]; - cVals[11][1] = (*mGridData)[ind[2][3][1]]; - cVals[11][2] = (*mGridData)[ind[2][3][2]]; - cVals[11][3] = extrapolation((*mGridData)[ind[2][3][2]], (*mGridData)[ind[2][3][1]], (*mGridData)[ind[2][3][0]]); - cVals[12][0] = (*mGridData)[ind[3][0][0]]; - cVals[12][1] = (*mGridData)[ind[3][0][1]]; - cVals[12][2] = (*mGridData)[ind[3][0][2]]; - cVals[13][0] = (*mGridData)[ind[3][1][0]]; - cVals[12][3] = extrapolation((*mGridData)[ind[3][0][2]], (*mGridData)[ind[3][0][1]], (*mGridData)[ind[3][0][0]]); - cVals[13][1] = (*mGridData)[ind[3][1][1]]; - cVals[13][2] = (*mGridData)[ind[3][1][2]]; - cVals[13][3] = extrapolation((*mGridData)[ind[3][1][2]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][1][0]]); - cVals[14][0] = (*mGridData)[ind[3][2][0]]; - cVals[14][1] = (*mGridData)[ind[3][2][1]]; - cVals[14][2] = (*mGridData)[ind[3][2][2]]; - cVals[14][3] = extrapolation((*mGridData)[ind[3][2][2]], (*mGridData)[ind[3][2][1]], (*mGridData)[ind[3][2][0]]); - cVals[15][0] = (*mGridData)[ind[3][3][0]]; - cVals[15][1] = (*mGridData)[ind[3][3][1]]; - cVals[15][2] = (*mGridData)[ind[3][3][2]]; - cVals[15][3] = extrapolation((*mGridData)[ind[3][3][2]], (*mGridData)[ind[3][3][1]], (*mGridData)[ind[3][3][0]]); - } break; - - case GridPos::SideYRight: - case GridPos::LineB: - case GridPos::LineF: { - const int ind[4][3][4]{ - {{ii_x_y_z + deltaPhi[i0] + deltaR[i0] + deltaZ[i0], ind[0][0][0] - deltaZ[i0], ind[0][0][1] - deltaZ[i0], ind[0][0][2] - deltaZ[i0]}, - {ind[0][0][0] - deltaR[i0], ind[0][1][0] - deltaZ[i0], ind[0][1][1] - deltaZ[i0], ind[0][1][2] - deltaZ[i0]}, - {ind[0][1][0] - deltaR[i0], ind[0][2][0] - deltaZ[i0], ind[0][2][1] - deltaZ[i0], ind[0][2][2] - deltaZ[i0]}}, - {{ii_x_y_z + deltaR[i0] + deltaZ[i0], ind[1][0][0] - deltaZ[i0], ind[1][0][1] - deltaZ[i0], ind[1][0][2] - deltaZ[i0]}, - {ind[1][0][0] - deltaR[i0], ind[1][1][0] - deltaZ[i0], ind[1][1][1] - deltaZ[i0], ind[1][1][2] - deltaZ[i0]}, - {ind[1][1][0] - deltaR[i0], ind[1][2][0] - deltaZ[i0], ind[1][2][1] - deltaZ[i0], ind[1][2][2] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i1] + deltaR[i0] + deltaZ[i0], ind[2][0][0] - deltaZ[i0], ind[2][0][1] - deltaZ[i0], ind[2][0][2] - deltaZ[i0]}, - {ind[2][0][0] - deltaR[i0], ind[2][1][0] - deltaZ[i0], ind[2][1][1] - deltaZ[i0], ind[2][1][2] - deltaZ[i0]}, - {ind[2][1][0] - deltaR[i0], ind[2][2][0] - deltaZ[i0], ind[2][2][1] - deltaZ[i0], ind[2][2][2] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i2] + deltaR[i0] + deltaZ[i0], ind[3][0][0] - deltaZ[i0], ind[3][0][1] - deltaZ[i0], ind[3][0][2] - deltaZ[i0]}, - {ind[3][0][0] - deltaR[i0], ind[3][1][0] - deltaZ[i0], ind[3][1][1] - deltaZ[i0], ind[3][1][2] - deltaZ[i0]}, - {ind[3][1][0] - deltaR[i0], ind[3][2][0] - deltaZ[i0], ind[3][2][1] - deltaZ[i0], ind[3][2][2] - deltaZ[i0]}}}; - - cVals[0][0] = (*mGridData)[ind[0][0][0]]; - cVals[0][1] = (*mGridData)[ind[0][0][1]]; - cVals[0][2] = (*mGridData)[ind[0][0][2]]; - cVals[0][3] = (*mGridData)[ind[0][0][3]]; - cVals[1][0] = (*mGridData)[ind[0][1][0]]; - cVals[1][1] = (*mGridData)[ind[0][1][1]]; - cVals[1][2] = (*mGridData)[ind[0][1][2]]; - cVals[1][3] = (*mGridData)[ind[0][1][3]]; - cVals[2][0] = (*mGridData)[ind[0][2][0]]; - cVals[2][1] = (*mGridData)[ind[0][2][1]]; - cVals[2][2] = (*mGridData)[ind[0][2][2]]; - cVals[2][3] = (*mGridData)[ind[0][2][3]]; - cVals[3][0] = extrapolation((*mGridData)[ind[0][2][0]], (*mGridData)[ind[0][1][0]], (*mGridData)[ind[0][0][0]]); - cVals[3][1] = extrapolation((*mGridData)[ind[0][2][1]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][0][1]]); - cVals[3][2] = extrapolation((*mGridData)[ind[0][2][2]], (*mGridData)[ind[0][1][2]], (*mGridData)[ind[0][0][2]]); - cVals[3][3] = extrapolation((*mGridData)[ind[0][2][3]], (*mGridData)[ind[0][1][3]], (*mGridData)[ind[0][0][3]]); - cVals[4][0] = (*mGridData)[ind[1][0][0]]; - cVals[4][1] = (*mGridData)[ind[1][0][1]]; - cVals[4][2] = (*mGridData)[ind[1][0][2]]; - cVals[4][3] = (*mGridData)[ind[1][0][3]]; - cVals[5][0] = (*mGridData)[ind[1][1][0]]; - cVals[5][2] = (*mGridData)[ind[1][1][2]]; - cVals[5][3] = (*mGridData)[ind[1][1][3]]; - cVals[6][0] = (*mGridData)[ind[1][2][0]]; - cVals[6][1] = (*mGridData)[ind[1][2][1]]; - cVals[6][2] = (*mGridData)[ind[1][2][2]]; - cVals[6][3] = (*mGridData)[ind[1][2][3]]; - cVals[7][0] = extrapolation((*mGridData)[ind[1][2][0]], (*mGridData)[ind[1][1][0]], (*mGridData)[ind[1][0][0]]); - cVals[7][1] = extrapolation((*mGridData)[ind[1][2][1]], (*mGridData)[ii_x_y_z], (*mGridData)[ind[1][0][1]]); - cVals[7][2] = extrapolation((*mGridData)[ind[1][2][2]], (*mGridData)[ind[1][1][2]], (*mGridData)[ind[1][0][2]]); - cVals[7][3] = extrapolation((*mGridData)[ind[1][2][3]], (*mGridData)[ind[1][1][3]], (*mGridData)[ind[1][0][3]]); - cVals[8][0] = (*mGridData)[ind[2][0][0]]; - cVals[8][1] = (*mGridData)[ind[2][0][1]]; - cVals[8][2] = (*mGridData)[ind[2][0][2]]; - cVals[8][3] = (*mGridData)[ind[2][0][3]]; - cVals[9][0] = (*mGridData)[ind[2][1][0]]; - cVals[9][1] = (*mGridData)[ind[2][1][1]]; - cVals[9][2] = (*mGridData)[ind[2][1][2]]; - cVals[9][3] = (*mGridData)[ind[2][1][3]]; - cVals[10][0] = (*mGridData)[ind[2][2][0]]; - cVals[10][1] = (*mGridData)[ind[2][2][1]]; - cVals[10][2] = (*mGridData)[ind[2][2][2]]; - cVals[10][3] = (*mGridData)[ind[2][2][3]]; - cVals[11][0] = extrapolation((*mGridData)[ind[2][2][0]], (*mGridData)[ind[2][1][0]], (*mGridData)[ind[2][0][0]]); - cVals[11][1] = extrapolation((*mGridData)[ind[2][2][1]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][0][1]]); - cVals[11][2] = extrapolation((*mGridData)[ind[2][2][2]], (*mGridData)[ind[2][1][2]], (*mGridData)[ind[2][0][2]]); - cVals[11][3] = extrapolation((*mGridData)[ind[2][2][3]], (*mGridData)[ind[2][1][3]], (*mGridData)[ind[2][0][3]]); - cVals[12][0] = (*mGridData)[ind[3][0][0]]; - cVals[12][1] = (*mGridData)[ind[3][0][1]]; - cVals[12][2] = (*mGridData)[ind[3][0][2]]; - cVals[12][3] = (*mGridData)[ind[3][0][3]]; - cVals[13][0] = (*mGridData)[ind[3][1][0]]; - cVals[13][1] = (*mGridData)[ind[3][1][1]]; - cVals[13][2] = (*mGridData)[ind[3][1][2]]; - cVals[13][3] = (*mGridData)[ind[3][1][3]]; - cVals[14][0] = (*mGridData)[ind[3][2][0]]; - cVals[14][1] = (*mGridData)[ind[3][2][1]]; - cVals[14][2] = (*mGridData)[ind[3][2][2]]; - cVals[14][3] = (*mGridData)[ind[3][2][3]]; - cVals[15][0] = extrapolation((*mGridData)[ind[3][2][0]], (*mGridData)[ind[3][1][0]], (*mGridData)[ind[3][0][0]]); - cVals[15][1] = extrapolation((*mGridData)[ind[3][2][1]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][0][1]]); - cVals[15][2] = extrapolation((*mGridData)[ind[3][2][2]], (*mGridData)[ind[3][1][2]], (*mGridData)[ind[3][0][2]]); - cVals[15][3] = extrapolation((*mGridData)[ind[3][2][3]], (*mGridData)[ind[3][1][3]], (*mGridData)[ind[3][0][3]]); - } break; - - case GridPos::SideYLeft: - case GridPos::LineA: - case GridPos::LineE: { - const int ind[4][3][4]{ - {{ii_x_y_z + deltaPhi[i0] + deltaZ[i0], ind[0][0][0] - deltaZ[i0], ind[0][0][1] - deltaZ[i0], ind[0][0][2] - deltaZ[i0]}, - {ind[0][0][0] - deltaR[i0], ind[0][1][0] - deltaZ[i0], ind[0][1][1] - deltaZ[i0], ind[0][1][2] - deltaZ[i0]}, - {ind[0][1][0] - deltaR[i0], ind[0][2][0] - deltaZ[i0], ind[0][2][1] - deltaZ[i0], ind[0][2][2] - deltaZ[i0]}}, - {{ii_x_y_z + deltaZ[i0], ind[1][0][0] - deltaZ[i0], ind[1][0][1] - deltaZ[i0], ind[1][0][2] - deltaZ[i0]}, - {ind[1][0][0] - deltaR[i0], ind[1][1][0] - deltaZ[i0], ind[1][1][1] - deltaZ[i0], ind[1][1][2] - deltaZ[i0]}, - {ind[1][1][0] - deltaR[i0], ind[1][2][0] - deltaZ[i0], ind[1][2][1] - deltaZ[i0], ind[1][2][2] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i1] + deltaZ[i0], ind[2][0][0] - deltaZ[i0], ind[2][0][1] - deltaZ[i0], ind[2][0][2] - deltaZ[i0]}, - {ind[2][0][0] - deltaR[i0], ind[2][1][0] - deltaZ[i0], ind[2][1][1] - deltaZ[i0], ind[2][1][2] - deltaZ[i0]}, - {ind[2][1][0] - deltaR[i0], ind[2][2][0] - deltaZ[i0], ind[2][2][1] - deltaZ[i0], ind[2][2][2] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i2] + deltaZ[i0], ind[3][0][0] - deltaZ[i0], ind[3][0][1] - deltaZ[i0], ind[3][0][2] - deltaZ[i0]}, - {ind[3][0][0] - deltaR[i0], ind[3][1][0] - deltaZ[i0], ind[3][1][1] - deltaZ[i0], ind[3][1][2] - deltaZ[i0]}, - {ind[3][1][0] - deltaR[i0], ind[3][2][0] - deltaZ[i0], ind[3][2][1] - deltaZ[i0], ind[3][2][2] - deltaZ[i0]}}}; - - cVals[0][0] = extrapolation((*mGridData)[ind[0][0][0]], (*mGridData)[ind[0][1][0]], (*mGridData)[ind[0][2][0]]); - cVals[0][1] = extrapolation((*mGridData)[ind[0][0][1]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][2][1]]); - cVals[0][2] = extrapolation((*mGridData)[ind[0][0][2]], (*mGridData)[ind[0][1][2]], (*mGridData)[ind[0][2][2]]); - cVals[0][3] = extrapolation((*mGridData)[ind[0][0][3]], (*mGridData)[ind[0][1][3]], (*mGridData)[ind[0][2][3]]); - cVals[1][0] = (*mGridData)[ind[0][0][0]]; - cVals[1][1] = (*mGridData)[ind[0][0][1]]; - cVals[1][2] = (*mGridData)[ind[0][0][2]]; - cVals[1][3] = (*mGridData)[ind[0][0][3]]; - cVals[2][0] = (*mGridData)[ind[0][1][0]]; - cVals[2][1] = (*mGridData)[ind[0][1][1]]; - cVals[2][2] = (*mGridData)[ind[0][1][2]]; - cVals[2][3] = (*mGridData)[ind[0][1][3]]; - cVals[3][0] = (*mGridData)[ind[0][2][0]]; - cVals[3][1] = (*mGridData)[ind[0][2][1]]; - cVals[3][2] = (*mGridData)[ind[0][2][2]]; - cVals[3][3] = (*mGridData)[ind[0][2][3]]; - cVals[4][0] = extrapolation((*mGridData)[ind[1][0][0]], (*mGridData)[ind[1][1][0]], (*mGridData)[ind[1][2][0]]); - cVals[4][1] = extrapolation((*mGridData)[ii_x_y_z], (*mGridData)[ind[1][1][1]], (*mGridData)[ind[1][2][1]]); - cVals[4][2] = extrapolation((*mGridData)[ind[1][0][2]], (*mGridData)[ind[1][1][2]], (*mGridData)[ind[1][2][2]]); - cVals[4][3] = extrapolation((*mGridData)[ind[1][0][3]], (*mGridData)[ind[1][1][3]], (*mGridData)[ind[1][2][3]]); - cVals[5][0] = (*mGridData)[ind[1][0][0]]; - cVals[5][2] = (*mGridData)[ind[1][0][2]]; - cVals[5][3] = (*mGridData)[ind[1][0][3]]; - cVals[6][0] = (*mGridData)[ind[1][1][0]]; - cVals[6][1] = (*mGridData)[ind[1][1][1]]; - cVals[6][2] = (*mGridData)[ind[1][1][2]]; - cVals[6][3] = (*mGridData)[ind[1][1][3]]; - cVals[7][0] = (*mGridData)[ind[1][2][0]]; - cVals[7][1] = (*mGridData)[ind[1][2][1]]; - cVals[7][2] = (*mGridData)[ind[1][2][2]]; - cVals[7][3] = (*mGridData)[ind[1][2][3]]; - cVals[8][0] = extrapolation((*mGridData)[ind[2][0][0]], (*mGridData)[ind[2][1][0]], (*mGridData)[ind[2][2][0]]); - cVals[8][1] = extrapolation((*mGridData)[ind[2][0][1]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][2][1]]); - cVals[8][2] = extrapolation((*mGridData)[ind[2][0][2]], (*mGridData)[ind[2][1][2]], (*mGridData)[ind[2][2][2]]); - cVals[8][3] = extrapolation((*mGridData)[ind[2][0][3]], (*mGridData)[ind[2][1][3]], (*mGridData)[ind[2][2][3]]); - cVals[9][0] = (*mGridData)[ind[2][0][0]]; - cVals[9][1] = (*mGridData)[ind[2][0][1]]; - cVals[9][2] = (*mGridData)[ind[2][0][2]]; - cVals[9][3] = (*mGridData)[ind[2][0][3]]; - cVals[10][0] = (*mGridData)[ind[2][1][0]]; - cVals[10][1] = (*mGridData)[ind[2][1][1]]; - cVals[10][2] = (*mGridData)[ind[2][1][2]]; - cVals[10][3] = (*mGridData)[ind[2][1][3]]; - cVals[11][0] = (*mGridData)[ind[2][2][0]]; - cVals[11][1] = (*mGridData)[ind[2][2][1]]; - cVals[11][2] = (*mGridData)[ind[2][2][2]]; - cVals[11][3] = (*mGridData)[ind[2][2][3]]; - cVals[12][0] = extrapolation((*mGridData)[ind[3][0][0]], (*mGridData)[ind[3][1][0]], (*mGridData)[ind[3][2][0]]); - cVals[12][1] = extrapolation((*mGridData)[ind[3][0][1]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][2][1]]); - cVals[12][2] = extrapolation((*mGridData)[ind[3][0][2]], (*mGridData)[ind[3][1][2]], (*mGridData)[ind[3][2][2]]); - cVals[12][3] = extrapolation((*mGridData)[ind[3][0][3]], (*mGridData)[ind[3][1][3]], (*mGridData)[ind[3][2][3]]); - cVals[13][0] = (*mGridData)[ind[3][0][0]]; - cVals[13][1] = (*mGridData)[ind[3][0][1]]; - cVals[13][2] = (*mGridData)[ind[3][0][2]]; - cVals[13][3] = (*mGridData)[ind[3][0][3]]; - cVals[14][0] = (*mGridData)[ind[3][1][0]]; - cVals[14][1] = (*mGridData)[ind[3][1][1]]; - cVals[14][2] = (*mGridData)[ind[3][1][2]]; - cVals[14][3] = (*mGridData)[ind[3][1][3]]; - cVals[15][0] = (*mGridData)[ind[3][2][0]]; - cVals[15][1] = (*mGridData)[ind[3][2][1]]; - cVals[15][2] = (*mGridData)[ind[3][2][2]]; - cVals[15][3] = (*mGridData)[ind[3][2][3]]; - } break; - - case GridPos::SideXLeft: - case GridPos::LineC: - case GridPos::LineG: { - const int ind[4][4][3]{ - {{ii_x_y_z + deltaPhi[i0] + deltaR[i0], ind[0][0][0] - deltaZ[i0], ind[0][0][1] - deltaZ[i0]}, - {ind[0][0][0] - deltaR[i0], ind[0][1][0] - deltaZ[i0], ind[0][1][1] - deltaZ[i0]}, - {ind[0][1][0] - deltaR[i0], ind[0][2][0] - deltaZ[i0], ind[0][2][1] - deltaZ[i0]}, - {ind[0][2][0] - deltaR[i0], ind[0][3][0] - deltaZ[i0], ind[0][3][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaR[i0], ind[1][0][0] - deltaZ[i0], ind[1][0][1] - deltaZ[i0]}, - {ind[1][0][0] - deltaR[i0], ind[1][1][0] - deltaZ[i0], ind[1][1][1] - deltaZ[i0]}, - {ind[1][1][0] - deltaR[i0], ind[1][2][0] - deltaZ[i0], ind[1][2][1] - deltaZ[i0]}, - {ind[1][2][0] - deltaR[i0], ind[1][3][0] - deltaZ[i0], ind[1][3][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i1] + deltaR[i0], ind[2][0][0] - deltaZ[i0], ind[2][0][1] - deltaZ[i0]}, - {ind[2][0][0] - deltaR[i0], ind[2][1][0] - deltaZ[i0], ind[2][1][1] - deltaZ[i0]}, - {ind[2][1][0] - deltaR[i0], ind[2][2][0] - deltaZ[i0], ind[2][2][1] - deltaZ[i0]}, - {ind[2][2][0] - deltaR[i0], ind[2][3][0] - deltaZ[i0], ind[2][3][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i2] + deltaR[i0], ind[3][0][0] - deltaZ[i0], ind[3][0][1] - deltaZ[i0]}, - {ind[3][0][0] - deltaR[i0], ind[3][1][0] - deltaZ[i0], ind[3][1][1] - deltaZ[i0]}, - {ind[3][1][0] - deltaR[i0], ind[3][2][0] - deltaZ[i0], ind[3][2][1] - deltaZ[i0]}, - {ind[3][2][0] - deltaR[i0], ind[3][3][0] - deltaZ[i0], ind[3][3][1] - deltaZ[i0]}}}; - - cVals[0][0] = extrapolation((*mGridData)[ind[0][0][0]], (*mGridData)[ind[0][0][1]], (*mGridData)[ind[0][0][2]]); - cVals[0][1] = (*mGridData)[ind[0][0][0]]; - cVals[0][2] = (*mGridData)[ind[0][0][1]]; - cVals[0][3] = (*mGridData)[ind[0][0][2]]; - cVals[1][0] = extrapolation((*mGridData)[ind[0][1][0]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][1][2]]); - cVals[1][1] = (*mGridData)[ind[0][1][0]]; - cVals[1][2] = (*mGridData)[ind[0][1][1]]; - cVals[1][3] = (*mGridData)[ind[0][1][2]]; - cVals[2][0] = extrapolation((*mGridData)[ind[0][2][0]], (*mGridData)[ind[0][2][1]], (*mGridData)[ind[0][2][2]]); - cVals[2][1] = (*mGridData)[ind[0][2][0]]; - cVals[2][2] = (*mGridData)[ind[0][2][1]]; - cVals[2][3] = (*mGridData)[ind[0][2][2]]; - cVals[3][0] = extrapolation((*mGridData)[ind[0][3][0]], (*mGridData)[ind[0][3][1]], (*mGridData)[ind[0][3][2]]); - cVals[3][1] = (*mGridData)[ind[0][3][0]]; - cVals[3][2] = (*mGridData)[ind[0][3][1]]; - cVals[3][3] = (*mGridData)[ind[0][3][2]]; - cVals[4][0] = extrapolation((*mGridData)[ind[1][0][0]], (*mGridData)[ind[1][0][1]], (*mGridData)[ind[1][0][2]]); - cVals[4][1] = (*mGridData)[ind[1][0][0]]; - cVals[4][2] = (*mGridData)[ind[1][0][1]]; - cVals[4][3] = (*mGridData)[ind[1][0][2]]; - cVals[5][0] = extrapolation((*mGridData)[ii_x_y_z], (*mGridData)[ind[1][1][1]], (*mGridData)[ind[1][1][2]]); - cVals[5][2] = (*mGridData)[ind[1][1][1]]; - cVals[5][3] = (*mGridData)[ind[1][1][2]]; - cVals[6][0] = extrapolation((*mGridData)[ind[1][2][0]], (*mGridData)[ind[1][2][1]], (*mGridData)[ind[1][2][2]]); - cVals[6][1] = (*mGridData)[ind[1][2][0]]; - cVals[6][2] = (*mGridData)[ind[1][2][1]]; - cVals[6][3] = (*mGridData)[ind[1][2][2]]; - cVals[7][0] = extrapolation((*mGridData)[ind[1][3][0]], (*mGridData)[ind[1][3][1]], (*mGridData)[ind[1][3][2]]); - cVals[7][1] = (*mGridData)[ind[1][3][0]]; - cVals[7][2] = (*mGridData)[ind[1][3][1]]; - cVals[7][3] = (*mGridData)[ind[1][3][2]]; - cVals[8][0] = extrapolation((*mGridData)[ind[2][0][0]], (*mGridData)[ind[2][0][1]], (*mGridData)[ind[2][0][2]]); - cVals[8][1] = (*mGridData)[ind[2][0][0]]; - cVals[8][2] = (*mGridData)[ind[2][0][1]]; - cVals[8][3] = (*mGridData)[ind[2][0][2]]; - cVals[9][0] = extrapolation((*mGridData)[ind[2][1][0]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][1][2]]); - cVals[9][1] = (*mGridData)[ind[2][1][0]]; - cVals[9][2] = (*mGridData)[ind[2][1][1]]; - cVals[9][3] = (*mGridData)[ind[2][1][2]]; - cVals[10][0] = extrapolation((*mGridData)[ind[2][2][0]], (*mGridData)[ind[2][2][1]], (*mGridData)[ind[2][2][2]]); - cVals[10][1] = (*mGridData)[ind[2][2][0]]; - cVals[10][2] = (*mGridData)[ind[2][2][1]]; - cVals[10][3] = (*mGridData)[ind[2][2][2]]; - cVals[11][0] = extrapolation((*mGridData)[ind[2][3][0]], (*mGridData)[ind[2][3][1]], (*mGridData)[ind[2][3][2]]); - cVals[11][1] = (*mGridData)[ind[2][3][0]]; - cVals[11][2] = (*mGridData)[ind[2][3][1]]; - cVals[11][3] = (*mGridData)[ind[2][3][2]]; - cVals[12][0] = extrapolation((*mGridData)[ind[3][0][0]], (*mGridData)[ind[3][0][1]], (*mGridData)[ind[3][0][2]]); - cVals[12][1] = (*mGridData)[ind[3][0][0]]; - cVals[12][2] = (*mGridData)[ind[3][0][1]]; - cVals[12][3] = (*mGridData)[ind[3][0][2]]; - cVals[13][0] = extrapolation((*mGridData)[ind[3][1][0]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][1][2]]); - cVals[13][1] = (*mGridData)[ind[3][1][0]]; - cVals[13][2] = (*mGridData)[ind[3][1][1]]; - cVals[13][3] = (*mGridData)[ind[3][1][2]]; - cVals[14][0] = extrapolation((*mGridData)[ind[3][2][0]], (*mGridData)[ind[3][2][1]], (*mGridData)[ind[3][2][2]]); - cVals[14][1] = (*mGridData)[ind[3][2][0]]; - cVals[14][2] = (*mGridData)[ind[3][2][1]]; - cVals[14][3] = (*mGridData)[ind[3][2][2]]; - cVals[15][0] = extrapolation((*mGridData)[ind[3][3][0]], (*mGridData)[ind[3][3][1]], (*mGridData)[ind[3][3][2]]); - cVals[15][1] = (*mGridData)[ind[3][3][0]]; - cVals[15][2] = (*mGridData)[ind[3][3][1]]; - cVals[15][3] = (*mGridData)[ind[3][3][2]]; - } break; - - case GridPos::Edge0: - case GridPos::Edge4: - case GridPos::LineI: { - const int ind[4][3][3]{ - {{ii_x_y_z + deltaPhi[i0], ind[0][0][0] - deltaZ[i0], ind[0][0][1] - deltaZ[i0]}, - {ind[0][0][0] - deltaR[i0], ind[0][1][0] - deltaZ[i0], ind[0][1][1] - deltaZ[i0]}, - {ind[0][1][0] - deltaR[i0], ind[0][2][0] - deltaZ[i0], ind[0][2][1] - deltaZ[i0]}}, - {{ii_x_y_z, ind[1][0][0] - deltaZ[i0], ind[1][0][1] - deltaZ[i0]}, - {ind[1][0][0] - deltaR[i0], ind[1][1][0] - deltaZ[i0], ind[1][1][1] - deltaZ[i0]}, - {ind[1][1][0] - deltaR[i0], ind[1][2][0] - deltaZ[i0], ind[1][2][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i1], ind[2][0][0] - deltaZ[i0], ind[2][0][1] - deltaZ[i0]}, - {ind[2][0][0] - deltaR[i0], ind[2][1][0] - deltaZ[i0], ind[2][1][1] - deltaZ[i0]}, - {ind[2][1][0] - deltaR[i0], ind[2][2][0] - deltaZ[i0], ind[2][2][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i2], ind[3][0][0] - deltaZ[i0], ind[3][0][1] - deltaZ[i0]}, - {ind[3][0][0] - deltaR[i0], ind[3][1][0] - deltaZ[i0], ind[3][1][1] - deltaZ[i0]}, - {ind[3][1][0] - deltaR[i0], ind[3][2][0] - deltaZ[i0], ind[3][2][1] - deltaZ[i0]}}}; - - cVals[0][0] = extrapolation((*mGridData)[ind[0][0][0]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][2][2]]); - cVals[0][1] = extrapolation((*mGridData)[ind[0][0][0]], (*mGridData)[ind[0][1][0]], (*mGridData)[ind[0][2][0]]); - cVals[0][2] = extrapolation((*mGridData)[ind[0][0][1]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][2][1]]); - cVals[0][3] = extrapolation((*mGridData)[ind[0][0][2]], (*mGridData)[ind[0][1][2]], (*mGridData)[ind[0][2][2]]); - cVals[1][0] = extrapolation((*mGridData)[ind[0][0][0]], (*mGridData)[ind[0][0][1]], (*mGridData)[ind[0][0][2]]); - cVals[1][1] = (*mGridData)[ind[0][0][0]]; - cVals[1][2] = (*mGridData)[ind[0][0][1]]; - cVals[1][3] = (*mGridData)[ind[0][0][2]]; - cVals[2][0] = extrapolation((*mGridData)[ind[0][1][0]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][1][2]]); - cVals[2][1] = (*mGridData)[ind[0][1][0]]; - cVals[2][2] = (*mGridData)[ind[0][1][1]]; - cVals[2][3] = (*mGridData)[ind[0][1][2]]; - cVals[3][0] = extrapolation((*mGridData)[ind[0][2][0]], (*mGridData)[ind[0][2][1]], (*mGridData)[ind[0][2][2]]); - cVals[3][1] = (*mGridData)[ind[0][2][0]]; - cVals[3][2] = (*mGridData)[ind[0][2][1]]; - cVals[3][3] = (*mGridData)[ind[0][2][2]]; - cVals[4][0] = extrapolation((*mGridData)[ii_x_y_z], (*mGridData)[ind[1][1][1]], (*mGridData)[ind[1][2][2]]); - cVals[4][1] = extrapolation((*mGridData)[ii_x_y_z], (*mGridData)[ind[1][1][0]], (*mGridData)[ind[1][2][0]]); - cVals[4][2] = extrapolation((*mGridData)[ind[1][0][1]], (*mGridData)[ind[1][1][1]], (*mGridData)[ind[1][2][1]]); - cVals[4][3] = extrapolation((*mGridData)[ind[1][0][2]], (*mGridData)[ind[1][1][2]], (*mGridData)[ind[1][2][2]]); - cVals[5][0] = extrapolation((*mGridData)[ii_x_y_z], (*mGridData)[ind[1][0][1]], (*mGridData)[ind[1][0][2]]); - cVals[5][2] = (*mGridData)[ind[1][0][1]]; - cVals[5][3] = (*mGridData)[ind[1][0][2]]; - cVals[6][0] = extrapolation((*mGridData)[ind[1][1][0]], (*mGridData)[ind[1][1][1]], (*mGridData)[ind[1][1][2]]); - cVals[6][1] = (*mGridData)[ind[1][1][0]]; - cVals[6][2] = (*mGridData)[ind[1][1][1]]; - cVals[6][3] = (*mGridData)[ind[1][1][2]]; - cVals[7][0] = extrapolation((*mGridData)[ind[1][2][0]], (*mGridData)[ind[1][2][1]], (*mGridData)[ind[1][2][2]]); - cVals[7][1] = (*mGridData)[ind[1][2][0]]; - cVals[7][2] = (*mGridData)[ind[1][2][1]]; - cVals[7][3] = (*mGridData)[ind[1][2][2]]; - cVals[8][0] = extrapolation((*mGridData)[ind[2][0][0]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][2][2]]); - cVals[8][1] = extrapolation((*mGridData)[ind[2][0][0]], (*mGridData)[ind[2][1][0]], (*mGridData)[ind[2][2][0]]); - cVals[8][2] = extrapolation((*mGridData)[ind[2][0][1]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][2][1]]); - cVals[8][3] = extrapolation((*mGridData)[ind[2][0][2]], (*mGridData)[ind[2][1][2]], (*mGridData)[ind[2][2][2]]); - cVals[9][0] = extrapolation((*mGridData)[ind[2][0][0]], (*mGridData)[ind[2][0][1]], (*mGridData)[ind[2][0][2]]); - cVals[9][1] = (*mGridData)[ind[2][0][0]]; - cVals[9][2] = (*mGridData)[ind[2][0][1]]; - cVals[9][3] = (*mGridData)[ind[2][0][2]]; - cVals[10][0] = extrapolation((*mGridData)[ind[2][1][0]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][1][2]]); - cVals[10][1] = (*mGridData)[ind[2][1][0]]; - cVals[10][2] = (*mGridData)[ind[2][1][1]]; - cVals[10][3] = (*mGridData)[ind[2][1][2]]; - cVals[11][0] = extrapolation((*mGridData)[ind[2][2][0]], (*mGridData)[ind[2][2][1]], (*mGridData)[ind[2][2][2]]); - cVals[11][1] = (*mGridData)[ind[2][2][0]]; - cVals[11][2] = (*mGridData)[ind[2][2][1]]; - cVals[11][3] = (*mGridData)[ind[2][2][2]]; - cVals[12][0] = extrapolation((*mGridData)[ind[3][0][0]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][2][2]]); - cVals[12][1] = extrapolation((*mGridData)[ind[3][0][0]], (*mGridData)[ind[3][1][0]], (*mGridData)[ind[3][2][0]]); - cVals[12][2] = extrapolation((*mGridData)[ind[3][0][1]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][2][1]]); - cVals[12][3] = extrapolation((*mGridData)[ind[3][0][2]], (*mGridData)[ind[3][1][2]], (*mGridData)[ind[3][2][2]]); - cVals[13][0] = extrapolation((*mGridData)[ind[3][0][0]], (*mGridData)[ind[3][0][1]], (*mGridData)[ind[3][0][2]]); - cVals[13][1] = (*mGridData)[ind[3][0][0]]; - cVals[13][2] = (*mGridData)[ind[3][0][1]]; - cVals[13][3] = (*mGridData)[ind[3][0][2]]; - cVals[14][0] = extrapolation((*mGridData)[ind[3][1][0]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][1][2]]); - cVals[14][1] = (*mGridData)[ind[3][1][0]]; - cVals[14][2] = (*mGridData)[ind[3][1][1]]; - cVals[14][3] = (*mGridData)[ind[3][1][2]]; - cVals[15][0] = extrapolation((*mGridData)[ind[3][2][0]], (*mGridData)[ind[3][2][1]], (*mGridData)[ind[3][2][2]]); - cVals[15][1] = (*mGridData)[ind[3][2][0]]; - cVals[15][2] = (*mGridData)[ind[3][2][1]]; - cVals[15][3] = (*mGridData)[ind[3][2][2]]; - } break; - - case GridPos::Edge1: - case GridPos::Edge5: - case GridPos::LineJ: { - const int ind[4][3][3]{ - {{ii_x_y_z + deltaPhi[i0] + deltaZ[i0], ind[0][0][0] - deltaZ[i0], ind[0][0][1] - deltaZ[i0]}, - {ind[0][0][0] - deltaR[i0], ind[0][1][0] - deltaZ[i0], ind[0][1][1] - deltaZ[i0]}, - {ind[0][1][0] - deltaR[i0], ind[0][2][0] - deltaZ[i0], ind[0][2][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaZ[i0], ind[1][0][0] - deltaZ[i0], ind[1][0][1] - deltaZ[i0]}, - {ind[1][0][0] - deltaR[i0], ind[1][1][0] - deltaZ[i0], ind[1][1][1] - deltaZ[i0]}, - {ind[1][1][0] - deltaR[i0], ind[1][2][0] - deltaZ[i0], ind[1][2][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i1] + deltaZ[i0], ind[2][0][0] - deltaZ[i0], ind[2][0][1] - deltaZ[i0]}, - {ind[2][0][0] - deltaR[i0], ind[2][1][0] - deltaZ[i0], ind[2][1][1] - deltaZ[i0]}, - {ind[2][1][0] - deltaR[i0], ind[2][2][0] - deltaZ[i0], ind[2][2][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i2] + deltaZ[i0], ind[3][0][0] - deltaZ[i0], ind[3][0][1] - deltaZ[i0]}, - {ind[3][0][0] - deltaR[i0], ind[3][1][0] - deltaZ[i0], ind[3][1][1] - deltaZ[i0]}, - {ind[3][1][0] - deltaR[i0], ind[3][2][0] - deltaZ[i0], ind[3][2][1] - deltaZ[i0]}}}; - - cVals[0][0] = extrapolation((*mGridData)[ind[0][0][0]], (*mGridData)[ind[0][1][0]], (*mGridData)[ind[0][2][0]]); - cVals[0][1] = extrapolation((*mGridData)[ind[0][0][1]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][2][1]]); - cVals[0][2] = extrapolation((*mGridData)[ind[0][0][1]], (*mGridData)[ind[0][1][0]], (*mGridData)[ind[0][2][0] + deltaZ[i0]]); - cVals[0][3] = extrapolation((*mGridData)[ind[0][0][2]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][2][0]]); - cVals[1][0] = (*mGridData)[ind[0][0][0]]; - cVals[1][1] = (*mGridData)[ind[0][0][1]]; - cVals[1][2] = (*mGridData)[ind[0][0][2]]; - cVals[1][3] = extrapolation((*mGridData)[ind[0][0][2]], (*mGridData)[ind[0][0][1]], (*mGridData)[ind[0][0][0]]); - cVals[2][0] = (*mGridData)[ind[0][1][0]]; - cVals[2][1] = (*mGridData)[ind[0][1][1]]; - cVals[2][2] = (*mGridData)[ind[0][1][2]]; - cVals[2][3] = extrapolation((*mGridData)[ind[0][1][2]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][1][0]]); - cVals[3][0] = (*mGridData)[ind[0][2][0]]; - cVals[3][1] = (*mGridData)[ind[0][2][1]]; - cVals[3][2] = (*mGridData)[ind[0][2][2]]; - cVals[3][3] = extrapolation((*mGridData)[ind[0][2][2]], (*mGridData)[ind[0][2][1]], (*mGridData)[ind[0][2][0]]); - cVals[4][0] = extrapolation((*mGridData)[ind[1][0][0]], (*mGridData)[ind[1][1][0]], (*mGridData)[ind[1][2][0]]); - cVals[4][1] = extrapolation((*mGridData)[ii_x_y_z], (*mGridData)[ind[1][1][1]], (*mGridData)[ind[1][2][1]]); - cVals[4][2] = extrapolation((*mGridData)[ii_x_y_z], (*mGridData)[ind[1][1][0]], (*mGridData)[ind[1][2][0] + deltaZ[i0]]); - cVals[4][3] = extrapolation((*mGridData)[ind[1][0][2]], (*mGridData)[ind[1][1][1]], (*mGridData)[ind[1][2][0]]); - cVals[5][0] = (*mGridData)[ind[1][0][0]]; - cVals[5][2] = (*mGridData)[ind[1][0][2]]; - cVals[5][3] = extrapolation((*mGridData)[ind[1][0][2]], (*mGridData)[ii_x_y_z], (*mGridData)[ind[1][0][0]]); - cVals[6][0] = (*mGridData)[ind[1][1][0]]; - cVals[6][1] = (*mGridData)[ind[1][1][1]]; - cVals[6][2] = (*mGridData)[ind[1][1][2]]; - cVals[6][3] = extrapolation((*mGridData)[ind[1][1][2]], (*mGridData)[ind[1][1][1]], (*mGridData)[ind[1][1][0]]); - cVals[7][0] = (*mGridData)[ind[1][2][0]]; - cVals[7][1] = (*mGridData)[ind[1][2][1]]; - cVals[7][2] = (*mGridData)[ind[1][2][2]]; - cVals[7][3] = extrapolation((*mGridData)[ind[1][2][2]], (*mGridData)[ind[1][2][1]], (*mGridData)[ind[1][2][0]]); - cVals[8][0] = extrapolation((*mGridData)[ind[2][0][0]], (*mGridData)[ind[2][1][0]], (*mGridData)[ind[2][2][0]]); - cVals[8][1] = extrapolation((*mGridData)[ind[2][0][1]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][2][1]]); - cVals[8][2] = extrapolation((*mGridData)[ind[2][0][1]], (*mGridData)[ind[2][1][0]], (*mGridData)[ind[2][2][0] + deltaZ[i0]]); - cVals[8][3] = extrapolation((*mGridData)[ind[2][0][2]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][2][0]]); - cVals[9][0] = (*mGridData)[ind[2][0][0]]; - cVals[9][1] = (*mGridData)[ind[2][0][1]]; - cVals[9][2] = (*mGridData)[ind[2][0][2]]; - cVals[9][3] = extrapolation((*mGridData)[ind[2][0][2]], (*mGridData)[ind[2][0][1]], (*mGridData)[ind[2][0][0]]); - cVals[10][0] = (*mGridData)[ind[2][1][0]]; - cVals[10][1] = (*mGridData)[ind[2][1][1]]; - cVals[10][2] = (*mGridData)[ind[2][1][2]]; - cVals[10][3] = extrapolation((*mGridData)[ind[2][1][2]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][1][0]]); - cVals[11][0] = (*mGridData)[ind[2][2][0]]; - cVals[11][1] = (*mGridData)[ind[2][2][1]]; - cVals[11][2] = (*mGridData)[ind[2][2][2]]; - cVals[11][3] = extrapolation((*mGridData)[ind[2][2][2]], (*mGridData)[ind[2][2][1]], (*mGridData)[ind[2][2][0]]); - cVals[12][0] = extrapolation((*mGridData)[ind[3][0][0]], (*mGridData)[ind[3][1][0]], (*mGridData)[ind[3][2][0]]); - cVals[12][1] = extrapolation((*mGridData)[ind[3][0][1]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][2][1]]); - cVals[12][2] = extrapolation((*mGridData)[ind[3][0][1]], (*mGridData)[ind[3][1][0]], (*mGridData)[ind[3][2][0] + deltaZ[i0]]); - cVals[12][3] = extrapolation((*mGridData)[ind[3][0][2]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][2][0]]); - cVals[13][0] = (*mGridData)[ind[3][0][0]]; - cVals[13][1] = (*mGridData)[ind[3][0][1]]; - cVals[13][2] = (*mGridData)[ind[3][0][2]]; - cVals[13][3] = extrapolation((*mGridData)[ind[3][0][2]], (*mGridData)[ind[3][0][1]], (*mGridData)[ind[3][0][0]]); - cVals[14][0] = (*mGridData)[ind[3][1][0]]; - cVals[14][1] = (*mGridData)[ind[3][1][1]]; - cVals[14][2] = (*mGridData)[ind[3][1][2]]; - cVals[14][3] = extrapolation((*mGridData)[ind[3][1][2]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][1][0]]); - cVals[15][0] = (*mGridData)[ind[3][2][0]]; - cVals[15][1] = (*mGridData)[ind[3][2][1]]; - cVals[15][2] = (*mGridData)[ind[3][2][2]]; - cVals[15][3] = extrapolation((*mGridData)[ind[3][2][2]], (*mGridData)[ind[3][2][1]], (*mGridData)[ind[3][2][0]]); - } break; - - case GridPos::Edge2: - case GridPos::Edge6: - case GridPos::LineK: { - const int ind[4][3][3]{ - {{ii_x_y_z + deltaPhi[i0] + deltaR[i0], ind[0][0][0] - deltaZ[i0], ind[0][0][1] - deltaZ[i0]}, - {ind[0][0][0] - deltaR[i0], ind[0][1][0] - deltaZ[i0], ind[0][1][1] - deltaZ[i0]}, - {ind[0][1][0] - deltaR[i0], ind[0][2][0] - deltaZ[i0], ind[0][2][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaR[i0], ind[1][0][0] - deltaZ[i0], ind[1][0][1] - deltaZ[i0]}, - {ind[1][0][0] - deltaR[i0], ind[1][1][0] - deltaZ[i0], ind[1][1][1] - deltaZ[i0]}, - {ind[1][1][0] - deltaR[i0], ind[1][2][0] - deltaZ[i0], ind[1][2][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i1] + deltaR[i0], ind[2][0][0] - deltaZ[i0], ind[2][0][1] - deltaZ[i0]}, - {ind[2][0][0] - deltaR[i0], ind[2][1][0] - deltaZ[i0], ind[2][1][1] - deltaZ[i0]}, - {ind[2][1][0] - deltaR[i0], ind[2][2][0] - deltaZ[i0], ind[2][2][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i2] + deltaR[i0], ind[3][0][0] - deltaZ[i0], ind[3][0][1] - deltaZ[i0]}, - {ind[3][0][0] - deltaR[i0], ind[3][1][0] - deltaZ[i0], ind[3][1][1] - deltaZ[i0]}, - {ind[3][1][0] - deltaR[i0], ind[3][2][0] - deltaZ[i0], ind[3][2][1] - deltaZ[i0]}}}; - - cVals[0][0] = extrapolation((*mGridData)[ind[0][0][0]], (*mGridData)[ind[0][0][1]], (*mGridData)[ind[0][0][2]]); - cVals[0][1] = (*mGridData)[ind[0][0][0]]; - cVals[0][2] = (*mGridData)[ind[0][0][1]]; - cVals[0][3] = (*mGridData)[ind[0][0][2]]; - cVals[1][0] = extrapolation((*mGridData)[ind[0][1][0]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][1][2]]); - cVals[1][1] = (*mGridData)[ind[0][1][0]]; - cVals[1][2] = (*mGridData)[ind[0][1][1]]; - cVals[1][3] = (*mGridData)[ind[0][1][2]]; - cVals[2][0] = extrapolation((*mGridData)[ind[0][1][0]], (*mGridData)[ind[0][0][1]], (*mGridData)[ind[0][0][2] + deltaR[i0]]); - cVals[2][1] = (*mGridData)[ind[0][2][0]]; - cVals[2][2] = (*mGridData)[ind[0][2][1]]; - cVals[2][3] = (*mGridData)[ind[0][2][2]]; - cVals[3][0] = extrapolation((*mGridData)[ind[0][2][0]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][0][2]]); - cVals[3][1] = extrapolation((*mGridData)[ind[0][2][0]], (*mGridData)[ind[0][1][0]], (*mGridData)[ind[0][0][0]]); - cVals[3][2] = extrapolation((*mGridData)[ind[0][2][1]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][0][1]]); - cVals[3][3] = extrapolation((*mGridData)[ind[0][2][2]], (*mGridData)[ind[0][1][2]], (*mGridData)[ind[0][0][2]]); - cVals[4][0] = extrapolation((*mGridData)[ind[1][0][0]], (*mGridData)[ind[1][0][1]], (*mGridData)[ind[1][0][2]]); - cVals[4][1] = (*mGridData)[ind[1][0][0]]; - cVals[4][2] = (*mGridData)[ind[1][0][1]]; - cVals[4][3] = (*mGridData)[ind[1][0][2]]; - cVals[5][0] = extrapolation((*mGridData)[ii_x_y_z], (*mGridData)[ind[1][1][1]], (*mGridData)[ind[1][1][2]]); - cVals[5][2] = (*mGridData)[ind[1][1][1]]; - cVals[5][3] = (*mGridData)[ind[1][1][2]]; - cVals[6][0] = extrapolation((*mGridData)[ii_x_y_z], (*mGridData)[ind[1][0][1]], (*mGridData)[ind[1][0][2] + deltaR[i0]]); - cVals[6][1] = (*mGridData)[ind[1][2][0]]; - cVals[6][2] = (*mGridData)[ind[1][2][1]]; - cVals[6][3] = (*mGridData)[ind[1][2][2]]; - cVals[7][0] = extrapolation((*mGridData)[ind[1][2][0]], (*mGridData)[ind[1][1][1]], (*mGridData)[ind[1][0][2]]); - cVals[7][1] = extrapolation((*mGridData)[ind[1][2][0]], (*mGridData)[ii_x_y_z], (*mGridData)[ind[1][0][0]]); - cVals[7][2] = extrapolation((*mGridData)[ind[1][2][1]], (*mGridData)[ind[1][1][1]], (*mGridData)[ind[1][0][1]]); - cVals[8][0] = extrapolation((*mGridData)[ind[2][0][0]], (*mGridData)[ind[2][0][1]], (*mGridData)[ind[2][0][2]]); - cVals[7][3] = extrapolation((*mGridData)[ind[1][2][2]], (*mGridData)[ind[1][1][2]], (*mGridData)[ind[1][0][2]]); - cVals[8][1] = (*mGridData)[ind[2][0][0]]; - cVals[8][2] = (*mGridData)[ind[2][0][1]]; - cVals[8][3] = (*mGridData)[ind[2][0][2]]; - cVals[9][0] = extrapolation((*mGridData)[ind[2][1][0]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][1][2]]); - cVals[9][1] = (*mGridData)[ind[2][1][0]]; - cVals[9][2] = (*mGridData)[ind[2][1][1]]; - cVals[9][3] = (*mGridData)[ind[2][1][2]]; - cVals[10][0] = extrapolation((*mGridData)[ind[2][1][0]], (*mGridData)[ind[2][0][1]], (*mGridData)[ind[2][0][2] + deltaR[i0]]); - cVals[10][1] = (*mGridData)[ind[2][2][0]]; - cVals[10][2] = (*mGridData)[ind[2][2][1]]; - cVals[10][3] = (*mGridData)[ind[2][2][2]]; - cVals[11][0] = extrapolation((*mGridData)[ind[2][2][0]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][0][2]]); - cVals[11][1] = extrapolation((*mGridData)[ind[2][2][0]], (*mGridData)[ind[2][1][0]], (*mGridData)[ind[2][0][0]]); - cVals[11][2] = extrapolation((*mGridData)[ind[2][2][1]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][0][1]]); - cVals[11][3] = extrapolation((*mGridData)[ind[2][2][2]], (*mGridData)[ind[2][1][2]], (*mGridData)[ind[2][0][2]]); - cVals[12][0] = extrapolation((*mGridData)[ind[3][0][0]], (*mGridData)[ind[3][0][1]], (*mGridData)[ind[3][0][2]]); - cVals[12][1] = (*mGridData)[ind[3][0][0]]; - cVals[12][2] = (*mGridData)[ind[3][0][1]]; - cVals[12][3] = (*mGridData)[ind[3][0][2]]; - cVals[13][0] = extrapolation((*mGridData)[ind[3][1][0]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][1][2]]); - cVals[13][1] = (*mGridData)[ind[3][1][0]]; - cVals[13][2] = (*mGridData)[ind[3][1][1]]; - cVals[13][3] = (*mGridData)[ind[3][1][2]]; - cVals[14][0] = extrapolation((*mGridData)[ind[3][1][0]], (*mGridData)[ind[3][0][1]], (*mGridData)[ind[3][0][2] + deltaR[i0]]); - cVals[14][1] = (*mGridData)[ind[3][2][0]]; - cVals[14][2] = (*mGridData)[ind[3][2][1]]; - cVals[14][3] = (*mGridData)[ind[3][2][2]]; - cVals[15][0] = extrapolation((*mGridData)[ind[3][2][0]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][0][2]]); - cVals[15][1] = extrapolation((*mGridData)[ind[3][2][0]], (*mGridData)[ind[3][1][0]], (*mGridData)[ind[3][0][0]]); - cVals[15][2] = extrapolation((*mGridData)[ind[3][2][1]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][0][1]]); - cVals[15][3] = extrapolation((*mGridData)[ind[3][2][2]], (*mGridData)[ind[3][1][2]], (*mGridData)[ind[3][0][2]]); - } break; - - case GridPos::Edge3: - case GridPos::Edge7: - case GridPos::LineL: { - const int ind[4][3][3]{ - {{ii_x_y_z + deltaPhi[i0] + deltaR[i0] + deltaZ[i0], ind[0][0][0] - deltaZ[i0], ind[0][0][1] - deltaZ[i0]}, - {ind[0][0][0] - deltaR[i0], ind[0][1][0] - deltaZ[i0], ind[0][1][1] - deltaZ[i0]}, - {ind[0][1][0] - deltaR[i0], ind[0][2][0] - deltaZ[i0], ind[0][2][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaR[i0] + deltaZ[i0], ind[1][0][0] - deltaZ[i0], ind[1][0][1] - deltaZ[i0]}, - {ind[1][0][0] - deltaR[i0], ind[1][1][0] - deltaZ[i0], ind[1][1][1] - deltaZ[i0]}, - {ind[1][1][0] - deltaR[i0], ind[1][2][0] - deltaZ[i0], ind[1][2][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i1] + deltaR[i0] + deltaZ[i0], ind[2][0][0] - deltaZ[i0], ind[2][0][1] - deltaZ[i0]}, - {ind[2][0][0] - deltaR[i0], ind[2][1][0] - deltaZ[i0], ind[2][1][1] - deltaZ[i0]}, - {ind[2][1][0] - deltaR[i0], ind[2][2][0] - deltaZ[i0], ind[2][2][1] - deltaZ[i0]}}, - {{ii_x_y_z + deltaPhi[i2] + deltaR[i0] + deltaZ[i0], ind[3][0][0] - deltaZ[i0], ind[3][0][1] - deltaZ[i0]}, - {ind[3][0][0] - deltaR[i0], ind[3][1][0] - deltaZ[i0], ind[3][1][1] - deltaZ[i0]}, - {ind[3][1][0] - deltaR[i0], ind[3][2][0] - deltaZ[i0], ind[3][2][1] - deltaZ[i0]}}}; - - cVals[0][0] = (*mGridData)[ind[0][0][0]]; - cVals[0][1] = (*mGridData)[ind[0][0][1]]; - cVals[0][2] = (*mGridData)[ind[0][0][2]]; - cVals[0][3] = extrapolation((*mGridData)[ind[0][0][2]], (*mGridData)[ind[0][0][1]], (*mGridData)[ind[0][0][0]]); - cVals[1][0] = (*mGridData)[ind[0][1][0]]; - cVals[1][1] = (*mGridData)[ind[0][1][1]]; - cVals[1][2] = (*mGridData)[ind[0][1][2]]; - cVals[1][3] = extrapolation((*mGridData)[ind[0][1][2]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][1][0]]); - cVals[2][0] = (*mGridData)[ind[0][2][0]]; - cVals[2][1] = (*mGridData)[ind[0][2][1]]; - cVals[2][2] = (*mGridData)[ind[0][2][2]]; - cVals[2][3] = extrapolation((*mGridData)[ind[0][2][2]], (*mGridData)[ind[0][2][1]], (*mGridData)[ind[0][2][0]]); - cVals[3][0] = extrapolation((*mGridData)[ind[0][2][0]], (*mGridData)[ind[0][1][0]], (*mGridData)[ind[0][0][0]]); - cVals[3][1] = extrapolation((*mGridData)[ind[0][2][1]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][0][1]]); - cVals[3][2] = extrapolation((*mGridData)[ind[0][2][2]], (*mGridData)[ind[0][1][2]], (*mGridData)[ind[0][0][2]]); - cVals[3][3] = extrapolation((*mGridData)[ind[0][2][2]], (*mGridData)[ind[0][1][1]], (*mGridData)[ind[0][0][0]]); - cVals[4][0] = (*mGridData)[ind[1][0][0]]; - cVals[4][1] = (*mGridData)[ind[1][0][1]]; - cVals[4][2] = (*mGridData)[ind[1][0][2]]; - cVals[4][3] = extrapolation((*mGridData)[ind[1][0][2]], (*mGridData)[ind[1][0][1]], (*mGridData)[ind[1][0][0]]); - cVals[5][0] = (*mGridData)[ind[1][1][0]]; - cVals[5][2] = (*mGridData)[ind[1][1][2]]; - cVals[5][3] = extrapolation((*mGridData)[ind[1][1][2]], (*mGridData)[ii_x_y_z], (*mGridData)[ind[1][1][0]]); - cVals[6][0] = (*mGridData)[ind[1][2][0]]; - cVals[6][1] = (*mGridData)[ind[1][2][1]]; - cVals[6][2] = (*mGridData)[ind[1][2][2]]; - cVals[6][3] = extrapolation((*mGridData)[ind[1][2][2]], (*mGridData)[ind[1][2][1]], (*mGridData)[ind[1][2][0]]); - cVals[7][0] = extrapolation((*mGridData)[ind[1][2][0]], (*mGridData)[ind[1][1][0]], (*mGridData)[ind[1][0][0]]); - cVals[7][1] = extrapolation((*mGridData)[ind[1][2][1]], (*mGridData)[ii_x_y_z], (*mGridData)[ind[1][0][1]]); - cVals[7][2] = extrapolation((*mGridData)[ind[1][2][2]], (*mGridData)[ind[1][1][2]], (*mGridData)[ind[1][0][2]]); - cVals[7][3] = extrapolation((*mGridData)[ind[1][2][2]], (*mGridData)[ind[1][1][1]], (*mGridData)[ind[1][0][0]]); - cVals[8][0] = (*mGridData)[ind[2][0][0]]; - cVals[8][1] = (*mGridData)[ind[2][0][1]]; - cVals[8][2] = (*mGridData)[ind[2][0][2]]; - cVals[8][3] = extrapolation((*mGridData)[ind[2][0][2]], (*mGridData)[ind[2][0][1]], (*mGridData)[ind[2][0][0]]); - cVals[9][0] = (*mGridData)[ind[2][1][0]]; - cVals[9][1] = (*mGridData)[ind[2][1][1]]; - cVals[9][2] = (*mGridData)[ind[2][1][2]]; - cVals[9][3] = extrapolation((*mGridData)[ind[2][1][2]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][1][0]]); - cVals[10][0] = (*mGridData)[ind[2][2][0]]; - cVals[10][1] = (*mGridData)[ind[2][2][1]]; - cVals[10][2] = (*mGridData)[ind[2][2][2]]; - cVals[10][3] = extrapolation((*mGridData)[ind[2][2][2]], (*mGridData)[ind[2][2][1]], (*mGridData)[ind[2][2][0]]); - cVals[11][0] = extrapolation((*mGridData)[ind[2][2][0]], (*mGridData)[ind[2][1][0]], (*mGridData)[ind[2][0][0]]); - cVals[11][1] = extrapolation((*mGridData)[ind[2][2][1]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][0][1]]); - cVals[11][2] = extrapolation((*mGridData)[ind[2][2][2]], (*mGridData)[ind[2][1][2]], (*mGridData)[ind[2][0][2]]); - cVals[11][3] = extrapolation((*mGridData)[ind[2][2][2]], (*mGridData)[ind[2][1][1]], (*mGridData)[ind[2][0][0]]); - cVals[12][0] = (*mGridData)[ind[3][0][0]]; - cVals[12][1] = (*mGridData)[ind[3][0][1]]; - cVals[12][2] = (*mGridData)[ind[3][0][2]]; - cVals[12][3] = extrapolation((*mGridData)[ind[3][0][2]], (*mGridData)[ind[3][0][1]], (*mGridData)[ind[3][0][0]]); - cVals[13][0] = (*mGridData)[ind[3][1][0]]; - cVals[13][1] = (*mGridData)[ind[3][1][1]]; - cVals[13][2] = (*mGridData)[ind[3][1][2]]; - cVals[13][3] = extrapolation((*mGridData)[ind[3][1][2]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][1][0]]); - cVals[14][0] = (*mGridData)[ind[3][2][0]]; - cVals[14][1] = (*mGridData)[ind[3][2][1]]; - cVals[14][2] = (*mGridData)[ind[3][2][2]]; - cVals[14][3] = extrapolation((*mGridData)[ind[3][2][2]], (*mGridData)[ind[3][2][1]], (*mGridData)[ind[3][2][0]]); - cVals[15][0] = extrapolation((*mGridData)[ind[3][2][0]], (*mGridData)[ind[3][1][0]], (*mGridData)[ind[3][0][0]]); - cVals[15][1] = extrapolation((*mGridData)[ind[3][2][1]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][0][1]]); - cVals[15][2] = extrapolation((*mGridData)[ind[3][2][2]], (*mGridData)[ind[3][1][2]], (*mGridData)[ind[3][0][2]]); - cVals[15][3] = extrapolation((*mGridData)[ind[3][2][2]], (*mGridData)[ind[3][1][1]], (*mGridData)[ind[3][0][0]]); - } break; + const int sliceOffset[4]{deltaPhi[0], 0, deltaPhi[1], deltaPhi[2]}; + const int strideR = mGridProperties->getDeltaDataIndex(1, FR); + const int strideZ = mGridProperties->getDeltaDataIndex(1, FZ); + const int dataIndex = data.getDataIndex(iz, ir, iphi); + + // value of the point with the relative indices (row - 1, col - 1) to the cell in r and z + const auto value = [&](const int slice, const int row, const int col) { return data[dataIndex + sliceOffset[slice] + (row - 1) * strideR + (col - 1) * strideZ]; }; + + for (int slice = 0; slice < 4; ++slice) { + for (int row = 0; row < 4; ++row) { + // points outside of the grid in r and/or z are extrapolated from the next 3 points towards the inside of the grid (along the diagonal if outside in r and z) + const int indexR = ir + row - 1; + const int stepR = (indexR < 0) ? 1 : ((indexR >= nR) ? -1 : 0); + for (int col = 0; col < 4; ++col) { + const int indexZ = iz + col - 1; + const int stepZ = (indexZ < 0) ? 1 : ((indexZ >= nZ) ? -1 : 0); + if ((stepR == 0) && (stepZ == 0)) { + cVals[4 * slice + row][col] = value(slice, row, col); + } else { + cVals[4 * slice + row][col] = extrapolation(value(slice, row + stepR, col + stepZ), value(slice, row + 2 * stepR, col + 2 * stepZ), value(slice, row + 3 * stepR, col + 3 * stepZ)); + } + } + } } } diff --git a/Detectors/TPC/spacecharge/test/testO2TPCPoissonSolver.cxx b/Detectors/TPC/spacecharge/test/testO2TPCPoissonSolver.cxx index 41c5d3ed5ed8c..86ad6b48998a0 100644 --- a/Detectors/TPC/spacecharge/test/testO2TPCPoissonSolver.cxx +++ b/Detectors/TPC/spacecharge/test/testO2TPCPoissonSolver.cxx @@ -29,15 +29,11 @@ namespace tpc { using DataT = double; // using float actually takes alot longer than double (doesnt converge when using float) -static constexpr DataT TOLERANCE = 3; // relative tolerance for 3D (maximum large error is at phi=90 since there the potential is 0!) -static constexpr DataT TOLERANCE2D = 8.5; // relative tolerance for 2D TODO check why the difference between numerical and analyticial is larger than for 3D! +static constexpr DataT TOLERANCE = 3; // relative tolerance (maximum large error is at phi=90 since there the potential is 0!) static constexpr DataT ABSTOLERANCE = 0.01; // absolute tolerance is taken at small values near 0 static constexpr unsigned short NR = 65; // grid in r static constexpr unsigned short NZ = 65; // grid in z static constexpr unsigned short NPHI = 180; // grid in phi -static constexpr unsigned short NR2D = 129; // grid in r -static constexpr unsigned short NZ2D = 129; // grid in z -static constexpr unsigned short NPHI2D = 1; // grid in phi /// Get phi vertex position for index in phi direction /// \param indexPhi index in phi direction @@ -153,22 +149,6 @@ void testAlmostEqualArray(o2::tpc::DataContainer3D& analytical, o2::tpc:: } } -template -void testAlmostEqualArray2D(o2::tpc::DataContainer3D& analytical, o2::tpc::DataContainer3D& numerical) -{ - for (size_t iPhi = 0; iPhi < numerical.getNPhi(); ++iPhi) { - for (size_t iR = 0; iR < numerical.getNR(); ++iR) { - for (size_t iZ = 0; iZ < numerical.getNZ(); ++iZ) { - if (std::fabs(analytical(iZ, iR, iPhi)) < ABSTOLERANCE) { - BOOST_CHECK_SMALL(numerical(iZ, iR, iPhi) - analytical(iZ, iR, iPhi), ABSTOLERANCE); - } else { - BOOST_CHECK_CLOSE(numerical(iZ, iR, iPhi), analytical(iZ, iR, iPhi), TOLERANCE2D); - } - } - } - } -} - template void poissonSolver3D() { @@ -191,56 +171,92 @@ void poissonSolver3D() // calculate numerical potential PoissonSolver poissonSolver(grid3D); - const int symmetry = 0; - poissonSolver.poissonSolver3D(potentialNumerical, charge, symmetry); + poissonSolver.poissonSolver3D(potentialNumerical, charge); // compare numerical with analytical solution of the potential testAlmostEqualArray(potentialAnalytical, potentialNumerical); } +/// solve the poisson equation in 3D for the analytical charge density and boundary potential +/// \param nR number of vertices in r +/// \param nZ number of vertices in z +/// \param nPhi number of vertices in phi +/// \param potentialNumerical numerical solution (output) +/// \param potentialAnalytical analytical solution (output) template -void poissonSolver2D() +void solvePoisson3D(const unsigned short nR, const unsigned short nZ, const unsigned short nPhi, o2::tpc::DataContainer3D& potentialNumerical, o2::tpc::DataContainer3D& potentialAnalytical) { using GridProp = GridProperties; - const ParamSpaceCharge params{NR2D, NZ2D, NPHI2D}; - const o2::tpc::RegularGrid3D grid3D{GridProp::ZMIN, GridProp::RMIN, GridProp::PHIMIN, GridProp::getGridSpacingZ(NZ2D), GridProp::getGridSpacingR(NR2D), GridProp::getGridSpacingPhi(NPHI2D), params}; - - using DataContainer = o2::tpc::DataContainer3D; - DataContainer potentialNumerical(NZ2D, NR2D, NPHI2D); - DataContainer potentialAnalytical(NZ2D, NR2D, NPHI2D); - DataContainer charge(NZ2D, NR2D, NPHI2D); + const ParamSpaceCharge params{nR, nZ, nPhi}; + const o2::tpc::RegularGrid3D grid3D{GridProp::ZMIN, GridProp::RMIN, GridProp::PHIMIN, GridProp::getGridSpacingZ(nZ), GridProp::getGridSpacingR(nR), GridProp::getGridSpacingPhi(nPhi), params}; + potentialNumerical = o2::tpc::DataContainer3D(nZ, nR, nPhi); + potentialAnalytical = o2::tpc::DataContainer3D(nZ, nR, nPhi); + o2::tpc::DataContainer3D charge(nZ, nR, nPhi); - // set the boudnary and charge for numerical poisson solver const o2::tpc::AnalyticalFields analyticalFields; setChargeDensityFromFormula(analyticalFields, grid3D, charge); setPotentialBoundaryFromFormula(analyticalFields, grid3D, potentialNumerical); - - // set analytical potential setPotentialFromFormula(analyticalFields, grid3D, potentialAnalytical); - // calculate numerical potential PoissonSolver poissonSolver(grid3D); - poissonSolver.poissonSolver2D(potentialNumerical, charge); + poissonSolver.poissonSolver3D(potentialNumerical, charge); +} - // compare numerical with analytical solution of the potential - testAlmostEqualArray2D(potentialAnalytical, potentialNumerical); +/// \return returns the maximum absolute difference of the two containers +template +DataT getMaxAbsDiff(const o2::tpc::DataContainer3D& a, const o2::tpc::DataContainer3D& b) +{ + DataT maxDiff = 0; + for (size_t i = 0; i < a.getNDataPoints(); ++i) { + maxDiff = std::max(maxDiff, static_cast(std::abs(a[i] - b[i]))); + } + return maxDiff; } BOOST_AUTO_TEST_CASE(PoissonSolver3D_test) { - o2::tpc::MGParameters::isFull3D = true; // 3D poissonSolver3D(); } -BOOST_AUTO_TEST_CASE(PoissonSolver3D2D_test) +BOOST_AUTO_TEST_CASE(PoissonSolver3D_phiCoarsening_test) { - o2::tpc::MGParameters::isFull3D = false; // 3D2D - poissonSolver3D(); + // for a coarse grid in phi the phi direction is weakly coupled and is not coarsened on the finest multi grid levels (see MGParameters::phiCoarseningThreshold). + // Check that the solution agrees with the one obtained when phi is always coarsened and with the analytical solution + const unsigned short nPhi = 90; + DataContainer3D potentialAnalytical; + DataContainer3D potentialDefault; + solvePoisson3D(NR, NZ, nPhi, potentialDefault, potentialAnalytical); + + const float thresholdDefault = o2::tpc::MGParameters::phiCoarseningThreshold; + o2::tpc::MGParameters::phiCoarseningThreshold = 0; // always coarsen phi + DataContainer3D potentialAlwaysCoarsen; + solvePoisson3D(NR, NZ, nPhi, potentialAlwaysCoarsen, potentialAnalytical); + o2::tpc::MGParameters::phiCoarseningThreshold = thresholdDefault; + + const DataT maxPotential = getMaxAbsDiff(potentialAnalytical, DataContainer3D(NZ, NR, nPhi)); + BOOST_CHECK_SMALL(getMaxAbsDiff(potentialDefault, potentialAlwaysCoarsen) / maxPotential, DataT(1e-6)); + BOOST_CHECK_SMALL(getMaxAbsDiff(potentialDefault, potentialAnalytical) / maxPotential, DataT(5e-4)); + BOOST_CHECK_SMALL(getMaxAbsDiff(potentialAlwaysCoarsen, potentialAnalytical) / maxPotential, DataT(5e-4)); } -BOOST_AUTO_TEST_CASE(PoissonSolver2D_test) +BOOST_AUTO_TEST_CASE(PoissonSolver3D_linePhi_test) { - poissonSolver2D(); + // for a fine grid in phi the phi direction is strongly coupled and line relaxation in phi is used on the finest multi grid levels (see MGParameters::phiLineRelaxThreshold). + // Check that the solution agrees with the one obtained with point relaxation only and with the analytical solution + const unsigned short nPhi = 720; + DataContainer3D potentialAnalytical; + DataContainer3D potentialLine; + solvePoisson3D(NR, NZ, nPhi, potentialLine, potentialAnalytical); + + const float thresholdDefault = o2::tpc::MGParameters::phiLineRelaxThreshold; + o2::tpc::MGParameters::phiLineRelaxThreshold = 0; // point relaxation only + DataContainer3D potentialPoint; + solvePoisson3D(NR, NZ, nPhi, potentialPoint, potentialAnalytical); + o2::tpc::MGParameters::phiLineRelaxThreshold = thresholdDefault; + + const DataT maxPotential = getMaxAbsDiff(potentialAnalytical, DataContainer3D(NZ, NR, nPhi)); + BOOST_CHECK_SMALL(getMaxAbsDiff(potentialLine, potentialPoint) / maxPotential, DataT(1e-6)); + BOOST_CHECK_SMALL(getMaxAbsDiff(potentialLine, potentialAnalytical) / maxPotential, DataT(5e-4)); } } // namespace tpc diff --git a/Detectors/TPC/spacecharge/test/testO2TPCSpaceCharge.cxx b/Detectors/TPC/spacecharge/test/testO2TPCSpaceCharge.cxx new file mode 100644 index 0000000000000..a4b5a23ded653 --- /dev/null +++ b/Detectors/TPC/spacecharge/test/testO2TPCSpaceCharge.cxx @@ -0,0 +1,234 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file testO2TPCSpaceCharge.cxx +/// \brief closure tests of the global distortions and corrections calculated from analytical electric fields, reading of stored objects + +#define BOOST_TEST_MODULE Test TPC SpaceCharge class +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK +#include +#include "TPCSpaceCharge/SpaceCharge.h" +#include "TPCSpaceCharge/SpaceChargeHelpers.h" +#include "TMemFile.h" +#include +#include +#include +#include +#include + +namespace o2 +{ +namespace tpc +{ + +using DataT = double; +using SC = SpaceCharge; + +namespace +{ +struct Residuals { + DataT rms[3]{}; ///< rms of the residuals in r, rphi, z + DataT max[3]{}; ///< maximum absolute residual in r, rphi, z +}; + +/// distort random points with the global distortions and correct them with the global corrections: the original point should be recovered +Residuals closure(const SC& sc, const Side side) +{ + std::mt19937 gen(42); + std::uniform_real_distribution distR(95, 240); + std::uniform_real_distribution distPhi(0, 2 * M_PI); + std::uniform_real_distribution distZ(5, 245); + const int nPoints = 20000; + Residuals res; + for (int i = 0; i < nPoints; ++i) { + const DataT r = distR(gen); + const DataT phi = distPhi(gen); + const DataT z = (side == Side::A) ? distZ(gen) : -distZ(gen); + DataT dZ, dR, dRPhi; + sc.getDistortionsCyl(z, r, phi, side, dZ, dR, dRPhi); + const DataT rDist = r + dR; + const DataT phiDist = phi + dRPhi / r; + const DataT zDist = z + dZ; + DataT cZ, cR, cRPhi; + sc.getCorrectionsCyl(zDist, rDist, phiDist, side, cZ, cR, cRPhi); + const DataT diff[3]{rDist + cR - r, (phiDist + cRPhi / rDist - phi) * r, zDist + cZ - z}; + for (int k = 0; k < 3; ++k) { + res.rms[k] += diff[k] * diff[k]; + res.max[k] = std::max(res.max[k], std::abs(diff[k])); + } + } + for (int k = 0; k < 3; ++k) { + res.rms[k] = std::sqrt(res.rms[k] / nPoints); + } + return res; +} + +struct GlobalDist { + std::vector r, rphi, z; +}; + +GlobalDist getGlobalDistortions(const SC& sc, const Side side, const int nZ, const int nR, const int nPhi) +{ + GlobalDist dist; + for (int iPhi = 0; iPhi < nPhi; ++iPhi) { + for (int iR = 0; iR < nR; ++iR) { + for (int iZ = 0; iZ < nZ; ++iZ) { + dist.r.emplace_back(sc.getGlobalDistR(iZ, iR, iPhi, side)); + dist.rphi.emplace_back(sc.getGlobalDistRPhi(iZ, iR, iPhi, side)); + dist.z.emplace_back(sc.getGlobalDistZ(iZ, iR, iPhi, side)); + } + } + } + return dist; +} +} // namespace + +/// global distortions with the standard method (integration of local distortions) and with the fast method (inversion of the global corrections) +/// have to be consistent with the global corrections and with each other +BOOST_AUTO_TEST_CASE(SpaceCharge_closure_test) +{ + const int nZ = 65; + const int nR = 65; + const int nPhi = 90; + SC::setNThreads(4); + for (const Side side : {Side::A, Side::C}) { + SC sc(0, nZ, nR, nPhi); + AnalyticalFields fields(side); + fields.setParameters(5e-7, 0.5, 1e-4); // global distortions of O(cm) + + sc.calcLocalDistortionsCorrections(SC::Type::Distortions, fields); + sc.calcLocalDistortionsCorrections(SC::Type::Corrections, fields); + sc.calcGlobalCorrections(sc.getLocalCorrInterpolator(side)); + + // standard method + sc.calcGlobalDistortions(sc.getLocalDistInterpolator(side)); + const GlobalDist distStd = getGlobalDistortions(sc, side, nZ, nR, nPhi); + const Residuals resStd = closure(sc, side); + + // fast method + sc.calcGlobalDistWithGlobalCorrIterative(sc.getGlobalCorrInterpolator(side)); + const GlobalDist distFast = getGlobalDistortions(sc, side, nZ, nR, nPhi); + const Residuals resFast = closure(sc, side); + + // magnitude of the distortions and difference between the methods (the first z vertex of the fast method is extrapolated) + DataT maxDist[3]{}; + DataT meanDiff[3]{}; + DataT maxDiff[3]{}; + int nDiff = 0; + for (size_t i = 0; i < distStd.z.size(); ++i) { + const DataT valStd[3]{distStd.r[i], distStd.rphi[i], distStd.z[i]}; + const DataT valFast[3]{distFast.r[i], distFast.rphi[i], distFast.z[i]}; + if ((i % nZ) == 0) { + continue; + } + ++nDiff; + for (int k = 0; k < 3; ++k) { + maxDist[k] = std::max(maxDist[k], std::abs(valStd[k])); + meanDiff[k] += valStd[k] - valFast[k]; + maxDiff[k] = std::max(maxDiff[k], std::abs(valStd[k] - valFast[k])); + } + } + for (int k = 0; k < 3; ++k) { + meanDiff[k] /= nDiff; + } + + BOOST_TEST_MESSAGE("side " << static_cast(side) << " max global distortion (r, rphi, z) [cm]: " << maxDist[0] << " " << maxDist[1] << " " << maxDist[2]); + BOOST_TEST_MESSAGE(" closure standard rms: " << resStd.rms[0] << " " << resStd.rms[1] << " " << resStd.rms[2] << " max: " << resStd.max[0] << " " << resStd.max[1] << " " << resStd.max[2]); + BOOST_TEST_MESSAGE(" closure fast rms: " << resFast.rms[0] << " " << resFast.rms[1] << " " << resFast.rms[2] << " max: " << resFast.max[0] << " " << resFast.max[1] << " " << resFast.max[2]); + BOOST_TEST_MESSAGE(" standard - fast mean: " << meanDiff[0] << " " << meanDiff[1] << " " << meanDiff[2] << " max: " << maxDiff[0] << " " << maxDiff[1] << " " << maxDiff[2]); + + // the distortions must be large enough for a meaningful test + BOOST_CHECK(maxDist[0] > 0.5); + BOOST_CHECK(maxDist[2] > 0.1); + // closure and agreement of the methods: O(0.01 um) for these smooth fields, tolerance 0.2 um (rms) and 0.1 um (mean difference) + for (int k = 0; k < 3; ++k) { + BOOST_CHECK_SMALL(resStd.rms[k], 2e-5); + BOOST_CHECK_SMALL(resFast.rms[k], 2e-5); + BOOST_CHECK_SMALL(meanDiff[k], 1e-5); + } + } +} + +/// unphysically large fields: the electrons drift out of the grid during the calculation of the global corrections. +/// The calculation must not access memory outside of the grid, invalid positions give NaN values +BOOST_AUTO_TEST_CASE(SpaceCharge_largeFields_test) +{ + const int nZ = 33; + const int nR = 33; + const int nPhi = 36; + SC::setNThreads(4); + const Side side = Side::A; + SC sc(0, nZ, nR, nPhi); + AnalyticalFields fields(side); + fields.setParameters(5e-5, 0.5, 1e-4); // local corrections of several cm per z bin + sc.calcLocalDistortionsCorrections(SC::Type::Corrections, fields); + sc.calcGlobalCorrections(sc.getLocalCorrInterpolator(side)); + sc.calcGlobalDistWithGlobalCorrIterative(sc.getGlobalCorrInterpolator(side)); + + // query at a non-finite position + DataT cZ, cR, cRPhi; + sc.getCorrectionsCyl(std::numeric_limits::quiet_NaN(), 150, 1, side, cZ, cR, cRPhi); + BOOST_CHECK(std::isnan(cZ) && std::isnan(cR) && std::isnan(cRPhi)); + sc.getCorrectionsCyl(100, 150, std::numeric_limits::infinity(), side, cZ, cR, cRPhi); + BOOST_CHECK(std::isnan(cZ) && std::isnan(cR) && std::isnan(cRPhi)); + // query at a phi far outside of the grid (periodic): has to terminate and give the same value as the phi in the grid + DataT cZFar, cRFar, cRPhiFar; + sc.getCorrectionsCyl(100, 150, 1e300, side, cZFar, cRFar, cRPhiFar); + const DataT periodPhi = nPhi * sc.getGridSpacingPhi(side); // period of the grid (not exactly 2 pi in double precision) + sc.getCorrectionsCyl(100, 150, 1 + 1000 * periodPhi, side, cZFar, cRFar, cRPhiFar); + sc.getCorrectionsCyl(100, 150, 1, side, cZ, cR, cRPhi); + BOOST_CHECK_CLOSE(cRFar, cR, 1e-6); +} + +/// write the object to a file and read it back: the grid (transient) has to be set up from the stored number of vertices +/// without calling initAfterReadingFromFile() (read rule in the LinkDef) +template +void checkReadFromFile() +{ + const int nZ = 33; + const int nR = 65; + const int nPhi = 90; // not the default number of vertices + const Side side = Side::A; + SpaceCharge sc(0, nZ, nR, nPhi); + AnalyticalFields fields(side); + fields.setParameters(5e-7, 0.5, 1e-4); + sc.calcLocalDistortionsCorrections(SpaceCharge::Type::Corrections, fields); + sc.calcGlobalCorrections(sc.getLocalCorrInterpolator(side)); + + TMemFile file("sc.root", "RECREATE"); + file.WriteObject(&sc, "sc"); + std::unique_ptr> scRead(file.Get>("sc")); + BOOST_REQUIRE(scRead); + BOOST_CHECK_EQUAL(scRead->getNPhiVertices(), nPhi); + BOOST_CHECK_EQUAL(scRead->getGridSpacingPhi(side), sc.getGridSpacingPhi(side)); + BOOST_CHECK_EQUAL(scRead->getGridSpacingR(side), sc.getGridSpacingR(side)); + BOOST_CHECK_EQUAL(scRead->getGridSpacingZ(side), sc.getGridSpacingZ(side)); + for (const T phi : {0.1, 1.3, 4.7}) { + T cZ, cR, cRPhi, cZRead, cRRead, cRPhiRead; + sc.getCorrectionsCyl(120, 150, phi, side, cZ, cR, cRPhi); + scRead->getCorrectionsCyl(120, 150, phi, side, cZRead, cRRead, cRPhiRead); + BOOST_CHECK_EQUAL(cRRead, cR); + BOOST_CHECK_EQUAL(cRPhiRead, cRPhi); + BOOST_CHECK_EQUAL(cZRead, cZ); + } +} + +BOOST_AUTO_TEST_CASE(SpaceCharge_readFromFile_test) +{ + SC::setNThreads(4); + SpaceCharge::setNThreads(4); + checkReadFromFile(); + checkReadFromFile(); +} + +} // namespace tpc +} // namespace o2 diff --git a/Detectors/TPC/spacecharge/test/testO2TPCTricubic.cxx b/Detectors/TPC/spacecharge/test/testO2TPCTricubic.cxx index 3e9d9b6c58f47..22db057e5c21a 100644 --- a/Detectors/TPC/spacecharge/test/testO2TPCTricubic.cxx +++ b/Detectors/TPC/spacecharge/test/testO2TPCTricubic.cxx @@ -22,6 +22,8 @@ #include "TPCSpaceCharge/DataContainer3D.h" #include "TPCSpaceCharge/PoissonSolverHelpers.h" #include "TPCSpaceCharge/SpaceChargeHelpers.h" +#include +#include namespace o2 { @@ -76,7 +78,7 @@ BOOST_AUTO_TEST_CASE(PoissonSolver3D_test) const int nphiPointsLoop = NPHI * nFacLoop; const DataT rSpacingLoop = GridProperties::getGridSpacingR(nrPointsLoop); const DataT zSpacingLoop = GridProperties::getGridSpacingZ(nzPointsLoop); - const DataT phiSpacingLoop = GridProperties::getGridSpacingZ(nphiPointsLoop); + const DataT phiSpacingLoop = GridProperties::getGridSpacingPhi(nphiPointsLoop); for (int iR = -2; iR < nrPointsLoop + 2; ++iR) { const DataT r = rmin + iR * rSpacingLoop; @@ -99,5 +101,99 @@ BOOST_AUTO_TEST_CASE(PoissonSolver3D_test) } } +BOOST_AUTO_TEST_CASE(TriCubic_vertices_test) +{ + // the tricubic interpolation has to reproduce the values at the vertices of the grid: checks the indexing in the inner volume and at the boundaries of the grid (extrapolation) + const ParamSpaceCharge params{NR, NZ, NPHI}; + const DataT rSpacing = o2::tpc::GridProperties::getGridSpacingR(NR); + const DataT zSpacing = o2::tpc::GridProperties::getGridSpacingZ(NZ); + const DataT phiSpacing = o2::tpc::GridProperties::getGridSpacingPhi(NPHI); + const o2::tpc::RegularGrid3D grid3D(GridProperties::ZMIN, GridProperties::RMIN, GridProperties::PHIMIN, zSpacing, rSpacing, phiSpacing, params); + o2::tpc::DataContainer3D data3D(NZ, NR, NPHI); + + // fill with values without any symmetry + for (int iz = 0; iz < NZ; ++iz) { + for (int ir = 0; ir < NR; ++ir) { + for (int iphi = 0; iphi < NPHI; ++iphi) { + data3D(iz, ir, iphi) = 1 + iz + 1000 * ir + 1e6 * iphi + std::sin(iz * ir + iphi); + } + } + } + + for (const bool extrapolate : {true, false}) { + o2::tpc::TriCubicInterpolator interpolator(data3D, grid3D); + interpolator.setExtrapolateValues(extrapolate); + for (int iz = 0; iz < NZ; ++iz) { + for (int ir = 0; ir < NR; ++ir) { + for (int iphi = 0; iphi < NPHI; ++iphi) { + const DataT interpolated = interpolator(grid3D.getZVertex(iz), grid3D.getRVertex(ir), grid3D.getPhiVertex(iphi)); + BOOST_CHECK_CLOSE(interpolated, data3D(iz, ir, iphi), 1e-8); + } + } + } + } +} + +BOOST_AUTO_TEST_CASE(TriCubic_threeComponents_test) +{ + // the combined interpolation of three data containers has to give the same results as three separate interpolations + const ParamSpaceCharge params{NR, NZ, NPHI}; + const DataT rSpacing = o2::tpc::GridProperties::getGridSpacingR(NR); + const DataT zSpacing = o2::tpc::GridProperties::getGridSpacingZ(NZ); + const DataT phiSpacing = o2::tpc::GridProperties::getGridSpacingPhi(NPHI); + const o2::tpc::RegularGrid3D grid3D(GridProperties::ZMIN, GridProperties::RMIN, GridProperties::PHIMIN, zSpacing, rSpacing, phiSpacing, params); + o2::tpc::DataContainer3D dataR(NZ, NR, NPHI); + o2::tpc::DataContainer3D dataZ(NZ, NR, NPHI); + o2::tpc::DataContainer3D dataPhi(NZ, NR, NPHI); + o2::tpc::AnalyticalFields field; + for (int iz = 0; iz < NZ; ++iz) { + for (int ir = 0; ir < NR; ++ir) { + for (int iphi = 0; iphi < NPHI; ++iphi) { + const DataT z = grid3D.getZVertex(iz); + const DataT r = grid3D.getRVertex(ir); + const DataT phi = grid3D.getPhiVertex(iphi); + dataR(iz, ir, iphi) = field.evalFieldR(z, r, phi); + dataZ(iz, ir, iphi) = field.evalFieldZ(z, r, phi); + dataPhi(iz, ir, iphi) = field.evalFieldPhi(z, r, phi); + } + } + } + + // the combined interpolation uses the same operations as the separate interpolations, but the compiler may contract + // multiplications and additions (FMA) differently in the two loops: compare with a tolerance at the rounding level + const auto maxAbs = [](const auto& data) { DataT maxVal = 0; for (const auto val : data.getData()) { maxVal = std::max(maxVal, std::abs(val)); } return maxVal; }; + const DataT tolR = 1e-12 * maxAbs(dataR); + const DataT tolZ = 1e-12 * maxAbs(dataZ); + const DataT tolPhi = 1e-12 * maxAbs(dataPhi); + + for (const bool extrapolate : {true, false}) { + o2::tpc::TriCubicInterpolator interpolatorR(dataR, grid3D); + o2::tpc::TriCubicInterpolator interpolatorZ(dataZ, grid3D); + o2::tpc::TriCubicInterpolator interpolatorPhi(dataPhi, grid3D); + interpolatorR.setExtrapolateValues(extrapolate); + interpolatorZ.setExtrapolateValues(extrapolate); + interpolatorPhi.setExtrapolateValues(extrapolate); + + // query points inside and outside of the grid + const int nPoints = 40; + for (int iR = -2; iR < nPoints + 2; ++iR) { + const DataT r = GridProperties::RMIN + iR * (GridProperties::RMAX - GridProperties::RMIN) / nPoints; + for (int iZ = -2; iZ < nPoints + 2; ++iZ) { + const DataT z = GridProperties::ZMIN + iZ * (GridProperties::ZMAX - GridProperties::ZMIN) / nPoints; + for (int iPhi = -2; iPhi < nPoints + 2; ++iPhi) { + const DataT phi = iPhi * GridProperties::PHIMAX / nPoints; + DataT valR = 0; + DataT valZ = 0; + DataT valPhi = 0; + interpolatorR.interpolate(z, r, phi, interpolatorZ, interpolatorPhi, valR, valZ, valPhi); + BOOST_CHECK_SMALL(valR - interpolatorR(z, r, phi), tolR); + BOOST_CHECK_SMALL(valZ - interpolatorZ(z, r, phi), tolZ); + BOOST_CHECK_SMALL(valPhi - interpolatorPhi(z, r, phi), tolPhi); + } + } + } + } +} + } // namespace tpc } // namespace o2 diff --git a/GPU/TPCFastTransformation/CMakeLists.txt b/GPU/TPCFastTransformation/CMakeLists.txt index d48f7660c45b9..7fbf0ce689216 100644 --- a/GPU/TPCFastTransformation/CMakeLists.txt +++ b/GPU/TPCFastTransformation/CMakeLists.txt @@ -47,7 +47,7 @@ if(${ALIGPU_BUILD_TYPE} STREQUAL "O2") PUBLIC_LINK_LIBRARIES O2::GPUCommon O2::GPUUtils O2::GPUDataTypes - O2::TPCSpaceCharge + O2::DataFormatsTPC Vc::Vc ROOT::Core ROOT::Matrix ROOT::Tree ROOT::Gpad ROOT::Minuit ) diff --git a/GPU/TPCFastTransformation/TPCFastTransform.cxx b/GPU/TPCFastTransformation/TPCFastTransform.cxx index 4435505472991..1acab94782324 100644 --- a/GPU/TPCFastTransformation/TPCFastTransform.cxx +++ b/GPU/TPCFastTransformation/TPCFastTransform.cxx @@ -28,10 +28,6 @@ #include "GPUCommonLogger.h" #endif -#if !defined(GPUCA_STANDALONE) -#include "TPCSpaceCharge/SpaceCharge.h" -#endif - using namespace o2::gpu; TPCFastTransform::TPCFastTransform() @@ -220,27 +216,6 @@ TPCFastTransform* TPCFastTransform::loadFromFile(std::string inpFName, std::stri #endif -#if !defined(GPUCA_STANDALONE) -TPCSlowSpaceChargeCorrection::~TPCSlowSpaceChargeCorrection() -{ - delete mCorr; -} - -void TPCSlowSpaceChargeCorrection::getCorrections(const float gx, const float gy, const float gz, const int32_t slice, float& gdxC, float& gdyC, float& gdzC) const -{ - const o2::tpc::Side side = (slice < o2::tpc::SECTORSPERSIDE) ? o2::tpc::Side::A : o2::tpc::Side::C; - mCorr->getCorrections(gx, gy, gz, side, gdxC, gdyC, gdzC); -} - -void TPCFastTransform::setSlowTPCSCCorrection(TFile& inpf) -{ - mCorrectionSlow = new TPCSlowSpaceChargeCorrection; - mCorrectionSlow->mCorr = new o2::tpc::SpaceCharge(); - mCorrectionSlow->mCorr->setGlobalCorrectionsFromFile(inpf, o2::tpc::Side::A); - mCorrectionSlow->mCorr->setGlobalCorrectionsFromFile(inpf, o2::tpc::Side::C); -} -#endif - float TPCFastTransform::getIDC() const { auto val = mIDC; diff --git a/GPU/TPCFastTransformation/TPCFastTransform.h b/GPU/TPCFastTransformation/TPCFastTransform.h index 1a33170a600ec..a55a9e39ebc21 100644 --- a/GPU/TPCFastTransformation/TPCFastTransform.h +++ b/GPU/TPCFastTransformation/TPCFastTransform.h @@ -29,39 +29,14 @@ #include #endif // !GPUCA_GPUCODE -namespace o2::tpc -{ -template -class SpaceCharge; -} - namespace o2::gpu { -/// simple struct to hold the space charge object which can be used for CPU reconstruction only +/// Empty stub of the removed slow space-charge correction. TPCFastTransform objects written with ClassDef version <= 5 (e.g. the existing CCDB correction maps) +/// contain the streamer info of this class. Without a dictionary for it, ROOT prints "no dictionary for class o2::gpu::TPCSlowSpaceChargeCorrection is available" when opening such a file. +/// Do not use, only kept for reading old files. struct TPCSlowSpaceChargeCorrection { - -#if !defined(GPUCA_GPUCODE) && !defined(GPUCA_STANDALONE) - /// destructor - ~TPCSlowSpaceChargeCorrection(); - - /// getting the corrections for global coordinates - void getCorrections(const float gx, const float gy, const float gz, const int32_t sector, float& gdxC, float& gdyC, float& gdzC) const; - - o2::tpc::SpaceCharge* mCorr{nullptr}; ///< reference space charge corrections -#else - ~TPCSlowSpaceChargeCorrection() = default; - - /// setting dummy corrections for GPU - GPUdi() void getCorrections(const float gx, const float gy, const float gz, const int32_t sector, float& gdxC, float& gdyC, float& gdzC) const - { - gdxC = 0; - gdyC = 0; - gdzC = 0; - } -#endif - - ClassDefNV(TPCSlowSpaceChargeCorrection, 2); + ClassDefNV(TPCSlowSpaceChargeCorrection, 3); }; /// @@ -116,15 +91,8 @@ class TPCFastTransform : public FlatObject FlatObject::destroy(); } -/// Destructor -#if !defined(GPUCA_GPUCODE) - ~TPCFastTransform() - { - delete mCorrectionSlow; - } -#else + /// Destructor ~TPCFastTransform() = default; -#endif /// _____________ FlatObject functionality, see FlatObject class for description ____________ @@ -279,12 +247,6 @@ class TPCFastTransform : public FlatObject static TPCFastTransform* loadFromFile(std::string inpFName = "", std::string name = ""); - /// setting the reference corrections - void setSlowTPCSCCorrection(TFile& inpf); - - /// \return returns the space charge object which is used for the slow correction - const auto& getCorrectionSlow() const { return *mCorrectionSlow; } - #endif // !GPUCA_GPUCODE /// Print method @@ -332,12 +294,9 @@ class TPCFastTransform : public FlatObject float mIDCError; ///< error on IDC float mCTP2IDCFallBackThreshold; ///< if IDC is not set but requested, use Lumi if it does not exceed this threshold - /// Correction of (x,u,v) with tricubic interpolator on a regular grid - TPCSlowSpaceChargeCorrection* mCorrectionSlow{nullptr}; ///< reference space charge corrections - GPUd() void TransformLocal(int32_t sector, int32_t row, float& x, float& y, float& z, const TPCFastTransform* ref, const TPCFastTransform* ref2, float scale, float scale2, int32_t scaleMode) const; - ClassDefNV(TPCFastTransform, 5); + ClassDefNV(TPCFastTransform, 6); }; // ======================================================================= @@ -412,15 +371,6 @@ GPUdi() void TPCFastTransform::TransformLocal(int32_t sector, int32_t row, float float dx = 0.f, dy = 0.f, dz = 0.f; if ((scale >= 0.f) || (scaleMode == 1) || (scaleMode == 2)) { -#ifndef GPUCA_GPUCODE - if (mCorrectionSlow) { - float gx, gy, gz; - getGeometry().convLocalToGlobal(sector, x, y, z, gx, gy, gz); - float gdxC, gdyC, gdzC; - mCorrectionSlow->getCorrections(gx, gy, gz, sector, gdxC, gdyC, gdzC); - getGeometry().convGlobalToLocal(sector, gdxC, gdyC, gdzC, dx, dy, dz); - } else -#endif // GPUCA_GPUCODE { mCorrection.getCorrectionLocal(sector, row, y, z, dx, dy, dz); if (ref) { diff --git a/GPU/TPCFastTransformation/TPCFastTransformationLinkDef_O2.h b/GPU/TPCFastTransformation/TPCFastTransformationLinkDef_O2.h index c9f89f3dcac76..3fe4cc40b9045 100644 --- a/GPU/TPCFastTransformation/TPCFastTransformationLinkDef_O2.h +++ b/GPU/TPCFastTransformation/TPCFastTransformationLinkDef_O2.h @@ -53,6 +53,7 @@ #pragma read sourceClass = "o2::gpu::TPCFastSpaceChargeCorrection" targetClass = "o2::gpu::TPCFastSpaceChargeCorrection" source = "size_t mSliceDataSizeBytes[3]" version = "[-3]" target = "mSectorDataSizeBytes" code = "{ for (int i=0; i<3; i++) mSectorDataSizeBytes[i] = onfile.mSliceDataSizeBytes[i]; }"; #pragma read sourceClass = "o2::gpu::TPCFastSpaceChargeCorrection" targetClass = "o2::gpu::TPCFastSpaceChargeCorrection" source = "float fInterpolationSafetyMargin" version = "[-3]" target = "" code = "{}"; +// empty stub of the removed class, only kept to read old TPCFastTransform objects without a missing-dictionary warning (see TPCFastTransform.h) #pragma link C++ struct o2::gpu::TPCSlowSpaceChargeCorrection + ; #pragma link C++ class o2::gpu::TPCFastTransformPOD + ;