Add particle-grid periodic migrate

CMakeLists.txt

@@ -168,6 +168,19 @@ if(Cabana_ENABLE_CORE)
   add_subdirectory(core)
 endif()
 
+option(Cabana_ENABLE_PARTICLEGRID "Build ParticleGrid" ${Cabana_ENABLE_CAJITA})
+if(Cabana_ENABLE_PARTICLEGRID)
+  if(Cabana_ENABLE_CORE)
+    if(Cabana_ENABLE_CAJITA)
+      add_subdirectory(particle_grid)
+    else()
+      message(FATAL_ERROR "Cabana Particle-Grid requires Cajita")
+    endif()
+  else()
+    message(FATAL_ERROR "Cabana Particle-Grid requires Core")
+  endif()
+endif()
+
 option(Cabana_ENABLE_EXAMPLES "Build examples" OFF)
 if(Cabana_ENABLE_EXAMPLES)
   add_subdirectory(example)
@@ -192,7 +205,7 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/CabanaConfig.cmake" "${CMAKE_CURRENT_
 ## Clang Format
 ##---------------------------------------------------------------------------##
 if(CLANG_FORMAT_FOUND)
-  file(GLOB_RECURSE FORMAT_SOURCES core/*.cpp core/*.hpp cajita/*hpp cajita/*cpp)
+  file(GLOB_RECURSE FORMAT_SOURCES core/*.cpp core/*.hpp cajita/*hpp cajita/*cpp particle_grid/*.hpp)
   add_custom_target(format
     COMMAND ${CLANG_FORMAT_EXECUTABLE} -i -style=file ${FORMAT_SOURCES}
     DEPENDS ${FORMAT_SOURCES})

cajita/unit_test/CMakeLists.txt

@@ -38,7 +38,7 @@ macro(Cajita_add_tests)
       list(APPEND CAJITA_UNIT_TEST_NUMTHREADS ${_nt})
     endif()
   endforeach()
-  set(CAJITA_UNIT_TEST_MAIN unit_test_main.cpp)
+  set(CAJITA_UNIT_TEST_MAIN ../../core/unit_test/mpi_unit_test_main.cpp)
   foreach(_device ${CAJITA_TEST_DEVICES})
     set(_dir ${CMAKE_CURRENT_BINARY_DIR}/${_device})
     file(MAKE_DIRECTORY ${_dir})

particle_grid/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_subdirectory(src)
+
+if(Cabana_ENABLE_TESTING)
+  add_subdirectory(unit_test)
+endif()

particle_grid/src/CMakeLists.txt

@@ -0,0 +1,30 @@
+set(HEADERS_PUBLIC
+  CabanaPG.hpp
+  Cabana_Periodic.hpp
+  )
+
+add_library(CabanaPG INTERFACE)
+
+target_link_libraries(CabanaPG INTERFACE
+  cabanacore
+  Cajita
+  )
+
+target_include_directories(CabanaPG
+  INTERFACE
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
+
+install(TARGETS CabanaPG
+  EXPORT CabanaPGTargets
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
+
+install(EXPORT CabanaPGTargets
+  FILE CabanaPGTargets.cmake
+  NAMESPACE Cabana::
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Cabana)
+
+install(FILES ${HEADERS_PUBLIC}
+  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})

cajita/unit_test/unit_test_main.cpp → particle_grid/src/CabanaPG.hpp

@@ -9,19 +9,9 @@
  * SPDX-License-Identifier: BSD-3-Clause                                    *
  ****************************************************************************/
 
-#include <gtest/gtest.h>
+#ifndef CABANAPG_HPP
+#define CABANAPG_HPP
 
-#include <Kokkos_Core.hpp>
+#include <Cabana_Periodic.hpp>
 
-#include <mpi.h>
-
-int main( int argc, char *argv[] )
-{
-    MPI_Init( &argc, &argv );
-    Kokkos::initialize( argc, argv );
-    ::testing::InitGoogleTest( &argc, argv );
-    int return_val = RUN_ALL_TESTS();
-    Kokkos::finalize();
-    MPI_Finalize();
-    return return_val;
-}
+#endif // end CABANAPG_HPP

particle_grid/src/Cabana_Periodic.hpp

@@ -0,0 +1,223 @@
+/****************************************************************************
+ * Copyright (c) 2018-2020 by the Cabana authors                            *
+ * All rights reserved.                                                     *
+ *                                                                          *
+ * This file is part of the Cabana library. Cabana is distributed under a   *
+ * BSD 3-clause license. For the licensing terms see the LICENSE file in    *
+ * the top-level directory.                                                 *
+ *                                                                          *
+ * SPDX-License-Identifier: BSD-3-Clause                                    *
+ ****************************************************************************/
+
+#ifndef CABANA_PERIODIC_HPP
+#define CABANA_PERIODIC_HPP
+
+#include <Cabana_Core.hpp>
+#include <Cajita.hpp>
+
+#include <Kokkos_Core.hpp>
+
+#include <vector>
+
+namespace CabanaPG
+{
+
+//---------------------------------------------------------------------------//
+template <class LocalGridType, class PositionSliceType, class NeighborRankView,
+          class DestinationRankView>
+void periodic_shift( const LocalGridType &local_grid,
+                     const NeighborRankView &neighbor_ranks,
+                     DestinationRankView &destinations,
+                     PositionSliceType &positions )
+{
+    using execution_space = typename PositionSliceType::execution_space;
+
+    // Locate the particles in the global grid and get their destination
+    // rank. The particle halo should be constructed such that particles will
+    // only move to a location in the 26 neighbor halo or stay on this
+    // rank. If the particle crosses a periodic boundary update it's
+    // positions to represent the shift.
+    auto local_mesh = Cajita::createLocalMesh<Kokkos::HostSpace>( local_grid );
+    const auto &global_grid = local_grid.globalGrid();
+    const auto &global_mesh = global_grid.globalMesh();
+    const Kokkos::Array<double, 3> local_low = {
+        local_mesh.lowCorner( Cajita::Own(), Cajita::Dim::I ),
+        local_mesh.lowCorner( Cajita::Own(), Cajita::Dim::J ),
+        local_mesh.lowCorner( Cajita::Own(), Cajita::Dim::K )};
+    const Kokkos::Array<double, 3> local_high = {
+        local_mesh.highCorner( Cajita::Own(), Cajita::Dim::I ),
+        local_mesh.highCorner( Cajita::Own(), Cajita::Dim::J ),
+        local_mesh.highCorner( Cajita::Own(), Cajita::Dim::K )};
+    const Kokkos::Array<bool, 3> period = {
+        global_grid.isPeriodic( Cajita::Dim::I ),
+        global_grid.isPeriodic( Cajita::Dim::J ),
+        global_grid.isPeriodic( Cajita::Dim::K )};
+    const Kokkos::Array<double, 3> global_low = {
+        global_mesh.lowCorner( Cajita::Dim::I ),
+        global_mesh.lowCorner( Cajita::Dim::J ),
+        global_mesh.lowCorner( Cajita::Dim::K )};
+    const Kokkos::Array<double, 3> global_high = {
+        global_mesh.highCorner( Cajita::Dim::I ),
+        global_mesh.highCorner( Cajita::Dim::J ),
+        global_mesh.highCorner( Cajita::Dim::K )};
+    const Kokkos::Array<double, 3> global_span = {
+        global_mesh.extent( Cajita::Dim::I ),
+        global_mesh.extent( Cajita::Dim::J ),
+        global_mesh.extent( Cajita::Dim::K )};
+    Kokkos::parallel_for(
+        "periodic_shift",
+        Kokkos::RangePolicy<execution_space>( 0, positions.size() ),
+        KOKKOS_LAMBDA( const int p ) {
+            // Compute the logical index of the neighbor we are sending to.
+            int nid[3] = {1, 1, 1};
+            for ( int d = 0; d < 3; ++d )
+            {
+                if ( positions( p, d ) < local_low[d] )
+                    nid[d] = 0;
+                else if ( positions( p, d ) > local_high[d] )
+                    nid[d] = 2;
+            }
+
+            // Compute the destination MPI rank.
+            destinations( p ) = neighbor_ranks(
+                nid[Cajita::Dim::I] +
+                3 * ( nid[Cajita::Dim::J] + 3 * nid[Cajita::Dim::K] ) );
+
+            // Shift periodic coordinates if needed.
+            for ( int d = 0; d < 3; ++d )
+            {
+                if ( period[d] )
+                {
+                    if ( positions( p, d ) > global_high[d] )
+                        positions( p, d ) -= global_span[d];
+                    else if ( positions( p, d ) < global_low[d] )
+                        positions( p, d ) += global_span[d];
+                }
+            }
+        } );
+}
+
+namespace Impl
+{
+
+//---------------------------------------------------------------------------//
+template <class DeviceType, class LocalGridType, class PositionType>
+Cabana::Distributor<DeviceType>
+prepare_migrate( const LocalGridType &local_grid, PositionType &positions )
+{
+    using device_type = DeviceType;
+
+    // Of the 27 potential local grids figure out which are in our topology.
+    // Some of the ranks in this list may be invalid. We will update this list
+    // after we compute destination ranks so it is unique and valid.
+    std::vector<int> topology( 27, -1 );
+    int nr = 0;
+    for ( int k = -1; k < 2; ++k )
+        for ( int j = -1; j < 2; ++j )
+            for ( int i = -1; i < 2; ++i, ++nr )
+                topology[nr] = local_grid.neighborRank( i, j, k );
+
+    // Locate the particles in the global grid and get their destination
+    // rank and shift periodic coordinates if necessary.
+    Kokkos::View<int *, Kokkos::HostSpace, Kokkos::MemoryUnmanaged>
+        neighbor_ranks( topology.data(), topology.size() );
+    auto nr_mirror =
+        Kokkos::create_mirror_view_and_copy( device_type(), neighbor_ranks );
+    Kokkos::View<int *, device_type> destinations(
+        Kokkos::ViewAllocateWithoutInitializing( "destinations" ),
+        positions.size() );
+    periodic_shift( local_grid, nr_mirror, destinations, positions );
+
+    // Make the topology a list of unique and valid ranks.
+    auto remove_end = std::remove( topology.begin(), topology.end(), -1 );
+    std::sort( topology.begin(), remove_end );
+    auto unique_end = std::unique( topology.begin(), remove_end );
+    topology.resize( std::distance( topology.begin(), unique_end ) );
+
+    // Create the Cabana distributor.
+    Cabana::Distributor<device_type> distributor(
+        local_grid.globalGrid().comm(), destinations, topology );
+
+    return distributor;
+}
+
+} // namespace Impl
+
+//---------------------------------------------------------------------------//
+/*!
+  \brief periodic_migrate uses a communication plan and migrates data from one
+  uniquely-owned decomposition to another uniquely-owned decomposition, taking
+  into account periodic boundary conditions with a Cajita grid. In-place
+  variant.
+*/
+template <class LocalGridType, class ParticleContainer,
+          std::size_t PositionIndex>
+void periodic_migrate( const LocalGridType &local_grid,
+                       ParticleContainer &particles,
+                       std::integral_constant<std::size_t, PositionIndex> )
+{
+    using device_type = typename ParticleContainer::device_type;
+
+    // Get the positions.
+    auto positions = Cabana::slice<PositionIndex>( particles );
+
+    // Periodic position shift.
+    auto distributor =
+        Impl::prepare_migrate<device_type>( local_grid, positions );
+
+    // Redistribute the particles.
+    Cabana::migrate( distributor, particles );
+}
+
+//---------------------------------------------------------------------------//
+/*!
+\brief periodic_migrate uses a communication plan and migrates data from one
+uniquely-owned decomposition to another uniquely-owned decomposition, taking
+into account periodic boundary conditions with a Cajita grid. Separate AoSoA
+variant.
+*/
+template <class LocalGridType, class ParticleContainer,
+          std::size_t PositionIndex>
+void periodic_migrate( const LocalGridType &local_grid,
+                       ParticleContainer &src_particles,
+                       std::integral_constant<std::size_t, PositionIndex>,
+                       ParticleContainer &dst_particles )
+{
+    using device_type = typename ParticleContainer::device_type;
+
+    // Get the positions.
+    auto positions = Cabana::slice<PositionIndex>( src_particles );
+
+    // Periodic position shift.
+    auto distributor =
+        Impl::prepare_migrate<device_type>( local_grid, positions );
+
+    // Redistribute the particles.
+    Cabana::migrate( distributor, src_particles, dst_particles );
+}
+
+//---------------------------------------------------------------------------//
+/*!
+\brief periodic_migrate uses a communication plan and migrates data from one
+uniquely-owned decomposition to another uniquely-owned decomposition, taking
+into account periodic boundary conditions with a Cajita grid. Separate
+slice/View variant.
+*/
+template <class LocalGridType, class ParticleContainer>
+void periodic_migrate( const LocalGridType &local_grid,
+                       ParticleContainer &src_positions,
+                       ParticleContainer &dst_positions )
+{
+    using device_type = typename ParticleContainer::device_type;
+
+    // Periodic position shift.
+    auto distributor =
+        Impl::prepare_migrate<device_type>( local_grid, src_positions );
+
+    // Redistribute the positions.
+    Cabana::migrate( distributor, src_positions, dst_positions );
+}
+
+} // end namespace CabanaPG
+
+#endif // end CABANA_PERIODIC_HPP

particle_grid/unit_test/CMakeLists.txt

@@ -0,0 +1,73 @@
+set(GTEST_SOURCE_DIR ${CMAKE_SOURCE_DIR}/gtest)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGTEST_HAS_PTHREAD=0")
+
+include_directories(${GTEST_SOURCE_DIR})
+add_library(cabana_particlegrid_gtest ${GTEST_SOURCE_DIR}/gtest/gtest-all.cc)
+set_target_properties(cabana_particlegrid_gtest PROPERTIES
+    CXX_STANDARD 11
+    CXX_STANDARD_REQUIRED YES
+    CXX_EXTENSIONS NO
+    )
+
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+set(gtest_args --gtest_color=yes)
+
+set(PARTICLEGRID_TEST_DEVICES)
+foreach(_device ${CABANA_SUPPORTED_DEVICES})
+  if(Kokkos_ENABLE_${_device})
+    list(APPEND PARTICLEGRID_TEST_DEVICES ${_device})
+    if(_device STREQUAL CUDA)
+      list(APPEND PARTICLEGRID_TEST_DEVICES CUDA_UVM)
+    endif()
+  endif()
+endforeach()
+
+macro(ParticleGrid_add_tests)
+  cmake_parse_arguments(PARTICLEGRID_UNIT_TEST "MPI" "" "NAMES" ${ARGN})
+  set(PARTICLEGRID_UNIT_TEST_MPIEXEC_NUMPROCS 1)
+  foreach( _np 2 4 8 )
+    if(MPIEXEC_MAX_NUMPROCS GREATER_EQUAL ${_np})
+      list(APPEND PARTICLEGRID_UNIT_TEST_MPIEXEC_NUMPROCS ${_np})
+    endif()
+  endforeach()
+  set(PARTICLEGRID_UNIT_TEST_NUMTHREADS 1)
+  foreach( _nt 2 4 8 )
+    if(MPIEXEC_MAX_NUMPROCS GREATER_EQUAL ${_nt})
+      list(APPEND PARTICLEGRID_UNIT_TEST_NUMTHREADS ${_nt})
+    endif()
+  endforeach()
+  set(PARTICLEGRID_UNIT_TEST_MAIN ../../core/unit_test/mpi_unit_test_main.cpp)
+  foreach(_device ${PARTICLEGRID_TEST_DEVICES})
+    set(_dir ${CMAKE_CURRENT_BINARY_DIR}/${_device})
+    file(MAKE_DIRECTORY ${_dir})
+    foreach(_test ${PARTICLEGRID_UNIT_TEST_NAMES})
+      set(_file ${_dir}/tst${_test}_${_device}.cpp)
+      file(WRITE ${_file} "#include <Test${_device}_Category.hpp>\n")
+      file(APPEND ${_file} "#include <tst${_test}.hpp>\n")
+      set(_target ParticleGrid_${_test}_test_${_device})
+      add_executable(${_target} ${_file} ${PARTICLEGRID_UNIT_TEST_MAIN})
+      target_include_directories(${_target} PRIVATE ${_dir} ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../../core/unit_test)
+      target_link_libraries(${_target} PRIVATE CabanaPG cabana_particlegrid_gtest)
+      if(PARTICLEGRID_UNIT_TEST_MPI)
+        foreach(_np ${PARTICLEGRID_UNIT_TEST_MPIEXEC_NUMPROCS})
+          add_test(NAME ${_target}_${_np} COMMAND
+            ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${_np} ${MPIEXEC_PREFLAGS}
+            ${_target} ${MPIEXEC_POSTFLAGS} ${gtest_args})
+        endforeach()
+      else()
+        if(_device STREQUAL OpenMP)
+          foreach(_thread ${PARTICLEGRID_UNIT_TEST_NUMTHREADS})
+            add_test(NAME ${_target}_${_thread} COMMAND
+              ${_target} ${gtest_args} --kokkos-threads=${_thread})
+          endforeach()
+        else()
+          add_test(NAME ${_target} COMMAND ${_target} ${gtest_args})
+        endif()
+      endif()
+    endforeach()
+  endforeach()
+endmacro()
+
+ParticleGrid_add_tests(MPI NAMES Periodic)