From 6093894d7329dff546944e84a0998ba96c337026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Mon, 9 Sep 2024 14:15:29 +0200 Subject: [PATCH 01/50] ci: upgrade Kokkos version --- .github/workflows/CI.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 98a0ba8..2fc1b12 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -15,7 +15,7 @@ jobs: cxx: ['g++', 'clang++'] backend: ['SERIAL', 'OPENMP'] cmake_build_type: ['Debug', 'Release'] - kokkos_ver: ['3.7.02'] + kokkos_ver: ['4.3.01'] arborx: ['ArborX', 'NoArborX'] nnp: ['ON', 'OFF'] layout: ['1'] @@ -27,7 +27,7 @@ jobs: cxx: 'g++' openmp: 'ON' cmake_build_type: 'Release' - kokkos_ver: '3.7.02' + kokkos_ver: '4.3.01' arborx: 'NoArborX' nnp: 'OFF' layout: '2' @@ -38,7 +38,7 @@ jobs: cxx: 'g++' openmp: 'ON' cmake_build_type: 'Release' - kokkos_ver: '3.7.02' + kokkos_ver: '4.3.01' arborx: 'NoArborX' nnp: 'ON' layout: '6' @@ -49,7 +49,7 @@ jobs: cxx: 'g++' openmp: 'ON' cmake_build_type: 'Release' - kokkos_ver: '3.7.02' + kokkos_ver: '4.3.01' arborx: 'NoArborX' nnp: 'OFF' layout: '6' From bf9fd6034274362095397319abd7910a8d4decae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Mon, 9 Sep 2024 14:23:23 +0200 Subject: [PATCH 02/50] remove unused variables --- src/integrator_nve_impl.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/integrator_nve_impl.h b/src/integrator_nve_impl.h index 564f4fd..0047456 100644 --- a/src/integrator_nve_impl.h +++ b/src/integrator_nve_impl.h @@ -64,10 +64,8 @@ void Integrator::initial_integrate( t_System *system ) f = system->f; type = system->type; - static int step = 1; Kokkos::parallel_for( "IntegratorNVE::initial_integrate", t_policy_initial( 0, system->N_local ), *this ); - step++; } template @@ -80,10 +78,8 @@ void Integrator::final_integrate( t_System *system ) f = system->f; type = system->type; - static int step = 1; Kokkos::parallel_for( "IntegratorNVE::final_integrate", t_policy_final( 0, system->N_local ), *this ); - step++; } template From e97bfac7466c284338fd4fa7c33e678cb7ad17ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Thu, 8 Aug 2024 20:44:36 +0200 Subject: [PATCH 03/50] avoid redundant initialization --- src/inputFile.h | 43 ++++++++++++++++++++++------------------- src/inputFile_impl.h | 46 +------------------------------------------- 2 files changed, 24 insertions(+), 65 deletions(-) diff --git a/src/inputFile.h b/src/inputFile.h index 16e9019..10503eb 100644 --- a/src/inputFile.h +++ b/src/inputFile.h @@ -58,6 +58,7 @@ #include #include +#include #include #include @@ -149,18 +150,20 @@ template class InputFile { private: - bool timestepflag; // input timestep? + bool timestepflag = false; // input timestep? public: InputCL commandline; t_System *system; bool _print_rank; - int units_style; - int lattice_style; - double lattice_constant, lattice_offset_x, lattice_offset_y, - lattice_offset_z; + + // defaults match ExaMiniMD LJ example + int units_style = UNITS_LJ; + int lattice_style = LATTICE_FCC; + double lattice_constant = 0.8442, lattice_offset_x = 0.0, + lattice_offset_y = 0.0, lattice_offset_z = 0.0; int lattice_nx, lattice_ny, lattice_nz; - int box[6]; + std::array box = { 0, 40, 0, 40, 0, 40 }; char *data_file; int data_file_type; @@ -168,33 +171,33 @@ class InputFile std::string output_file; std::string error_file; - double temperature_target; - int temperature_seed; + double temperature_target = 1.4; + int temperature_seed = 87287; - int integrator_type; - int nsteps; + int integrator_type = INTEGRATOR_NVE; + int nsteps = 100; - int binning_type; + int binning_type = BINNING_LINKEDCELL; - int comm_type; - int comm_exchange_rate; + int comm_type = COMM_MPI; + int comm_exchange_rate = 20; - int force_type; + int force_type = FORCE_LJ; int force_iteration_type; int force_neigh_parallel_type; - T_F_FLOAT force_cutoff; + T_F_FLOAT force_cutoff = 2.5; std::vector> force_coeff_lines; - T_F_FLOAT neighbor_skin; - int neighbor_type; - T_INT max_neigh_guess; + T_F_FLOAT neighbor_skin = 0.0; + int neighbor_type = NEIGH_VERLET_2D; + T_INT max_neigh_guess = 50; int layout_type; int nnp_layout_type; - int thermo_rate, dumpbinary_rate, correctness_rate; - bool dumpbinaryflag, correctnessflag; + int thermo_rate = 10, dumpbinary_rate = 0, correctness_rate = 0; + bool dumpbinaryflag = false, correctnessflag = false; char *dumpbinary_path, *reference_path, *correctness_file; std::string input_data_file; std::string output_data_file; diff --git a/src/inputFile_impl.h b/src/inputFile_impl.h index f4edc44..93fcf9b 100644 --- a/src/inputFile_impl.h +++ b/src/inputFile_impl.h @@ -49,7 +49,7 @@ #include #include -#include +#include #include std::vector split( const std::string &line ) @@ -71,56 +71,12 @@ InputFile::InputFile( InputCL commandline_, t_System *system_ ) : commandline( commandline_ ) , system( system_ ) { - comm_type = COMM_MPI; - integrator_type = INTEGRATOR_NVE; - neighbor_type = NEIGH_VERLET_2D; - force_type = FORCE_LJ; - binning_type = BINNING_LINKEDCELL; - neighbor_type = commandline.neighbor_type; force_iteration_type = commandline.force_iteration_type; force_neigh_parallel_type = commandline.force_neigh_parallel_type; output_file = commandline.output_file; error_file = commandline.error_file; - - // set defaults (matches ExaMiniMD LJ example) - - nsteps = 0; - - thermo_rate = 0; - dumpbinary_rate = 0; - correctness_rate = 0; - dumpbinaryflag = false; - correctnessflag = false; - timestepflag = false; - - lattice_offset_x = 0.0; - lattice_offset_y = 0.0; - lattice_offset_z = 0.0; - box[0] = 0; - box[2] = 0; - box[4] = 0; - box[1] = 40; - box[3] = 40; - box[5] = 40; - - units_style = UNITS_LJ; - lattice_style = LATTICE_FCC; - lattice_constant = 0.8442; - - temperature_target = 1.4; - temperature_seed = 87287; - - nsteps = 100; - thermo_rate = 10; - - neighbor_skin = 0.3; - neighbor_skin = 0.0; // for metal and real units - max_neigh_guess = 50; - comm_exchange_rate = 20; - - force_cutoff = 2.5; } template From 4ab9d0944788b70f3a4a6b5cad3a04698ce66810 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Wed, 28 Jul 2021 18:38:52 +0200 Subject: [PATCH 04/50] add vim temporary files to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 40e1f1a..c429bde 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ build .cproject .project +*.swp From f67680eceefd4372ef23d955de1c41b99df55cf3 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Wed, 28 Jul 2021 18:39:44 +0200 Subject: [PATCH 05/50] add ALL to CMakeLists.txt --- CMakeLists.txt | 9 +++++++++ src/CMakeLists.txt | 1 + 2 files changed, 10 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index d9f8ad2..af5e4d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,15 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) ##---------------------------------------------------------------------------## find_package(Cabana REQUIRED) +if( NOT Cabana_ENABLE_MPI ) + message( FATAL_ERROR "Cabana must be compiled with MPI" ) +endif() +if( NOT Cabana_ENABLE_CAJITA ) + message( FATAL_ERROR "Cabana must be compiled with Cajita" ) +endif() + +find_package(ALL 0.9.1 REQUIRED) + ##---------------------------------------------------------------------------## # Set up optional libraries ##---------------------------------------------------------------------------## diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4805039..e089b30 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,6 +89,7 @@ target_include_directories(CabanaMD PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/system_types $ $) +target_link_libraries(CabanaMD ALL::ALL) #------------------------------------------------------------ From 638fa070a168d6375f968cd7baced0527999ffec Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Wed, 28 Jul 2021 18:40:12 +0200 Subject: [PATCH 06/50] first integration of ALL --- src/cabanamd.h | 2 + src/cabanamd_impl.h | 5 ++ src/load_balancer.h | 102 +++++++++++++++++++++++++++++++ src/system.h | 35 +++++++++-- src/vtk_domain_writer.h | 130 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 268 insertions(+), 6 deletions(-) create mode 100644 src/load_balancer.h create mode 100644 src/vtk_domain_writer.h diff --git a/src/cabanamd.h b/src/cabanamd.h index bfb4582..eca3f7a 100644 --- a/src/cabanamd.h +++ b/src/cabanamd.h @@ -55,6 +55,7 @@ #include #include #include +#include #include class CabanaMD @@ -81,6 +82,7 @@ class CbnMD : public CabanaMD Comm *comm; Binning *binning; InputFile *input; + LoadBalancer *lb; void init( InputCL cl ) override; void run() override; diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index a2c743d..4d8e29e 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -224,6 +224,8 @@ void CbnMD::init( InputCL commandline ) comm->update_force(); } + lb = new LoadBalancer( system ); + // Initial output int step = 0; if ( input->thermo_rate > 0 ) @@ -373,6 +375,7 @@ void CbnMD::run() " ", T, " ", PE, " ", PE + KE, " ", time ); last_time = time; } + lb->output( step ); } if ( input->dumpbinaryflag ) @@ -381,6 +384,8 @@ void CbnMD::run() if ( input->correctnessflag ) check_correctness( step ); + lb->balance(); + other_time += other_timer.seconds(); } diff --git a/src/load_balancer.h b/src/load_balancer.h new file mode 100644 index 0000000..47239b6 --- /dev/null +++ b/src/load_balancer.h @@ -0,0 +1,102 @@ +/**************************************************************************** + * Copyright (c) 2018-2021 by the Cabana authors * + * All rights reserved. * + * * + * This file is part of the Cabana library. Cabana is distributed under a * + * BSD 3-clause license. For the licensing terms see the LICENSE file in * + * the top-level directory. * + * * + * SPDX-License-Identifier: BSD-3-Clause * + ****************************************************************************/ + +#ifndef LOADBALANCER_H +#define LOADBALANCER_H + +#include +#include + +#include + +#include + +#include + +template +class LoadBalancer +{ + public: + LoadBalancer( t_System *system ) + : _comm( MPI_COMM_WORLD ) + , _system( system ) + { + MPI_Comm_rank( _comm, &_rank ); + _liball = + std::make_shared>( ALL::TENSOR, 3, 0 ); + std::vector rank_dim_pos( _system->rank_dim_pos.begin(), + _system->rank_dim_pos.end() ); + std::vector ranks_per_dim( _system->ranks_per_dim.begin(), + _system->ranks_per_dim.end() ); + _liball->setProcGridParams( rank_dim_pos, ranks_per_dim ); + // todo(sschulz): Do we need a minimum domain size? + // _liball->setMinimumDomainSize(..); + _liball->setCommunicator( _comm ); + _liball->setProcTag( _rank ); + _liball->setup(); + + std::vector> lb_vertices( 2, + ALL::Point( 3 ) ); + lb_vertices.at( 0 )[0] = _system->local_mesh_lo_x; + lb_vertices.at( 0 )[1] = _system->local_mesh_lo_y; + lb_vertices.at( 0 )[2] = _system->local_mesh_lo_z; + lb_vertices.at( 1 )[0] = _system->local_mesh_hi_x; + lb_vertices.at( 1 )[1] = _system->local_mesh_hi_y; + lb_vertices.at( 1 )[2] = _system->local_mesh_hi_z; + _liball->setVertices( lb_vertices ); + } + + void balance() + { + _liball->setWork( _system->N_local + _system->N_ghost ); + _liball->balance(); + std::vector> updated_vertices = + _liball->getVertices(); + std::array low_corner = { updated_vertices.at( 0 )[0], + updated_vertices.at( 0 )[1], + updated_vertices.at( 0 )[2] }; + std::array high_corner = { updated_vertices.at( 1 )[0], + updated_vertices.at( 1 )[1], + updated_vertices.at( 1 )[2] }; + _system->update_domain( low_corner, high_corner ); + } + + void output( const int t ) const + { + std::string vtk_actual_domain_basename( "domain_act" ); + std::string vtk_lb_domain_basename( "domain_lb" ); + // _liball->printVTKoutlines( t ); + + std::array vertices = { + _system->local_mesh_lo_x, _system->local_mesh_lo_y, + _system->local_mesh_lo_z, _system->local_mesh_hi_x, + _system->local_mesh_hi_y, _system->local_mesh_hi_z }; + VTKDomainWriter::writeDomain( _comm, t, vertices, + vtk_actual_domain_basename ); + + std::vector> updated_vertices = + _liball->getVertices(); + for ( std::size_t d = 0; d < 3; ++d ) + vertices[d] = updated_vertices.at( 0 )[d]; + for ( std::size_t d = 3; d < 6; ++d ) + vertices[d] = updated_vertices.at( 1 )[d - 3]; + VTKDomainWriter::writeDomain( _comm, t, vertices, + vtk_lb_domain_basename ); + } + + private: + MPI_Comm _comm; + t_System *_system; + std::shared_ptr> _liball; + int _rank; +}; + +#endif diff --git a/src/system.h b/src/system.h index 5185f1f..46f4ad1 100644 --- a/src/system.h +++ b/src/system.h @@ -81,6 +81,8 @@ class SystemCommon typedef typename t_mass::HostMirror h_t_mass; t_mass mass; + int halo_width; + // Simulation total domain T_X_FLOAT global_mesh_x, global_mesh_y, global_mesh_z; @@ -93,6 +95,8 @@ class SystemCommon std::shared_ptr< Cabana::Grid::LocalGrid>> local_grid; + std::shared_ptr>> + global_grid; // Only needed for current comm std::array ranks_per_dim; @@ -143,8 +147,8 @@ class SystemCommon // Create the global grid. std::array is_periodic = { true, true, true }; - auto global_grid = Cabana::Grid::createGlobalGrid( - MPI_COMM_WORLD, global_mesh, is_periodic, partitioner ); + global_grid = Cajita::createGlobalGrid( MPI_COMM_WORLD, global_mesh, + is_periodic, partitioner ); for ( int d = 0; d < 3; d++ ) { @@ -152,10 +156,9 @@ class SystemCommon } // Create a local mesh - int halo_width = 1; - local_grid = Cabana::Grid::createLocalGrid( global_grid, halo_width ); - auto local_mesh = - Cabana::Grid::createLocalMesh( *local_grid ); + halo_width = 1; + local_grid = Cajita::createLocalGrid( global_grid, halo_width ); + auto local_mesh = Cajita::createLocalMesh( *local_grid ); local_mesh_lo_x = local_mesh.lowCorner( Cabana::Grid::Own(), 0 ); local_mesh_lo_y = local_mesh.lowCorner( Cabana::Grid::Own(), 1 ); @@ -173,6 +176,26 @@ class SystemCommon local_mesh_y = local_mesh.extent( Cabana::Grid::Own(), 1 ); local_mesh_z = local_mesh.extent( Cabana::Grid::Own(), 2 ); } + // low_corner and high_corner are local corners, not globa as in create_domain! + void update_domain( std::array low_corner, + std::array high_corner ) + { + local_mesh_lo_x = low_corner[0]; + local_mesh_lo_y = low_corner[1]; + local_mesh_lo_z = low_corner[2]; + local_mesh_hi_x = high_corner[0]; + local_mesh_hi_y = high_corner[1]; + local_mesh_hi_z = high_corner[2]; + local_mesh_x = local_mesh_hi_x - local_mesh_lo_x; + local_mesh_y = local_mesh_hi_y - local_mesh_lo_y; + local_mesh_z = local_mesh_hi_z - local_mesh_lo_z; + ghost_mesh_lo_x = local_mesh_lo_x - halo_width*local_mesh_x; + ghost_mesh_lo_y = local_mesh_lo_y - halo_width*local_mesh_y; + ghost_mesh_lo_z = local_mesh_lo_z - halo_width*local_mesh_z; + ghost_mesh_hi_x = local_mesh_hi_x + halo_width*local_mesh_x; + ghost_mesh_hi_y = local_mesh_hi_y + halo_width*local_mesh_y; + ghost_mesh_hi_z = local_mesh_hi_z + halo_width*local_mesh_z; + } void slice_all() { diff --git a/src/vtk_domain_writer.h b/src/vtk_domain_writer.h new file mode 100644 index 0000000..f27442c --- /dev/null +++ b/src/vtk_domain_writer.h @@ -0,0 +1,130 @@ +/**************************************************************************** + * Copyright (c) 2018-2021 by the Cabana authors * + * All rights reserved. * + * * + * This file is part of the Cabana library. Cabana is distributed under a * + * BSD 3-clause license. For the licensing terms see the LICENSE file in * + * the top-level directory. * + * * + * SPDX-License-Identifier: BSD-3-Clause * + ****************************************************************************/ + +#ifndef VTK_DOMAIN_WRITER_H +#define VTK_DOMAIN_WRITER_H +#include + +#include +#include + +namespace VTKDomainWriter +{ +// Write PVTU +void writeDomainParallelFile( MPI_Comm comm, int time_step, + std::string &basename ) +{ + // Should only be called from a single rank + int size; + MPI_Comm_size( comm, &size ); + std::stringstream filename; + filename << basename << "_" << time_step << ".pvtu"; + FILE *file = fopen( filename.str().c_str(), "w" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\t\n" ); + for ( std::size_t i = 0; i < size; ++i ) + fprintf( file, "\t\n", + basename.c_str(), time_step, i ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fclose( file ); +} +// Write VTU for domain (low corner, high corner) +void writeDomain( MPI_Comm comm, int time_step, + std::array &domain_vertices, + std::string &basename ) +{ + int rank; + MPI_Comm_rank( comm, &rank ); + if ( rank == 1 ) + writeDomainParallelFile( comm, time_step, basename ); + std::stringstream filename; + // todo(sschulz): properly format, according to max rank + filename << basename << "_" << time_step << "_" << rank << ".vtu"; + FILE *file = fopen( filename.str().c_str(), "w" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + std::array vertices; + vertices[0 * 3 + 0] = domain_vertices[0]; + vertices[2 * 3 + 0] = domain_vertices[0]; + vertices[4 * 3 + 0] = domain_vertices[0]; + vertices[6 * 3 + 0] = domain_vertices[0]; + vertices[0 * 3 + 1] = domain_vertices[1]; + vertices[1 * 3 + 1] = domain_vertices[1]; + vertices[4 * 3 + 1] = domain_vertices[1]; + vertices[5 * 3 + 1] = domain_vertices[1]; + vertices[0 * 3 + 2] = domain_vertices[2]; + vertices[1 * 3 + 2] = domain_vertices[2]; + vertices[2 * 3 + 2] = domain_vertices[2]; + vertices[3 * 3 + 2] = domain_vertices[2]; + vertices[1 * 3 + 0] = domain_vertices[3]; + vertices[3 * 3 + 0] = domain_vertices[3]; + vertices[5 * 3 + 0] = domain_vertices[3]; + vertices[7 * 3 + 0] = domain_vertices[3]; + vertices[2 * 3 + 1] = domain_vertices[4]; + vertices[3 * 3 + 1] = domain_vertices[4]; + vertices[6 * 3 + 1] = domain_vertices[4]; + vertices[7 * 3 + 1] = domain_vertices[4]; + vertices[4 * 3 + 2] = domain_vertices[5]; + vertices[5 * 3 + 2] = domain_vertices[5]; + vertices[6 * 3 + 2] = domain_vertices[5]; + vertices[7 * 3 + 2] = domain_vertices[5]; + std::array connectivity = { 0, 1, 2, 3, 4, 5, 6, 7 }; + fprintf( file, "\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "%d", rank ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + for ( const double &vert : vertices ) + fprintf( file, "%g ", vert ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + for ( const int &conn : connectivity ) + fprintf( file, "%d ", conn ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "8\n" ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "11\n" ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fclose( file ); +} + +} // end namespace VTKDomainWriter +#endif From 038ecc92ec1a7025bfabda881ecf91820338d54f Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Mon, 2 Aug 2021 15:56:07 +0200 Subject: [PATCH 07/50] format and move balance operation into neighbour update --- src/cabanamd_impl.h | 23 ++++++++++++++--------- src/system.h | 15 ++++++++------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index 4d8e29e..b604de7 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -278,11 +278,12 @@ void CbnMD::run() double comm_time = 0; double neigh_time = 0; double integrate_time = 0; + double lb_time = 0; double other_time = 0; double last_time = 0; Kokkos::Timer timer, force_timer, comm_timer, neigh_timer, integrate_timer, - other_timer; + lb_timer, other_timer; // Main timestep loop for ( int step = 1; step <= nsteps; step++ ) @@ -294,6 +295,11 @@ void CbnMD::run() if ( step % input->comm_exchange_rate == 0 && step > 0 ) { + // Update domain decomposition + lb_timer.reset(); + lb->balance(); + lb_time += lb_timer.seconds(); + // Exchange atoms across MPI ranks comm_timer.reset(); comm->exchange(); @@ -384,8 +390,6 @@ void CbnMD::run() if ( input->correctnessflag ) check_correctness( step ); - lb->balance(); - other_time += other_timer.seconds(); } @@ -397,15 +401,16 @@ void CbnMD::run() double steps_per_sec = 1.0 * nsteps / time; double atom_steps_per_sec = system->N * steps_per_sec; log( out, std::fixed, std::setprecision( 2 ), - "\n#Procs Atoms | Time T_Force T_Neigh T_Comm T_Int ", + "\n#Procs Atoms | Time T_Force T_Neigh T_Comm T_Int T_lb ", "T_Other |\n", comm->num_processes(), " ", system->N, " | ", time, " ", force_time, " ", neigh_time, " ", comm_time, " ", - integrate_time, " ", other_time, " | PERFORMANCE\n", std::fixed, - comm->num_processes(), " ", system->N, " | ", 1.0, " ", + integrate_time, " ", lb_time, " ", other_time, " | PERFORMANCE\n", + std::fixed, comm->num_processes(), " ", system->N, " | ", 1.0, " ", force_time / time, " ", neigh_time / time, " ", comm_time / time, - " ", integrate_time / time, " ", other_time / time, - " | FRACTION\n\n", "#Steps/s Atomsteps/s Atomsteps/(proc*s)\n", - std::scientific, steps_per_sec, " ", atom_steps_per_sec, " ", + " ", integrate_time / time, " ", lb_time / time, " ", + other_time / time, " | FRACTION\n\n", + "#Steps/s Atomsteps/s Atomsteps/(proc*s)\n", std::scientific, + steps_per_sec, " ", atom_steps_per_sec, " ", atom_steps_per_sec / comm->num_processes() ); } else diff --git a/src/system.h b/src/system.h index 46f4ad1..30cee75 100644 --- a/src/system.h +++ b/src/system.h @@ -176,7 +176,8 @@ class SystemCommon local_mesh_y = local_mesh.extent( Cabana::Grid::Own(), 1 ); local_mesh_z = local_mesh.extent( Cabana::Grid::Own(), 2 ); } - // low_corner and high_corner are local corners, not globa as in create_domain! + // low_corner and high_corner are local corners, not global as in + // create_domain! void update_domain( std::array low_corner, std::array high_corner ) { @@ -189,12 +190,12 @@ class SystemCommon local_mesh_x = local_mesh_hi_x - local_mesh_lo_x; local_mesh_y = local_mesh_hi_y - local_mesh_lo_y; local_mesh_z = local_mesh_hi_z - local_mesh_lo_z; - ghost_mesh_lo_x = local_mesh_lo_x - halo_width*local_mesh_x; - ghost_mesh_lo_y = local_mesh_lo_y - halo_width*local_mesh_y; - ghost_mesh_lo_z = local_mesh_lo_z - halo_width*local_mesh_z; - ghost_mesh_hi_x = local_mesh_hi_x + halo_width*local_mesh_x; - ghost_mesh_hi_y = local_mesh_hi_y + halo_width*local_mesh_y; - ghost_mesh_hi_z = local_mesh_hi_z + halo_width*local_mesh_z; + ghost_mesh_lo_x = local_mesh_lo_x - halo_width * local_mesh_x; + ghost_mesh_lo_y = local_mesh_lo_y - halo_width * local_mesh_y; + ghost_mesh_lo_z = local_mesh_lo_z - halo_width * local_mesh_z; + ghost_mesh_hi_x = local_mesh_hi_x + halo_width * local_mesh_x; + ghost_mesh_hi_y = local_mesh_hi_y + halo_width * local_mesh_y; + ghost_mesh_hi_z = local_mesh_hi_z + halo_width * local_mesh_z; } void slice_all() From 7ee7633bb7898e20e9da1e5a059f9261414bac1c Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Mon, 2 Aug 2021 15:58:24 +0200 Subject: [PATCH 08/50] debug printfs and pseudo unbalanced work --- src/inputFile_impl.h | 3 +++ src/load_balancer.h | 10 ++++++++- src/read_data.h | 4 ++++ src/system.h | 48 +++++++++++++++++++++++++++++--------------- 4 files changed, 48 insertions(+), 17 deletions(-) diff --git a/src/inputFile_impl.h b/src/inputFile_impl.h index 93fcf9b..96d3433 100644 --- a/src/inputFile_impl.h +++ b/src/inputFile_impl.h @@ -455,6 +455,9 @@ void InputFile::create_lattice( Comm *comm ) T_X_FLOAT max_z = lattice_constant * lattice_nz; std::array global_low = { 0.0, 0.0, 0.0 }; std::array global_high = { max_x, max_y, max_z }; + // global_high[0] *= 2; + // global_high[1] *= 2; + // global_high[2] *= 2; system->create_domain( global_low, global_high ); s = *system; diff --git a/src/load_balancer.h b/src/load_balancer.h index 47239b6..25682fa 100644 --- a/src/load_balancer.h +++ b/src/load_balancer.h @@ -56,10 +56,18 @@ class LoadBalancer void balance() { - _liball->setWork( _system->N_local + _system->N_ghost ); + int rank; + MPI_Comm_rank( _comm, &rank ); + double work = ( _system->N_local + _system->N_ghost ) / ( rank + 1 ); + printf( ">> Work: %g\n", work ); + _liball->setWork( work ); _liball->balance(); std::vector> updated_vertices = _liball->getVertices(); + printf( ">> New Vertices: %g %g %g %g %g %g\n", + updated_vertices.at( 0 )[0], updated_vertices.at( 0 )[1], + updated_vertices.at( 0 )[2], updated_vertices.at( 1 )[0], + updated_vertices.at( 1 )[1], updated_vertices.at( 1 )[2] ); std::array low_corner = { updated_vertices.at( 0 )[0], updated_vertices.at( 0 )[1], updated_vertices.at( 0 )[2] }; diff --git a/src/read_data.h b/src/read_data.h index 2a4490c..50a3081 100644 --- a/src/read_data.h +++ b/src/read_data.h @@ -158,6 +158,10 @@ void read_lammps_header( std::ifstream &file, std::ofstream &err, t_System *s ) } // Create mesh + //high_corner[0] *= 2; + //high_corner[1] *= 2; + //high_corner[2] *= 2; + printf( ">> %g %g %g\n", high_corner[0], high_corner[1], high_corner[2] ); s->create_domain( low_corner, high_corner ); } diff --git a/src/system.h b/src/system.h index 30cee75..de2e8df 100644 --- a/src/system.h +++ b/src/system.h @@ -160,27 +160,39 @@ class SystemCommon local_grid = Cajita::createLocalGrid( global_grid, halo_width ); auto local_mesh = Cajita::createLocalMesh( *local_grid ); - local_mesh_lo_x = local_mesh.lowCorner( Cabana::Grid::Own(), 0 ); - local_mesh_lo_y = local_mesh.lowCorner( Cabana::Grid::Own(), 1 ); - local_mesh_lo_z = local_mesh.lowCorner( Cabana::Grid::Own(), 2 ); - local_mesh_hi_x = local_mesh.highCorner( Cabana::Grid::Own(), 0 ); - local_mesh_hi_y = local_mesh.highCorner( Cabana::Grid::Own(), 1 ); - local_mesh_hi_z = local_mesh.highCorner( Cabana::Grid::Own(), 2 ); - ghost_mesh_lo_x = local_mesh.lowCorner( Cabana::Grid::Ghost(), 0 ); - ghost_mesh_lo_y = local_mesh.lowCorner( Cabana::Grid::Ghost(), 1 ); - ghost_mesh_lo_z = local_mesh.lowCorner( Cabana::Grid::Ghost(), 2 ); - ghost_mesh_hi_x = local_mesh.highCorner( Cabana::Grid::Ghost(), 0 ); - ghost_mesh_hi_y = local_mesh.highCorner( Cabana::Grid::Ghost(), 1 ); - ghost_mesh_hi_z = local_mesh.highCorner( Cabana::Grid::Ghost(), 2 ); - local_mesh_x = local_mesh.extent( Cabana::Grid::Own(), 0 ); - local_mesh_y = local_mesh.extent( Cabana::Grid::Own(), 1 ); - local_mesh_z = local_mesh.extent( Cabana::Grid::Own(), 2 ); + local_mesh_lo_x = local_mesh.lowCorner( Cajita::Own(), 0 ); + local_mesh_lo_y = local_mesh.lowCorner( Cajita::Own(), 1 ); + local_mesh_lo_z = local_mesh.lowCorner( Cajita::Own(), 2 ); + local_mesh_hi_x = local_mesh.highCorner( Cajita::Own(), 0 ); + local_mesh_hi_y = local_mesh.highCorner( Cajita::Own(), 1 ); + local_mesh_hi_z = local_mesh.highCorner( Cajita::Own(), 2 ); + ghost_mesh_lo_x = local_mesh.lowCorner( Cajita::Ghost(), 0 ); + ghost_mesh_lo_y = local_mesh.lowCorner( Cajita::Ghost(), 1 ); + ghost_mesh_lo_z = local_mesh.lowCorner( Cajita::Ghost(), 2 ); + ghost_mesh_hi_x = local_mesh.highCorner( Cajita::Ghost(), 0 ); + ghost_mesh_hi_y = local_mesh.highCorner( Cajita::Ghost(), 1 ); + ghost_mesh_hi_z = local_mesh.highCorner( Cajita::Ghost(), 2 ); + local_mesh_x = local_mesh.extent( Cajita::Own(), 0 ); + local_mesh_y = local_mesh.extent( Cajita::Own(), 1 ); + local_mesh_z = local_mesh.extent( Cajita::Own(), 2 ); + printf( ">> global: %g %g %g %g %g %g\n", low_corner[0], low_corner[1], + low_corner[2], high_corner[0], high_corner[1], high_corner[2] ); + printf( ">> local: %g %g %g %g %g %g\n", local_mesh_lo_x, + local_mesh_lo_y, local_mesh_lo_z, local_mesh_hi_x, + local_mesh_hi_y, local_mesh_hi_z ); } - // low_corner and high_corner are local corners, not global as in + // low_corner and high_corner are local corners, not globa as in // create_domain! void update_domain( std::array low_corner, std::array high_corner ) { + printf( "%s: Old extent: %g %g %g %g %g %g, %g %g %g\n", __func__, + local_mesh_lo_x, local_mesh_lo_y, local_mesh_lo_z, + local_mesh_hi_x, local_mesh_hi_y, local_mesh_hi_z, local_mesh_x, + local_mesh_y, local_mesh_z ); + printf( "%s: New corners: %g %g %g %g %g %g\n", __func__, low_corner[0], + low_corner[1], low_corner[2], high_corner[0], high_corner[1], + high_corner[2] ); local_mesh_lo_x = low_corner[0]; local_mesh_lo_y = low_corner[1]; local_mesh_lo_z = low_corner[2]; @@ -196,6 +208,10 @@ class SystemCommon ghost_mesh_hi_x = local_mesh_hi_x + halo_width * local_mesh_x; ghost_mesh_hi_y = local_mesh_hi_y + halo_width * local_mesh_y; ghost_mesh_hi_z = local_mesh_hi_z + halo_width * local_mesh_z; + printf( "%s: New extent: %g %g %g %g %g %g, %g %g %g\n", __func__, + local_mesh_lo_x, local_mesh_lo_y, local_mesh_lo_z, + local_mesh_hi_x, local_mesh_hi_y, local_mesh_hi_z, local_mesh_x, + local_mesh_y, local_mesh_z ); } void slice_all() From 30bf78a0e1c922dcbbb0861a3a3c8c7a10f7af19 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Mon, 2 Aug 2021 16:00:11 +0200 Subject: [PATCH 09/50] REVERT BEFORE MERGING increase minimum cmake version Using a slighlty higher minimum cmake version allows for automatic detection of dependencies via *_ROOT environment variables --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index af5e4d7..3a4b9ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.11) +cmake_minimum_required(VERSION 3.14) project(CabanaMD LANGUAGES CXX VERSION 0.1.0) set(CMAKE_CXX_STANDARD_REQUIRED ON) From f062793e726cc137d05a48b009bbf68e856b7c66 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Mon, 2 Aug 2021 16:02:50 +0200 Subject: [PATCH 10/50] update comment regarding minimum domain size --- src/load_balancer.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/load_balancer.h b/src/load_balancer.h index 25682fa..53628d7 100644 --- a/src/load_balancer.h +++ b/src/load_balancer.h @@ -38,7 +38,8 @@ class LoadBalancer _system->ranks_per_dim.end() ); _liball->setProcGridParams( rank_dim_pos, ranks_per_dim ); // todo(sschulz): Do we need a minimum domain size? - // _liball->setMinimumDomainSize(..); + //std::vector min_size( 3, 15 ); + //_liball->setMinDomainSize( min_size ); _liball->setCommunicator( _comm ); _liball->setProcTag( _rank ); _liball->setup(); From a5188e71b99abd73e8f189db8a27448505ffe66e Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Mon, 2 Aug 2021 19:24:30 +0200 Subject: [PATCH 11/50] add explicit halo width via input file --- input/in.lj | 1 + src/inputFile.h | 5 ++-- src/inputFile_impl.h | 63 +++++++++++++++++++++++++++++++++++++++++++- src/load_balancer.h | 2 +- src/read_data.h | 13 ++++++--- src/system.h | 38 +++++++++++++------------- 6 files changed, 94 insertions(+), 28 deletions(-) diff --git a/input/in.lj b/input/in.lj index 9b5664a..e8af5a6 100644 --- a/input/in.lj +++ b/input/in.lj @@ -17,6 +17,7 @@ pair_coeff 1 1 1.0 1.0 2.5 neighbor 0.3 bin neigh_modify every 20 one 50 +comm_modify cutoff * 20 fix 1 all nve thermo 10 diff --git a/src/inputFile.h b/src/inputFile.h index 10503eb..257ee0a 100644 --- a/src/inputFile.h +++ b/src/inputFile.h @@ -179,8 +179,9 @@ class InputFile int binning_type = BINNING_LINKEDCELL; - int comm_type = COMM_MPI; - int comm_exchange_rate = 20; + int comm_type; + int comm_exchange_rate; + double comm_ghost_cutoff; int force_type = FORCE_LJ; int force_iteration_type; diff --git a/src/inputFile_impl.h b/src/inputFile_impl.h index 96d3433..feff2e9 100644 --- a/src/inputFile_impl.h +++ b/src/inputFile_impl.h @@ -77,6 +77,46 @@ InputFile::InputFile( InputCL commandline_, t_System *system_ ) output_file = commandline.output_file; error_file = commandline.error_file; + + // set defaults (matches ExaMiniMD LJ example) + + nsteps = 0; + + thermo_rate = 0; + dumpbinary_rate = 0; + correctness_rate = 0; + dumpbinaryflag = false; + correctnessflag = false; + timestepflag = false; + + lattice_offset_x = 0.0; + lattice_offset_y = 0.0; + lattice_offset_z = 0.0; + box[0] = 0; + box[2] = 0; + box[4] = 0; + box[1] = 40; + box[3] = 40; + box[5] = 40; + + units_style = UNITS_LJ; + lattice_style = LATTICE_FCC; + lattice_constant = 0.8442; + + temperature_target = 1.4; + temperature_seed = 87287; + + nsteps = 100; + thermo_rate = 10; + + neighbor_skin = 0.3; + neighbor_skin = 0.0; // for metal and real units + max_neigh_guess = 50; + comm_exchange_rate = 20; + comm_ghost_cutoff = std::pow( ( 4.0 / lattice_constant ), ( 1.0 / 3.0 ) ) * + 20.0; // 20 lattice constants + + force_cutoff = 2.5; } template @@ -372,6 +412,27 @@ void InputFile::check_lammps_command( std::string line, } } } + if ( keyword.compare( "comm_modify" ) == 0 ) + { + if ( words.at( 1 ).compare( "cutoff" ) == 0 ) + { + if ( words.at( 2 ).compare( "*" ) == 0 ) + { + known = true; + comm_ghost_cutoff = std::stod( words.at( 3 ) ); + } + else + { + log_err( err, "LAMMPS-Command: 'comm_modify' command only " + "supported for all atom types '*' in CabanaMD" ); + } + } + else + { + log_err( err, "LAMMPS-Command: 'comm_modify' command only supports " + "single cutoff 'cutoff' in CabanaMD" ); + } + } if ( keyword.compare( "fix" ) == 0 ) { if ( words.at( 3 ).compare( "nve" ) == 0 ) @@ -458,7 +519,7 @@ void InputFile::create_lattice( Comm *comm ) // global_high[0] *= 2; // global_high[1] *= 2; // global_high[2] *= 2; - system->create_domain( global_low, global_high ); + system->create_domain( global_low, global_high, comm_ghost_cutoff ); s = *system; auto local_mesh_lo_x = s.local_mesh_lo_x; diff --git a/src/load_balancer.h b/src/load_balancer.h index 53628d7..ab9e309 100644 --- a/src/load_balancer.h +++ b/src/load_balancer.h @@ -38,7 +38,7 @@ class LoadBalancer _system->ranks_per_dim.end() ); _liball->setProcGridParams( rank_dim_pos, ranks_per_dim ); // todo(sschulz): Do we need a minimum domain size? - //std::vector min_size( 3, 15 ); + // std::vector min_size( 3, 15 ); //_liball->setMinDomainSize( min_size ); _liball->setCommunicator( _comm ); _liball->setProcTag( _rank ); diff --git a/src/read_data.h b/src/read_data.h index 50a3081..77f113f 100644 --- a/src/read_data.h +++ b/src/read_data.h @@ -158,11 +158,16 @@ void read_lammps_header( std::ifstream &file, std::ofstream &err, t_System *s ) } // Create mesh - //high_corner[0] *= 2; - //high_corner[1] *= 2; - //high_corner[2] *= 2; + // high_corner[0] *= 2; + // high_corner[1] *= 2; + // high_corner[2] *= 2; printf( ">> %g %g %g\n", high_corner[0], high_corner[1], high_corner[2] ); - s->create_domain( low_corner, high_corner ); + // todo(sschulz): Is this retrievable from the data file? + double lattice_constant = 0.8442; // default lattice constant + double comm_ghost_cutoff = + std::pow( ( 4.0 / lattice_constant ), ( 1.0 / 3.0 ) ) * + 20.0; // 20 default lattice constants + s->create_domain( low_corner, high_corner, comm_ghost_cutoff ); } template diff --git a/src/system.h b/src/system.h index de2e8df..5a28bff 100644 --- a/src/system.h +++ b/src/system.h @@ -81,8 +81,6 @@ class SystemCommon typedef typename t_mass::HostMirror h_t_mass; t_mass mass; - int halo_width; - // Simulation total domain T_X_FLOAT global_mesh_x, global_mesh_y, global_mesh_z; @@ -92,8 +90,8 @@ class SystemCommon T_X_FLOAT local_mesh_hi_x, local_mesh_hi_y, local_mesh_hi_z; T_X_FLOAT ghost_mesh_lo_x, ghost_mesh_lo_y, ghost_mesh_lo_z; T_X_FLOAT ghost_mesh_hi_x, ghost_mesh_hi_y, ghost_mesh_hi_z; - std::shared_ptr< - Cabana::Grid::LocalGrid>> + T_X_FLOAT halo_width; + std::shared_ptr>> local_grid; std::shared_ptr>> global_grid; @@ -131,8 +129,9 @@ class SystemCommon ~SystemCommon() {} void create_domain( std::array low_corner, - std::array high_corner ) + std::array high_corner, double ghost_cutoff ) { + halo_width = ghost_cutoff; // Create the MPI partitions. Cabana::Grid::DimBlockPartitioner<3> partitioner; ranks_per_dim = partitioner.ranksPerDimension( MPI_COMM_WORLD, {} ); @@ -156,8 +155,7 @@ class SystemCommon } // Create a local mesh - halo_width = 1; - local_grid = Cajita::createLocalGrid( global_grid, halo_width ); + local_grid = Cajita::createLocalGrid( global_grid, 0 ); auto local_mesh = Cajita::createLocalMesh( *local_grid ); local_mesh_lo_x = local_mesh.lowCorner( Cajita::Own(), 0 ); @@ -166,12 +164,12 @@ class SystemCommon local_mesh_hi_x = local_mesh.highCorner( Cajita::Own(), 0 ); local_mesh_hi_y = local_mesh.highCorner( Cajita::Own(), 1 ); local_mesh_hi_z = local_mesh.highCorner( Cajita::Own(), 2 ); - ghost_mesh_lo_x = local_mesh.lowCorner( Cajita::Ghost(), 0 ); - ghost_mesh_lo_y = local_mesh.lowCorner( Cajita::Ghost(), 1 ); - ghost_mesh_lo_z = local_mesh.lowCorner( Cajita::Ghost(), 2 ); - ghost_mesh_hi_x = local_mesh.highCorner( Cajita::Ghost(), 0 ); - ghost_mesh_hi_y = local_mesh.highCorner( Cajita::Ghost(), 1 ); - ghost_mesh_hi_z = local_mesh.highCorner( Cajita::Ghost(), 2 ); + ghost_mesh_lo_x = local_mesh_lo_x - halo_width; + ghost_mesh_lo_y = local_mesh_lo_y - halo_width; + ghost_mesh_lo_z = local_mesh_lo_z - halo_width; + ghost_mesh_hi_x = local_mesh_hi_x + halo_width; + ghost_mesh_hi_y = local_mesh_hi_y + halo_width; + ghost_mesh_hi_z = local_mesh_hi_z + halo_width; local_mesh_x = local_mesh.extent( Cajita::Own(), 0 ); local_mesh_y = local_mesh.extent( Cajita::Own(), 1 ); local_mesh_z = local_mesh.extent( Cajita::Own(), 2 ); @@ -181,7 +179,7 @@ class SystemCommon local_mesh_lo_y, local_mesh_lo_z, local_mesh_hi_x, local_mesh_hi_y, local_mesh_hi_z ); } - // low_corner and high_corner are local corners, not globa as in + // low_corner and high_corner are local corners, not global as in // create_domain! void update_domain( std::array low_corner, std::array high_corner ) @@ -202,12 +200,12 @@ class SystemCommon local_mesh_x = local_mesh_hi_x - local_mesh_lo_x; local_mesh_y = local_mesh_hi_y - local_mesh_lo_y; local_mesh_z = local_mesh_hi_z - local_mesh_lo_z; - ghost_mesh_lo_x = local_mesh_lo_x - halo_width * local_mesh_x; - ghost_mesh_lo_y = local_mesh_lo_y - halo_width * local_mesh_y; - ghost_mesh_lo_z = local_mesh_lo_z - halo_width * local_mesh_z; - ghost_mesh_hi_x = local_mesh_hi_x + halo_width * local_mesh_x; - ghost_mesh_hi_y = local_mesh_hi_y + halo_width * local_mesh_y; - ghost_mesh_hi_z = local_mesh_hi_z + halo_width * local_mesh_z; + ghost_mesh_lo_x = local_mesh_lo_x - halo_width; + ghost_mesh_lo_y = local_mesh_lo_y - halo_width; + ghost_mesh_lo_z = local_mesh_lo_z - halo_width; + ghost_mesh_hi_x = local_mesh_hi_x + halo_width; + ghost_mesh_hi_y = local_mesh_hi_y + halo_width; + ghost_mesh_hi_z = local_mesh_hi_z + halo_width; printf( "%s: New extent: %g %g %g %g %g %g, %g %g %g\n", __func__, local_mesh_lo_x, local_mesh_lo_y, local_mesh_lo_z, local_mesh_hi_x, local_mesh_hi_y, local_mesh_hi_z, local_mesh_x, From 9d93c268059169d5a2ba10cc2a53fd7efe0962be Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Tue, 3 Aug 2021 20:44:02 +0200 Subject: [PATCH 12/50] add vtk particles output via 'dump * * vtk n filname*%' command --- input/in.lj | 2 + src/cabanamd_impl.h | 8 +++ src/inputFile.h | 3 + src/inputFile_impl.h | 33 +++++++++ src/vtk_domain_writer.h | 149 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 195 insertions(+) diff --git a/input/in.lj b/input/in.lj index e8af5a6..6d804d1 100644 --- a/input/in.lj +++ b/input/in.lj @@ -21,4 +21,6 @@ comm_modify cutoff * 20 fix 1 all nve thermo 10 +dump dmpvtk all vtk 10 dump_*%.vtu + run 100 diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index b604de7..33c54fe 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -51,6 +51,8 @@ #include #include +#include + #include #include #include @@ -266,6 +268,7 @@ template void CbnMD::run() { std::ofstream out( input->output_file, std::ofstream::app ); + std::ofstream err( input->error_file, std::ofstream::app ); auto neigh_cutoff = input->force_cutoff + input->neighbor_skin; bool half_neigh = input->force_iteration_type == FORCE_ITER_NEIGH_HALF; @@ -384,6 +387,10 @@ void CbnMD::run() lb->output( step ); } + if ( step % input->vtk_rate == 0 ) + VTKDomainWriter::writeParticles( MPI_COMM_WORLD, step, system, + input->vtk_file, err ); + if ( input->dumpbinaryflag ) dump_binary( step ); @@ -419,6 +426,7 @@ void CbnMD::run() " procs for ", nsteps, " steps with ", system->N, " atoms" ); } out.close(); + err.close(); if ( input->write_data_flag ) write_data( system, input->output_data_file ); diff --git a/src/inputFile.h b/src/inputFile.h index 257ee0a..25b5d79 100644 --- a/src/inputFile.h +++ b/src/inputFile.h @@ -204,6 +204,9 @@ class InputFile std::string output_data_file; bool read_data_flag = false; bool write_data_flag = false; + bool write_vtk_flag = false; + int vtk_rate; + std::string vtk_file; InputFile( InputCL cl, t_System *s ); void read_file( const char *filename = NULL ); diff --git a/src/inputFile_impl.h b/src/inputFile_impl.h index feff2e9..16f5411 100644 --- a/src/inputFile_impl.h +++ b/src/inputFile_impl.h @@ -85,6 +85,7 @@ InputFile::InputFile( InputCL commandline_, t_System *system_ ) thermo_rate = 0; dumpbinary_rate = 0; correctness_rate = 0; + vtk_rate = 0; dumpbinaryflag = false; correctnessflag = false; timestepflag = false; @@ -331,6 +332,38 @@ void InputFile::check_lammps_command( std::string line, write_data_flag = true; output_data_file = words.at( 1 ); } + if ( keyword.compare( "dump" ) == 0 ) + { + if ( words.at( 3 ).compare( "vtk" ) == 0 ) + { + known = true; + write_vtk_flag = true; + vtk_file = words.at( 5 ); + vtk_rate = std::stod( words.at( 4 ) ); + if ( words.at( 2 ).compare( "all" ) != 0 ) + { + log_err( err, "LAMMPS-Command: 'dump' command only supports " + "dumping 'all' types in CabanaMD" ); + } + size_t pos = 0; + pos = vtk_file.find( "*", pos ); + if ( std::string::npos == pos ) + log_err( err, + "LAMMPS-Command: 'dump' requires '*' in file name, so " + "it can be replaced by the time step in CabanaMD" ); + pos = 0; + pos = vtk_file.find( "%", pos ); + if ( std::string::npos == pos ) + log_err( err, + "LAMMPS-Command: 'dump' requires '%' in file name, so " + "it can be replaced by the rank in CabanaMD" ); + } + else + { + log_err( err, "LAMMPS-Command: 'dump' command only supports 'vtk' " + "in CabanaMD" ); + } + } if ( keyword.compare( "pair_style" ) == 0 ) { if ( words.at( 1 ).compare( "lj/cut" ) == 0 ) diff --git a/src/vtk_domain_writer.h b/src/vtk_domain_writer.h index f27442c..9afdddb 100644 --- a/src/vtk_domain_writer.h +++ b/src/vtk_domain_writer.h @@ -16,6 +16,7 @@ #include #include +// todo(sschulz): Refactor name of VTKDomainWriter to VTKWriter namespace VTKDomainWriter { // Write PVTU @@ -126,5 +127,153 @@ void writeDomain( MPI_Comm comm, int time_step, fclose( file ); } +void writeParticlesParallelFile( MPI_Comm comm, const int step, + std::string filename ) +{ + // Should only be called from a single rank + int size; + MPI_Comm_size( comm, &size ); + // Prepare actual filename + // todo(sschulz): Also separate filename construction into function + size_t pos = 0; + pos = filename.find( "*", pos ); + std::stringstream time_string; + time_string << step; + filename.replace( pos, 1, time_string.str() ); + std::string parallel_filename( filename ); + pos = 0; + pos = parallel_filename.find( "%", pos ); + std::string empty_string( "" ); + parallel_filename.replace( pos, 1, empty_string ); + pos = 0; + pos = parallel_filename.find( ".vtu", pos ); + parallel_filename.replace( pos, 4, ".pvtu" ); + FILE *file = fopen( parallel_filename.c_str(), "w" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\t\n" ); + for ( std::size_t i = 0; i < size; ++i ) + { + std::string piece_filename( filename ); + pos = 0; + pos = piece_filename.find( "%", pos ); + std::stringstream rank_string; + rank_string << "_" << i; + piece_filename.replace( pos, 1, rank_string.str() ); + fprintf( file, "\t\n", piece_filename.c_str() ); + } + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fclose( file ); +} + +// Write particles to vtu file +// filename must contain * and % which will be replaced by time step and _rank. +// The filename dump_*%.vtu will be create the file dump_43_325.vtu in time +// step 43 on rank 325. +template +void writeParticles( MPI_Comm comm, const int step, t_System *system, + std::string filename, std::ofstream &err ) +{ + int rank; + MPI_Comm_rank( comm, &rank ); + // Write parallel file + if ( rank == 1 ) + writeParticlesParallelFile( comm, step, filename ); + // Prepare actual filename + // todo(sschulz): Separate filename construction into function + size_t pos = 0; + pos = filename.find( "*", pos ); + if ( std::string::npos == pos ) + log_err( err, "VTK output file does not contain required '*'" ); + std::stringstream time_string; + time_string << step; + filename.replace( pos, 1, time_string.str() ); + pos = 0; + pos = filename.find( "%", pos ); + if ( std::string::npos == pos ) + log_err( err, "VTK output file does not contain required '%'" ); + std::stringstream rank_string; + rank_string << "_" << rank; + filename.replace( pos, 1, rank_string.str() ); + // Prepare data + System, + CabanaMD_LAYOUT> + host_system; + system->slice_x(); + auto x = system->x; + host_system.resize( x.size() ); + host_system.slice_x(); + auto host_x = host_system.x; + host_system.deep_copy( *system ); + host_system.slice_all(); + host_x = host_system.x; + auto host_id = host_system.id; + auto host_type = host_system.type; + auto host_v = host_system.v; + FILE *file = fopen( filename.c_str(), "w" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n", + system->N_local ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + for ( int n = 0; n < system->N_local; ++n ) + fprintf( file, "%g %g %g ", host_v( n, 0 ), host_v( n, 1 ), + host_v( n, 2 ) ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + for ( int n = 0; n < system->N_local; ++n ) + fprintf( file, "%d ", host_id( n ) ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + for ( int n = 0; n < system->N_local; ++n ) + fprintf( file, "%d ", host_type( n ) ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + for ( int n = 0; n < system->N_local; ++n ) + fprintf( file, "%g %g %g ", host_x( n, 0 ), host_x( n, 1 ), + host_x( n, 2 ) ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fclose( file ); +} + } // end namespace VTKDomainWriter #endif From 88958adfa1f5de2bc592d1a826ac8662855b766c Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Wed, 4 Aug 2021 18:10:59 +0200 Subject: [PATCH 13/50] fix some warnings and potential bugs --- src/vtk_domain_writer.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/vtk_domain_writer.h b/src/vtk_domain_writer.h index 9afdddb..e2f1b27 100644 --- a/src/vtk_domain_writer.h +++ b/src/vtk_domain_writer.h @@ -40,8 +40,8 @@ void writeDomainParallelFile( MPI_Comm comm, int time_step, fprintf( file, "\t\t\n" ); fprintf( file, "\t\n" ); - for ( std::size_t i = 0; i < size; ++i ) - fprintf( file, "\t\n", + for ( int i = 0; i < size; ++i ) + fprintf( file, "\t\n", basename.c_str(), time_step, i ); fprintf( file, "\n" ); fprintf( file, "\n" ); @@ -164,7 +164,7 @@ void writeParticlesParallelFile( MPI_Comm comm, const int step, fprintf( file, "\t\t\n" ); fprintf( file, "\t\n" ); - for ( std::size_t i = 0; i < size; ++i ) + for ( int i = 0; i < size; ++i ) { std::string piece_filename( filename ); pos = 0; From c2671bd3f1c399ef3f3ecdb5f18ebbdde04d9bcf Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 5 Aug 2021 09:07:42 +0200 Subject: [PATCH 14/50] fix resizing of pack_ranks_migrate_all --- src/comm_mpi_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comm_mpi_impl.h b/src/comm_mpi_impl.h index bb7a924..0a1983e 100644 --- a/src/comm_mpi_impl.h +++ b/src/comm_mpi_impl.h @@ -230,7 +230,7 @@ void Comm::exchange() // resized as well if ( pack_ranks_migrate_all.extent( 0 ) < x.size() ) { - max_local *= 1.1; + max_local = x.size() * 1.1; Kokkos::realloc( pack_ranks_migrate_all, max_local ); } pack_ranks_migrate = From f5d76c18a2397f0342a9b7542642a37434bd5789 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 5 Aug 2021 09:09:45 +0200 Subject: [PATCH 15/50] update atom initialization to not assume fully filled system --- src/inputFile_impl.h | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/src/inputFile_impl.h b/src/inputFile_impl.h index 16f5411..27df166 100644 --- a/src/inputFile_impl.h +++ b/src/inputFile_impl.h @@ -50,6 +50,7 @@ #include #include +#include #include std::vector split( const std::string &line ) @@ -562,12 +563,34 @@ void InputFile::create_lattice( Comm *comm ) auto local_mesh_hi_y = s.local_mesh_hi_y; auto local_mesh_hi_z = s.local_mesh_hi_z; - T_INT ix_start = local_mesh_lo_x / s.global_mesh_x * lattice_nx - 0.5; - T_INT iy_start = local_mesh_lo_y / s.global_mesh_y * lattice_ny - 0.5; - T_INT iz_start = local_mesh_lo_z / s.global_mesh_z * lattice_nz - 0.5; - T_INT ix_end = local_mesh_hi_x / s.global_mesh_x * lattice_nx + 0.5; - T_INT iy_end = local_mesh_hi_y / s.global_mesh_y * lattice_ny + 0.5; - T_INT iz_end = local_mesh_hi_z / s.global_mesh_z * lattice_nz + 0.5; + // T_INT ix_start = local_mesh_lo_x / s.global_mesh_x * lattice_nx - 0.5; + // T_INT iy_start = local_mesh_lo_y / s.global_mesh_y * lattice_ny - 0.5; + // T_INT iz_start = local_mesh_lo_z / s.global_mesh_z * lattice_nz - 0.5; + // T_INT ix_end = local_mesh_hi_x / s.global_mesh_x * lattice_nx + 0.5; + // T_INT iy_end = local_mesh_hi_y / s.global_mesh_y * lattice_ny + 0.5; + // T_INT iz_end = local_mesh_hi_z / s.global_mesh_z * lattice_nz + 0.5; + // todo(sschulz): This should be checked for missing boundary cells. + T_INT ix_start = local_mesh_lo_x / lattice_constant - 0.5; + T_INT iy_start = local_mesh_lo_y / lattice_constant - 0.5; + T_INT iz_start = local_mesh_lo_z / lattice_constant - 0.5; + T_INT ix_end = + std::max( std::min( static_cast( lattice_nx ), + local_mesh_hi_x / lattice_constant + 0.5 ), + static_cast( ix_start ) ); + T_INT iy_end = + std::max( std::min( static_cast( lattice_ny ), + local_mesh_hi_y / lattice_constant + 0.5 ), + static_cast( iy_start ) ); + T_INT iz_end = + std::max( std::min( static_cast( lattice_nz ), + local_mesh_hi_z / lattice_constant + 0.5 ), + static_cast( iz_start ) ); + if ( ix_start == ix_end ) + ix_end -= 1; + if ( iy_start == iy_end ) + iy_end -= 1; + if ( iz_start == iz_end ) + iz_end -= 1; // Create Simple Cubic Lattice if ( lattice_style == LATTICE_SC ) From 957a3fb233c6af5605a7007c67888ec76e5eba9b Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 5 Aug 2021 09:09:52 +0200 Subject: [PATCH 16/50] update format --- src/vtk_domain_writer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vtk_domain_writer.h b/src/vtk_domain_writer.h index e2f1b27..a90745c 100644 --- a/src/vtk_domain_writer.h +++ b/src/vtk_domain_writer.h @@ -41,8 +41,8 @@ void writeDomainParallelFile( MPI_Comm comm, int time_step, "NumberOfComponents=\"3\"/>\n" ); fprintf( file, "\t\n" ); for ( int i = 0; i < size; ++i ) - fprintf( file, "\t\n", - basename.c_str(), time_step, i ); + fprintf( file, "\t\n", basename.c_str(), + time_step, i ); fprintf( file, "\n" ); fprintf( file, "\n" ); fclose( file ); From 6cdec01524c9f3408fb45d4e9cabafdf310a5427 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 5 Aug 2021 09:10:44 +0200 Subject: [PATCH 17/50] use actual number of particles for balancing --- src/load_balancer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/load_balancer.h b/src/load_balancer.h index ab9e309..a0c8f9d 100644 --- a/src/load_balancer.h +++ b/src/load_balancer.h @@ -59,7 +59,7 @@ class LoadBalancer { int rank; MPI_Comm_rank( _comm, &rank ); - double work = ( _system->N_local + _system->N_ghost ) / ( rank + 1 ); + double work = ( _system->N_local + _system->N_ghost ); printf( ">> Work: %g\n", work ); _liball->setWork( work ); _liball->balance(); From 831d011ba3c71b9ad91a6a2f41a8ed1afe4307ae Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 5 Aug 2021 09:11:25 +0200 Subject: [PATCH 18/50] REVERT BEFORE MERGING increase system box by *2 --- src/inputFile_impl.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/inputFile_impl.h b/src/inputFile_impl.h index 27df166..e861469 100644 --- a/src/inputFile_impl.h +++ b/src/inputFile_impl.h @@ -550,9 +550,9 @@ void InputFile::create_lattice( Comm *comm ) T_X_FLOAT max_z = lattice_constant * lattice_nz; std::array global_low = { 0.0, 0.0, 0.0 }; std::array global_high = { max_x, max_y, max_z }; - // global_high[0] *= 2; - // global_high[1] *= 2; - // global_high[2] *= 2; + global_high[0] *= 2; + global_high[1] *= 2; + global_high[2] *= 2; system->create_domain( global_low, global_high, comm_ghost_cutoff ); s = *system; From ff820d1303dbb975c96f4ce1af63a778ec1bd87b Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 5 Aug 2021 13:17:10 +0200 Subject: [PATCH 19/50] a little bit of cleanup --- src/read_data.h | 4 ---- src/system.h | 6 ++---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/read_data.h b/src/read_data.h index 77f113f..e0b7165 100644 --- a/src/read_data.h +++ b/src/read_data.h @@ -158,10 +158,6 @@ void read_lammps_header( std::ifstream &file, std::ofstream &err, t_System *s ) } // Create mesh - // high_corner[0] *= 2; - // high_corner[1] *= 2; - // high_corner[2] *= 2; - printf( ">> %g %g %g\n", high_corner[0], high_corner[1], high_corner[2] ); // todo(sschulz): Is this retrievable from the data file? double lattice_constant = 0.8442; // default lattice constant double comm_ghost_cutoff = diff --git a/src/system.h b/src/system.h index 5a28bff..63c18a4 100644 --- a/src/system.h +++ b/src/system.h @@ -93,8 +93,6 @@ class SystemCommon T_X_FLOAT halo_width; std::shared_ptr>> local_grid; - std::shared_ptr>> - global_grid; // Only needed for current comm std::array ranks_per_dim; @@ -146,8 +144,8 @@ class SystemCommon // Create the global grid. std::array is_periodic = { true, true, true }; - global_grid = Cajita::createGlobalGrid( MPI_COMM_WORLD, global_mesh, - is_periodic, partitioner ); + auto global_grid = Cajita::createGlobalGrid( + MPI_COMM_WORLD, global_mesh, is_periodic, partitioner ); for ( int d = 0; d < 3; d++ ) { From 6b4dcea18a951ce4d225165bbe793c89c086750f Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 5 Aug 2021 13:22:34 +0200 Subject: [PATCH 20/50] rename VTKDomainWriter to VTKWriter --- src/cabanamd_impl.h | 2 +- src/load_balancer.h | 4 ++-- src/vtk_domain_writer.h | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index 33c54fe..f937a6e 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -388,7 +388,7 @@ void CbnMD::run() } if ( step % input->vtk_rate == 0 ) - VTKDomainWriter::writeParticles( MPI_COMM_WORLD, step, system, + VTKWriter::writeParticles( MPI_COMM_WORLD, step, system, input->vtk_file, err ); if ( input->dumpbinaryflag ) diff --git a/src/load_balancer.h b/src/load_balancer.h index a0c8f9d..6289823 100644 --- a/src/load_balancer.h +++ b/src/load_balancer.h @@ -88,7 +88,7 @@ class LoadBalancer _system->local_mesh_lo_x, _system->local_mesh_lo_y, _system->local_mesh_lo_z, _system->local_mesh_hi_x, _system->local_mesh_hi_y, _system->local_mesh_hi_z }; - VTKDomainWriter::writeDomain( _comm, t, vertices, + VTKWriter::writeDomain( _comm, t, vertices, vtk_actual_domain_basename ); std::vector> updated_vertices = @@ -97,7 +97,7 @@ class LoadBalancer vertices[d] = updated_vertices.at( 0 )[d]; for ( std::size_t d = 3; d < 6; ++d ) vertices[d] = updated_vertices.at( 1 )[d - 3]; - VTKDomainWriter::writeDomain( _comm, t, vertices, + VTKWriter::writeDomain( _comm, t, vertices, vtk_lb_domain_basename ); } diff --git a/src/vtk_domain_writer.h b/src/vtk_domain_writer.h index a90745c..598cdb6 100644 --- a/src/vtk_domain_writer.h +++ b/src/vtk_domain_writer.h @@ -16,8 +16,7 @@ #include #include -// todo(sschulz): Refactor name of VTKDomainWriter to VTKWriter -namespace VTKDomainWriter +namespace VTKWriter { // Write PVTU void writeDomainParallelFile( MPI_Comm comm, int time_step, @@ -47,7 +46,9 @@ void writeDomainParallelFile( MPI_Comm comm, int time_step, fprintf( file, "\n" ); fclose( file ); } + // Write VTU for domain (low corner, high corner) +// basename will be appended with the corresponding time step, rank and extension void writeDomain( MPI_Comm comm, int time_step, std::array &domain_vertices, std::string &basename ) @@ -275,5 +276,5 @@ void writeParticles( MPI_Comm comm, const int step, t_System *system, fclose( file ); } -} // end namespace VTKDomainWriter +} // end namespace VTKWriter #endif From e2f50ff5e161ba88d62d427142aac6cba5a783b0 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 5 Aug 2021 13:24:49 +0200 Subject: [PATCH 21/50] rename vtk_domain_writer.h to vtk_writer.h --- src/cabanamd_impl.h | 1 + src/load_balancer.h | 2 +- src/{vtk_domain_writer.h => vtk_writer.h} | 0 3 files changed, 2 insertions(+), 1 deletion(-) rename src/{vtk_domain_writer.h => vtk_writer.h} (100%) diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index f937a6e..1af617e 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -58,6 +58,7 @@ #include #include #include +#include #include #include diff --git a/src/load_balancer.h b/src/load_balancer.h index 6289823..a910c48 100644 --- a/src/load_balancer.h +++ b/src/load_balancer.h @@ -13,7 +13,7 @@ #define LOADBALANCER_H #include -#include +#include #include diff --git a/src/vtk_domain_writer.h b/src/vtk_writer.h similarity index 100% rename from src/vtk_domain_writer.h rename to src/vtk_writer.h From 3309e85ba375c03422784a9abc917127564e2ab2 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 5 Aug 2021 13:30:27 +0200 Subject: [PATCH 22/50] also output local work in domain output --- src/cabanamd_impl.h | 2 +- src/load_balancer.h | 15 ++++++++------- src/vtk_writer.h | 10 ++++++++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index 1af617e..34dff20 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -390,7 +390,7 @@ void CbnMD::run() if ( step % input->vtk_rate == 0 ) VTKWriter::writeParticles( MPI_COMM_WORLD, step, system, - input->vtk_file, err ); + input->vtk_file, err ); if ( input->dumpbinaryflag ) dump_binary( step ); diff --git a/src/load_balancer.h b/src/load_balancer.h index a910c48..619f7f2 100644 --- a/src/load_balancer.h +++ b/src/load_balancer.h @@ -59,9 +59,9 @@ class LoadBalancer { int rank; MPI_Comm_rank( _comm, &rank ); - double work = ( _system->N_local + _system->N_ghost ); - printf( ">> Work: %g\n", work ); - _liball->setWork( work ); + _work = ( _system->N_local + _system->N_ghost ); + printf( ">> Work: %g\n", _work ); + _liball->setWork( _work ); _liball->balance(); std::vector> updated_vertices = _liball->getVertices(); @@ -88,8 +88,8 @@ class LoadBalancer _system->local_mesh_lo_x, _system->local_mesh_lo_y, _system->local_mesh_lo_z, _system->local_mesh_hi_x, _system->local_mesh_hi_y, _system->local_mesh_hi_z }; - VTKWriter::writeDomain( _comm, t, vertices, - vtk_actual_domain_basename ); + VTKWriter::writeDomain( _comm, t, vertices, _work, + vtk_actual_domain_basename ); std::vector> updated_vertices = _liball->getVertices(); @@ -97,8 +97,8 @@ class LoadBalancer vertices[d] = updated_vertices.at( 0 )[d]; for ( std::size_t d = 3; d < 6; ++d ) vertices[d] = updated_vertices.at( 1 )[d - 3]; - VTKWriter::writeDomain( _comm, t, vertices, - vtk_lb_domain_basename ); + VTKWriter::writeDomain( _comm, t, vertices, _work, + vtk_lb_domain_basename ); } private: @@ -106,6 +106,7 @@ class LoadBalancer t_System *_system; std::shared_ptr> _liball; int _rank; + double _work; }; #endif diff --git a/src/vtk_writer.h b/src/vtk_writer.h index 598cdb6..7335975 100644 --- a/src/vtk_writer.h +++ b/src/vtk_writer.h @@ -34,6 +34,7 @@ void writeDomainParallelFile( MPI_Comm comm, int time_step, fprintf( file, "\n" ); fprintf( file, "\t\n" ); fprintf( file, "\t\t\n" ); + fprintf( file, "\t\t\n" ); fprintf( file, "\t\n" ); fprintf( file, "\t\n" ); fprintf( file, "\t\t &domain_vertices, + std::array &domain_vertices, double work, std::string &basename ) { int rank; @@ -99,6 +101,10 @@ void writeDomain( MPI_Comm comm, int time_step, "NumberOfComponents=\"1\" format=\"ascii\">\n" ); fprintf( file, "%d", rank ); fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "%g", work ); + fprintf( file, "\n\t\t\n" ); fprintf( file, "\t\n" ); fprintf( file, "\t\n" ); fprintf( file, "\t\t Date: Thu, 5 Aug 2021 13:42:16 +0200 Subject: [PATCH 23/50] output current, not last work --- src/load_balancer.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/load_balancer.h b/src/load_balancer.h index 619f7f2..8e4e67b 100644 --- a/src/load_balancer.h +++ b/src/load_balancer.h @@ -59,9 +59,9 @@ class LoadBalancer { int rank; MPI_Comm_rank( _comm, &rank ); - _work = ( _system->N_local + _system->N_ghost ); - printf( ">> Work: %g\n", _work ); - _liball->setWork( _work ); + double work = ( _system->N_local + _system->N_ghost ); + printf( ">> Work: %g\n", work ); + _liball->setWork( work ); _liball->balance(); std::vector> updated_vertices = _liball->getVertices(); @@ -83,12 +83,13 @@ class LoadBalancer std::string vtk_actual_domain_basename( "domain_act" ); std::string vtk_lb_domain_basename( "domain_lb" ); // _liball->printVTKoutlines( t ); + double work = ( _system->N_local + _system->N_ghost ); std::array vertices = { _system->local_mesh_lo_x, _system->local_mesh_lo_y, _system->local_mesh_lo_z, _system->local_mesh_hi_x, _system->local_mesh_hi_y, _system->local_mesh_hi_z }; - VTKWriter::writeDomain( _comm, t, vertices, _work, + VTKWriter::writeDomain( _comm, t, vertices, work, vtk_actual_domain_basename ); std::vector> updated_vertices = @@ -97,7 +98,7 @@ class LoadBalancer vertices[d] = updated_vertices.at( 0 )[d]; for ( std::size_t d = 3; d < 6; ++d ) vertices[d] = updated_vertices.at( 1 )[d - 3]; - VTKWriter::writeDomain( _comm, t, vertices, _work, + VTKWriter::writeDomain( _comm, t, vertices, work, vtk_lb_domain_basename ); } @@ -106,7 +107,6 @@ class LoadBalancer t_System *_system; std::shared_ptr> _liball; int _rank; - double _work; }; #endif From a0b3f586bd67913dec3123db405341926d38bae0 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 5 Aug 2021 13:45:19 +0200 Subject: [PATCH 24/50] remove remaining debug output --- src/load_balancer.h | 5 ----- src/system.h | 16 ---------------- 2 files changed, 21 deletions(-) diff --git a/src/load_balancer.h b/src/load_balancer.h index 8e4e67b..0596cfa 100644 --- a/src/load_balancer.h +++ b/src/load_balancer.h @@ -60,15 +60,10 @@ class LoadBalancer int rank; MPI_Comm_rank( _comm, &rank ); double work = ( _system->N_local + _system->N_ghost ); - printf( ">> Work: %g\n", work ); _liball->setWork( work ); _liball->balance(); std::vector> updated_vertices = _liball->getVertices(); - printf( ">> New Vertices: %g %g %g %g %g %g\n", - updated_vertices.at( 0 )[0], updated_vertices.at( 0 )[1], - updated_vertices.at( 0 )[2], updated_vertices.at( 1 )[0], - updated_vertices.at( 1 )[1], updated_vertices.at( 1 )[2] ); std::array low_corner = { updated_vertices.at( 0 )[0], updated_vertices.at( 0 )[1], updated_vertices.at( 0 )[2] }; diff --git a/src/system.h b/src/system.h index 63c18a4..5c27b99 100644 --- a/src/system.h +++ b/src/system.h @@ -171,24 +171,12 @@ class SystemCommon local_mesh_x = local_mesh.extent( Cajita::Own(), 0 ); local_mesh_y = local_mesh.extent( Cajita::Own(), 1 ); local_mesh_z = local_mesh.extent( Cajita::Own(), 2 ); - printf( ">> global: %g %g %g %g %g %g\n", low_corner[0], low_corner[1], - low_corner[2], high_corner[0], high_corner[1], high_corner[2] ); - printf( ">> local: %g %g %g %g %g %g\n", local_mesh_lo_x, - local_mesh_lo_y, local_mesh_lo_z, local_mesh_hi_x, - local_mesh_hi_y, local_mesh_hi_z ); } // low_corner and high_corner are local corners, not global as in // create_domain! void update_domain( std::array low_corner, std::array high_corner ) { - printf( "%s: Old extent: %g %g %g %g %g %g, %g %g %g\n", __func__, - local_mesh_lo_x, local_mesh_lo_y, local_mesh_lo_z, - local_mesh_hi_x, local_mesh_hi_y, local_mesh_hi_z, local_mesh_x, - local_mesh_y, local_mesh_z ); - printf( "%s: New corners: %g %g %g %g %g %g\n", __func__, low_corner[0], - low_corner[1], low_corner[2], high_corner[0], high_corner[1], - high_corner[2] ); local_mesh_lo_x = low_corner[0]; local_mesh_lo_y = low_corner[1]; local_mesh_lo_z = low_corner[2]; @@ -204,10 +192,6 @@ class SystemCommon ghost_mesh_hi_x = local_mesh_hi_x + halo_width; ghost_mesh_hi_y = local_mesh_hi_y + halo_width; ghost_mesh_hi_z = local_mesh_hi_z + halo_width; - printf( "%s: New extent: %g %g %g %g %g %g, %g %g %g\n", __func__, - local_mesh_lo_x, local_mesh_lo_y, local_mesh_lo_z, - local_mesh_hi_x, local_mesh_hi_y, local_mesh_hi_z, local_mesh_x, - local_mesh_y, local_mesh_z ); } void slice_all() From bf97931d3ee0b77099551d6658a961896cf40788 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 5 Aug 2021 20:42:20 +0200 Subject: [PATCH 25/50] add ALL to ci --- .github/workflows/CI.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2fc1b12..5770183 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -126,6 +126,15 @@ jobs: mkdir -p $HOME/n2p2/include mkdir -p $HOME/n2p2/lib make libnnpif INTERFACES=CabanaMD PROJECT_INCLUDE=$HOME/n2p2/include PROJECT_LIB=$HOME/n2p2/lib + - name: Checkout ALL + run: | + git clone --depth 1 --branch master https://gitlab.jsc.fz-juelich.de/SLMS/loadbalancing ALL + - name: Build ALL + working-directory: ALL + run: | + cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$HOME/ALL + cmake --build build --parallel 2 + cmake --install build - name: Checkout CabanaMD uses: actions/checkout@v3 - name: Build CabanaMD @@ -135,7 +144,7 @@ jobs: -DMPIEXEC_MAX_NUMPROCS=2 -DMPIEXEC_PREFLAGS="--oversubscribe" \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ -DCMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \ - -DCMAKE_PREFIX_PATH="$HOME/Cabana" \ + -DCMAKE_PREFIX_PATH="$HOME/Cabana;$HOME/ALL" \ -DCabanaMD_ENABLE_TESTING=ON \ -DCabanaMD_LAYOUT=${{ matrix.layout }} \ -DCabanaMD_VECTORLENGTH=${{ matrix.vector }} \ From b768bfc866b034212ab4d8ec047a50a504f7833c Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 5 Aug 2021 20:56:44 +0200 Subject: [PATCH 26/50] create additional create_domain that estimates ghost_cutoff --- src/system.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/system.h b/src/system.h index 5c27b99..fe7abab 100644 --- a/src/system.h +++ b/src/system.h @@ -55,6 +55,7 @@ #include +#include #include #include @@ -126,6 +127,15 @@ class SystemCommon ~SystemCommon() {} + void create_domain( std::array low_corner, + std::array high_corner ) + { + double ghost_cutoff = + std::max( std::max( high_corner[0] - low_corner[0], + high_corner[2] - low_corner[1] ), + high_corner[2] - low_corner[2] ); + create_domain( low_corner, high_corner, ghost_cutoff ); + } void create_domain( std::array low_corner, std::array high_corner, double ghost_cutoff ) { From a85d8a104cf8fbc40fbbf77d151b9a55e2896625 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 5 Aug 2021 20:59:32 +0200 Subject: [PATCH 27/50] if reading a data file, use default ghost_offset --- src/read_data.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/read_data.h b/src/read_data.h index e0b7165..2a4490c 100644 --- a/src/read_data.h +++ b/src/read_data.h @@ -158,12 +158,7 @@ void read_lammps_header( std::ifstream &file, std::ofstream &err, t_System *s ) } // Create mesh - // todo(sschulz): Is this retrievable from the data file? - double lattice_constant = 0.8442; // default lattice constant - double comm_ghost_cutoff = - std::pow( ( 4.0 / lattice_constant ), ( 1.0 / 3.0 ) ) * - 20.0; // 20 default lattice constants - s->create_domain( low_corner, high_corner, comm_ghost_cutoff ); + s->create_domain( low_corner, high_corner ); } template From 6c98cb45f7dc3f3ba219f3dd0922dc27f0f805bb Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Fri, 13 Aug 2021 16:19:31 +0200 Subject: [PATCH 28/50] use CabanaMD_ENABLE_ALL to enable load balancing --- .github/workflows/CI.yml | 1 + CMakeLists.txt | 8 ++++++-- src/CMakeLists.txt | 9 ++++++++- src/cabanamd.h | 7 ++++++- src/cabanamd_impl.h | 7 +++++++ 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5770183..5d20167 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -146,6 +146,7 @@ jobs: -DCMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \ -DCMAKE_PREFIX_PATH="$HOME/Cabana;$HOME/ALL" \ -DCabanaMD_ENABLE_TESTING=ON \ + -DCabanaMD_ENBALE_ALL=ON \ -DCabanaMD_LAYOUT=${{ matrix.layout }} \ -DCabanaMD_VECTORLENGTH=${{ matrix.vector }} \ -DCabanaMD_ENABLE_NNP=${{ matrix.nnp }} \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a4b9ff..1b6352e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,8 +25,6 @@ if( NOT Cabana_ENABLE_CAJITA ) message( FATAL_ERROR "Cabana must be compiled with Cajita" ) endif() -find_package(ALL 0.9.1 REQUIRED) - ##---------------------------------------------------------------------------## # Set up optional libraries ##---------------------------------------------------------------------------## @@ -35,6 +33,12 @@ if(NOT DEFINED N2P2_DIR) set(N2P2_DIR ~/n2p2/) endif() +option(CabanaMD_ENABLE_ALL "Build CabanaMD with ALL load balancing" OFF) +if(CabanaMD_ENABLE_ALL) + # todo(sschulz): All versions from 0.9.2 onwards should be fine. + find_package(ALL 0.9.2 REQUIRED) +endif() + ##---------------------------------------------------------------------------## ## Print the Git revision number to stdout ##---------------------------------------------------------------------------## diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e089b30..6ecd4ec 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -62,6 +62,11 @@ configure_file(CabanaMD_config.hpp.cmakein CabanaMD_config.hpp @ONLY) file(GLOB HEADERS_PUBLIC GLOB *.h force_types/*.h neighbor_types/*.h system_types/*.h ) +if(NOT CabanaMD_ENABLE_ALL) + list(REMOVE_ITEM HEADERS_PUBLIC + "load_balancer.h" + ) +endif() file(GLOB SOURCES *.cpp @@ -89,7 +94,9 @@ target_include_directories(CabanaMD PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/system_types $ $) -target_link_libraries(CabanaMD ALL::ALL) +if(CabanaMD_ENBALE_ALL) + target_link_libraries(CabanaMD ALL::ALL) +endif() #------------------------------------------------------------ diff --git a/src/cabanamd.h b/src/cabanamd.h index eca3f7a..c4902c6 100644 --- a/src/cabanamd.h +++ b/src/cabanamd.h @@ -55,9 +55,12 @@ #include #include #include -#include #include +#ifdef CabanaMD_ENABLE_ALL +#include +#endif + class CabanaMD { public: @@ -82,7 +85,9 @@ class CbnMD : public CabanaMD Comm *comm; Binning *binning; InputFile *input; +#ifdef CabanaMD_ENABLE_ALL LoadBalancer *lb; +#endif void init( InputCL cl ) override; void run() override; diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index 34dff20..1b4635b 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -227,7 +227,9 @@ void CbnMD::init( InputCL commandline ) comm->update_force(); } +#ifdef CabanaMD_ENABLE_ALL lb = new LoadBalancer( system ); +#endif // Initial output int step = 0; @@ -299,10 +301,12 @@ void CbnMD::run() if ( step % input->comm_exchange_rate == 0 && step > 0 ) { +#ifdef CabanaMD_ENABLE_ALL // Update domain decomposition lb_timer.reset(); lb->balance(); lb_time += lb_timer.seconds(); +#endif // Exchange atoms across MPI ranks comm_timer.reset(); @@ -385,7 +389,9 @@ void CbnMD::run() " ", T, " ", PE, " ", PE + KE, " ", time ); last_time = time; } +#ifdef CabanaMD_ENABLE_ALL lb->output( step ); +#endif } if ( step % input->vtk_rate == 0 ) @@ -408,6 +414,7 @@ void CbnMD::run() { double steps_per_sec = 1.0 * nsteps / time; double atom_steps_per_sec = system->N * steps_per_sec; + // todo(sschulz): Properly remove lb timing if not enabled. log( out, std::fixed, std::setprecision( 2 ), "\n#Procs Atoms | Time T_Force T_Neigh T_Comm T_Int T_lb ", "T_Other |\n", comm->num_processes(), " ", system->N, " | ", time, From 87585ecb20352d210c19214ef7539c5f6364e340 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 26 Aug 2021 09:20:58 +0200 Subject: [PATCH 29/50] change ALL library detection to Cabana like automatism --- CMakeLists.txt | 23 ++++++++++++++++++----- src/CMakeLists.txt | 3 --- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b6352e..d6fc6d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,22 @@ if( NOT Cabana_ENABLE_CAJITA ) message( FATAL_ERROR "Cabana must be compiled with Cajita" ) endif() +##---------------------------------------------------------------------------## +# Macro for optional dependencies (taken from Cabana) +##---------------------------------------------------------------------------## +macro(CabanaMD_add_dependency) + cmake_parse_arguments(CABANAMD_DEPENDENCY "" "PACKAGE;VERSION" "" ${ARGN}) + find_package( ${CABANAMD_DEPENDENCY_PACKAGE} ${CABANAMD_DEPENDENCY_VERSION} QUIET ) + string(TOUPPER "${CABANAMD_DEPENDENCY_PACKAGE}" CABANAMD_DEPENDENCY_OPTION ) + option( + CabanaMD_REQUIRE_${CABANAMD_DEPENDENCY_OPTION} + "Require CabanaMD to build with ${CABANAMD_DEPENDENCY_PACKAGE} support" ${CABANAMD_DEPENDENCY_PACKAGE}_FOUND) + if(CabanaMD_REQUIRE_${CABANAMD_DEPENDENCY_OPTION}) + find_package( ${CABANAMD_DEPENDENCY_PACKAGE} ${CABANAMD_DEPENDENCY_VERSION} REQUIRED ) + endif() + set(CabanaMD_ENABLE_${CABANAMD_DEPENDENCY_OPTION} ${${CABANAMD_DEPENDENCY_PACKAGE}_FOUND}) +endmacro() + ##---------------------------------------------------------------------------## # Set up optional libraries ##---------------------------------------------------------------------------## @@ -33,11 +49,8 @@ if(NOT DEFINED N2P2_DIR) set(N2P2_DIR ~/n2p2/) endif() -option(CabanaMD_ENABLE_ALL "Build CabanaMD with ALL load balancing" OFF) -if(CabanaMD_ENABLE_ALL) - # todo(sschulz): All versions from 0.9.2 onwards should be fine. - find_package(ALL 0.9.2 REQUIRED) -endif() +# todo(sschulz): All versions from 0.9.2 onwards should be fine. +CabanaMD_add_dependency( PACKAGE "ALL" VERSION 0.9.2 ) ##---------------------------------------------------------------------------## ## Print the Git revision number to stdout diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6ecd4ec..e362d13 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -94,9 +94,6 @@ target_include_directories(CabanaMD PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/system_types $ $) -if(CabanaMD_ENBALE_ALL) - target_link_libraries(CabanaMD ALL::ALL) -endif() #------------------------------------------------------------ From 4310c85bc516aadf9da265f435062d971533d1f7 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Mon, 9 Aug 2021 12:10:11 +0200 Subject: [PATCH 30/50] use Cajita grid to balance --- src/system.h | 101 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 37 deletions(-) diff --git a/src/system.h b/src/system.h index fe7abab..5fb59b6 100644 --- a/src/system.h +++ b/src/system.h @@ -56,7 +56,10 @@ #include #include +#include +#include #include +#include #include template @@ -84,6 +87,8 @@ class SystemCommon // Simulation total domain T_X_FLOAT global_mesh_x, global_mesh_y, global_mesh_z; + T_X_FLOAT grid_cell_size; + int grid_num_cells; // Simulation sub domain (single MPI rank) T_X_FLOAT local_mesh_x, local_mesh_y, local_mesh_z; @@ -94,6 +99,8 @@ class SystemCommon T_X_FLOAT halo_width; std::shared_ptr>> local_grid; + std::shared_ptr>> + global_grid; // Only needed for current comm std::array ranks_per_dim; @@ -113,6 +120,9 @@ class SystemCommon mass = t_mass( "System::mass", ntypes ); + // todo(sschulz): Choose a suitably small grid cell automatically + grid_cell_size = 0.0; + grid_num_cells = 2000; global_mesh_x = global_mesh_y = global_mesh_z = 0.0; local_mesh_lo_x = local_mesh_lo_y = local_mesh_lo_z = 0.0; local_mesh_hi_x = local_mesh_hi_y = local_mesh_hi_z = 0.0; @@ -144,9 +154,21 @@ class SystemCommon Cabana::Grid::DimBlockPartitioner<3> partitioner; ranks_per_dim = partitioner.ranksPerDimension( MPI_COMM_WORLD, {} ); + // todo(sschulz): Generalize, so non cubic dimensions are allowed + grid_cell_size = ( high_corner[0] - low_corner[0] ) / grid_num_cells; + if ( std::abs( ( high_corner[0] - low_corner[0] ) - + ( high_corner[1] - low_corner[1] ) ) > + T_X_FLOAT( 100.0 ) * std::numeric_limits::epsilon() ) + throw std::logic_error( "Dimensions must be cubic" ); + else if ( std::abs( ( high_corner[1] - low_corner[1] ) - + ( high_corner[2] - low_corner[2] ) ) > + T_X_FLOAT( 100.0 ) * + std::numeric_limits::epsilon() ) + throw std::logic_error( "Dimensions must be cubic" ); + // Create global mesh of MPI partitions. - auto global_mesh = Cabana::Grid::createUniformGlobalMesh( - low_corner, high_corner, ranks_per_dim ); + auto global_mesh = Cajita::createUniformGlobalMesh( + low_corner, high_corner, grid_cell_size ); global_mesh_x = global_mesh->extent( 0 ); global_mesh_y = global_mesh->extent( 1 ); @@ -154,8 +176,8 @@ class SystemCommon // Create the global grid. std::array is_periodic = { true, true, true }; - auto global_grid = Cajita::createGlobalGrid( - MPI_COMM_WORLD, global_mesh, is_periodic, partitioner ); + global_grid = Cajita::createGlobalGrid( MPI_COMM_WORLD, global_mesh, + is_periodic, partitioner ); for ( int d = 0; d < 3; d++ ) { @@ -163,45 +185,27 @@ class SystemCommon } // Create a local mesh - local_grid = Cajita::createLocalGrid( global_grid, 0 ); - auto local_mesh = Cajita::createLocalMesh( *local_grid ); - - local_mesh_lo_x = local_mesh.lowCorner( Cajita::Own(), 0 ); - local_mesh_lo_y = local_mesh.lowCorner( Cajita::Own(), 1 ); - local_mesh_lo_z = local_mesh.lowCorner( Cajita::Own(), 2 ); - local_mesh_hi_x = local_mesh.highCorner( Cajita::Own(), 0 ); - local_mesh_hi_y = local_mesh.highCorner( Cajita::Own(), 1 ); - local_mesh_hi_z = local_mesh.highCorner( Cajita::Own(), 2 ); - ghost_mesh_lo_x = local_mesh_lo_x - halo_width; - ghost_mesh_lo_y = local_mesh_lo_y - halo_width; - ghost_mesh_lo_z = local_mesh_lo_z - halo_width; - ghost_mesh_hi_x = local_mesh_hi_x + halo_width; - ghost_mesh_hi_y = local_mesh_hi_y + halo_width; - ghost_mesh_hi_z = local_mesh_hi_z + halo_width; - local_mesh_x = local_mesh.extent( Cajita::Own(), 0 ); - local_mesh_y = local_mesh.extent( Cajita::Own(), 1 ); - local_mesh_z = local_mesh.extent( Cajita::Own(), 2 ); + int halo_width = std::ceil( ghost_cutoff / grid_cell_size ); + local_grid = Cajita::createLocalGrid( global_grid, halo_width ); + // Update local_mesh_* and ghost_mesh_* info + update_mesh_info(); } // low_corner and high_corner are local corners, not global as in // create_domain! void update_domain( std::array low_corner, std::array high_corner ) { - local_mesh_lo_x = low_corner[0]; - local_mesh_lo_y = low_corner[1]; - local_mesh_lo_z = low_corner[2]; - local_mesh_hi_x = high_corner[0]; - local_mesh_hi_y = high_corner[1]; - local_mesh_hi_z = high_corner[2]; - local_mesh_x = local_mesh_hi_x - local_mesh_lo_x; - local_mesh_y = local_mesh_hi_y - local_mesh_lo_y; - local_mesh_z = local_mesh_hi_z - local_mesh_lo_z; - ghost_mesh_lo_x = local_mesh_lo_x - halo_width; - ghost_mesh_lo_y = local_mesh_lo_y - halo_width; - ghost_mesh_lo_z = local_mesh_lo_z - halo_width; - ghost_mesh_hi_x = local_mesh_hi_x + halo_width; - ghost_mesh_hi_y = local_mesh_hi_y + halo_width; - ghost_mesh_hi_z = local_mesh_hi_z + halo_width; + // Calculate new local grid offset + std::array cell_index_lo, cell_index_hi; + for ( std::size_t d = 0; d < 3; ++d ) + cell_index_lo[d] = std::rint( low_corner[d] / grid_cell_size ); + for ( std::size_t d = 0; d < 3; ++d ) + cell_index_hi[d] = std::rint( high_corner[d] / grid_cell_size ); + std::array num_cell; + for ( std::size_t d = 0; d < 3; ++d ) + num_cell[d] = cell_index_hi[d] - cell_index_lo[d]; + global_grid->setNumCellAndOffset( num_cell, cell_index_lo ); + update_mesh_info(); } void slice_all() @@ -245,6 +249,29 @@ class SystemCommon migrate( std::shared_ptr> distributor ) = 0; virtual void gather( std::shared_ptr> halo ) = 0; virtual const char *name() { return "SystemNone"; } + + private: + // Update local_mesh_* and ghost_mesh* info from global grid + void update_mesh_info() + { + auto local_mesh = Cajita::createLocalMesh( *local_grid ); + + local_mesh_lo_x = local_mesh.lowCorner( Cajita::Own(), 0 ); + local_mesh_lo_y = local_mesh.lowCorner( Cajita::Own(), 1 ); + local_mesh_lo_z = local_mesh.lowCorner( Cajita::Own(), 2 ); + local_mesh_hi_x = local_mesh.highCorner( Cajita::Own(), 0 ); + local_mesh_hi_y = local_mesh.highCorner( Cajita::Own(), 1 ); + local_mesh_hi_z = local_mesh.highCorner( Cajita::Own(), 2 ); + ghost_mesh_lo_x = local_mesh.lowCorner( Cajita::Ghost(), 0 ); + ghost_mesh_lo_y = local_mesh.lowCorner( Cajita::Ghost(), 1 ); + ghost_mesh_lo_z = local_mesh.lowCorner( Cajita::Ghost(), 2 ); + ghost_mesh_hi_x = local_mesh.highCorner( Cajita::Ghost(), 0 ); + ghost_mesh_hi_y = local_mesh.highCorner( Cajita::Ghost(), 1 ); + ghost_mesh_hi_z = local_mesh.highCorner( Cajita::Ghost(), 2 ); + local_mesh_x = local_mesh.extent( Cajita::Own(), 0 ); + local_mesh_y = local_mesh.extent( Cajita::Own(), 1 ); + local_mesh_z = local_mesh.extent( Cajita::Own(), 2 ); + } }; template From c8f7a8d216aa8a54b5c2fc830443988c39047c10 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Fri, 13 Aug 2021 16:19:31 +0200 Subject: [PATCH 31/50] use CabanaMD_ENABLE_ALL to enable load balancing --- CMakeLists.txt | 23 +++++------------------ src/CMakeLists.txt | 3 +++ 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d6fc6d3..1b6352e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,22 +25,6 @@ if( NOT Cabana_ENABLE_CAJITA ) message( FATAL_ERROR "Cabana must be compiled with Cajita" ) endif() -##---------------------------------------------------------------------------## -# Macro for optional dependencies (taken from Cabana) -##---------------------------------------------------------------------------## -macro(CabanaMD_add_dependency) - cmake_parse_arguments(CABANAMD_DEPENDENCY "" "PACKAGE;VERSION" "" ${ARGN}) - find_package( ${CABANAMD_DEPENDENCY_PACKAGE} ${CABANAMD_DEPENDENCY_VERSION} QUIET ) - string(TOUPPER "${CABANAMD_DEPENDENCY_PACKAGE}" CABANAMD_DEPENDENCY_OPTION ) - option( - CabanaMD_REQUIRE_${CABANAMD_DEPENDENCY_OPTION} - "Require CabanaMD to build with ${CABANAMD_DEPENDENCY_PACKAGE} support" ${CABANAMD_DEPENDENCY_PACKAGE}_FOUND) - if(CabanaMD_REQUIRE_${CABANAMD_DEPENDENCY_OPTION}) - find_package( ${CABANAMD_DEPENDENCY_PACKAGE} ${CABANAMD_DEPENDENCY_VERSION} REQUIRED ) - endif() - set(CabanaMD_ENABLE_${CABANAMD_DEPENDENCY_OPTION} ${${CABANAMD_DEPENDENCY_PACKAGE}_FOUND}) -endmacro() - ##---------------------------------------------------------------------------## # Set up optional libraries ##---------------------------------------------------------------------------## @@ -49,8 +33,11 @@ if(NOT DEFINED N2P2_DIR) set(N2P2_DIR ~/n2p2/) endif() -# todo(sschulz): All versions from 0.9.2 onwards should be fine. -CabanaMD_add_dependency( PACKAGE "ALL" VERSION 0.9.2 ) +option(CabanaMD_ENABLE_ALL "Build CabanaMD with ALL load balancing" OFF) +if(CabanaMD_ENABLE_ALL) + # todo(sschulz): All versions from 0.9.2 onwards should be fine. + find_package(ALL 0.9.2 REQUIRED) +endif() ##---------------------------------------------------------------------------## ## Print the Git revision number to stdout diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e362d13..6ecd4ec 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -94,6 +94,9 @@ target_include_directories(CabanaMD PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/system_types $ $) +if(CabanaMD_ENBALE_ALL) + target_link_libraries(CabanaMD ALL::ALL) +endif() #------------------------------------------------------------ From c07ffa0cabf80482de2ff6feb159e7ae7110cdc2 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 26 Aug 2021 13:58:54 +0200 Subject: [PATCH 32/50] update load balancer to use cajita lb --- src/cabanamd.h | 9 +++------ src/cabanamd_impl.h | 27 +++++++++++++++++--------- src/system.h | 46 ++++++++++++++++++++++----------------------- 3 files changed, 44 insertions(+), 38 deletions(-) diff --git a/src/cabanamd.h b/src/cabanamd.h index c4902c6..099405b 100644 --- a/src/cabanamd.h +++ b/src/cabanamd.h @@ -57,9 +57,8 @@ #include #include -#ifdef CabanaMD_ENABLE_ALL -#include -#endif +#include +#include class CabanaMD { @@ -85,9 +84,7 @@ class CbnMD : public CabanaMD Comm *comm; Binning *binning; InputFile *input; -#ifdef CabanaMD_ENABLE_ALL - LoadBalancer *lb; -#endif + Cajita::LoadBalancer> *lb; void init( InputCL cl ) override; void run() override; diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index 1b4635b..fd21bd8 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -63,6 +63,7 @@ #include #include #include +#include #define MAXPATHLEN 1024 @@ -227,9 +228,8 @@ void CbnMD::init( InputCL commandline ) comm->update_force(); } -#ifdef CabanaMD_ENABLE_ALL - lb = new LoadBalancer( system ); -#endif + lb = new Cajita::LoadBalancer>( + MPI_COMM_WORLD, system->global_grid ); // Initial output int step = 0; @@ -280,6 +280,9 @@ void CbnMD::run() PotE pote( comm ); KinE kine( comm ); + std::string vtk_actual_domain_basename( "domain_act" ); + std::string vtk_lb_domain_basename( "domain_lb" ); + double force_time = 0; double comm_time = 0; double neigh_time = 0; @@ -301,12 +304,13 @@ void CbnMD::run() if ( step % input->comm_exchange_rate == 0 && step > 0 ) { -#ifdef CabanaMD_ENABLE_ALL // Update domain decomposition lb_timer.reset(); - lb->balance(); + double work = system->N_local + system->N_ghost; + auto new_global_grid = lb->createBalancedGlobalGrid( + system->global_mesh, *system->partitioner, work ); + system->update_global_grid( new_global_grid ); lb_time += lb_timer.seconds(); -#endif // Exchange atoms across MPI ranks comm_timer.reset(); @@ -389,9 +393,14 @@ void CbnMD::run() " ", T, " ", PE, " ", PE + KE, " ", time ); last_time = time; } -#ifdef CabanaMD_ENABLE_ALL - lb->output( step ); -#endif + double work = system->N_local + system->N_ghost; + std::array vertices; + vertices = lb->getVertices(); + VTKWriter::writeDomain( MPI_COMM_WORLD, step, vertices, work, + vtk_actual_domain_basename ); + vertices = lb->getInternalVertices(); + VTKWriter::writeDomain( MPI_COMM_WORLD, step, vertices, work, + vtk_lb_domain_basename ); } if ( step % input->vtk_rate == 0 ) diff --git a/src/system.h b/src/system.h index 5fb59b6..67b7859 100644 --- a/src/system.h +++ b/src/system.h @@ -97,6 +97,9 @@ class SystemCommon T_X_FLOAT ghost_mesh_lo_x, ghost_mesh_lo_y, ghost_mesh_lo_z; T_X_FLOAT ghost_mesh_hi_x, ghost_mesh_hi_y, ghost_mesh_hi_z; T_X_FLOAT halo_width; + std::shared_ptr partitioner; + std::shared_ptr>> + global_mesh; std::shared_ptr>> local_grid; std::shared_ptr>> @@ -151,10 +154,12 @@ class SystemCommon { halo_width = ghost_cutoff; // Create the MPI partitions. - Cabana::Grid::DimBlockPartitioner<3> partitioner; - ranks_per_dim = partitioner.ranksPerDimension( MPI_COMM_WORLD, {} ); + partitioner = std::make_shared(); + ranks_per_dim = partitioner->ranksPerDimension( MPI_COMM_WORLD, {} ); // todo(sschulz): Generalize, so non cubic dimensions are allowed + // estimate suitable number of grid cells and use that constructor + // instead grid_cell_size = ( high_corner[0] - low_corner[0] ) / grid_num_cells; if ( std::abs( ( high_corner[0] - low_corner[0] ) - ( high_corner[1] - low_corner[1] ) ) > @@ -167,8 +172,8 @@ class SystemCommon throw std::logic_error( "Dimensions must be cubic" ); // Create global mesh of MPI partitions. - auto global_mesh = Cajita::createUniformGlobalMesh( - low_corner, high_corner, grid_cell_size ); + global_mesh = Cajita::createUniformGlobalMesh( low_corner, high_corner, + grid_cell_size ); global_mesh_x = global_mesh->extent( 0 ); global_mesh_y = global_mesh->extent( 1 ); @@ -177,7 +182,7 @@ class SystemCommon // Create the global grid. std::array is_periodic = { true, true, true }; global_grid = Cajita::createGlobalGrid( MPI_COMM_WORLD, global_mesh, - is_periodic, partitioner ); + is_periodic, *partitioner ); for ( int d = 0; d < 3; d++ ) { @@ -185,26 +190,21 @@ class SystemCommon } // Create a local mesh - int halo_width = std::ceil( ghost_cutoff / grid_cell_size ); - local_grid = Cajita::createLocalGrid( global_grid, halo_width ); - // Update local_mesh_* and ghost_mesh_* info - update_mesh_info(); + halo_width = std::ceil( ghost_cutoff / grid_cell_size ); + // Update local_mesh_* and ghost_mesh_* info as well as create + // local_grid. + update_global_grid( global_grid ); } - // low_corner and high_corner are local corners, not global as in - // create_domain! - void update_domain( std::array low_corner, - std::array high_corner ) + + // Update domain info according to new global grid. We assume that the + // number of ranks (per dim) does not change. We also assume that the + // position of this rank in the cartesian grid of ranks does not change. + void update_global_grid( const std::shared_ptr< + Cajita::GlobalGrid>> + &new_global_grid ) { - // Calculate new local grid offset - std::array cell_index_lo, cell_index_hi; - for ( std::size_t d = 0; d < 3; ++d ) - cell_index_lo[d] = std::rint( low_corner[d] / grid_cell_size ); - for ( std::size_t d = 0; d < 3; ++d ) - cell_index_hi[d] = std::rint( high_corner[d] / grid_cell_size ); - std::array num_cell; - for ( std::size_t d = 0; d < 3; ++d ) - num_cell[d] = cell_index_hi[d] - cell_index_lo[d]; - global_grid->setNumCellAndOffset( num_cell, cell_index_lo ); + global_grid = new_global_grid; + local_grid = Cajita::createLocalGrid( global_grid, halo_width ); update_mesh_info(); } From 03e56ea966ed2749b4c7a5b9b586faf7ce618292 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 26 Aug 2021 13:59:01 +0200 Subject: [PATCH 33/50] update CI --- .github/workflows/CI.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5d20167..7c4efea 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -94,11 +94,20 @@ jobs: -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} cmake --build build --parallel 2 cmake --install build + - name: Checkout ALL + run: | + git clone --depth 1 --branch master https://gitlab.jsc.fz-juelich.de/SLMS/loadbalancing ALL + - name: Build ALL + working-directory: ALL + run: | + cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$HOME/ALL + cmake --build build --parallel 2 + cmake --install build - name: Checkout Cabana uses: actions/checkout@v3 with: - repository: ECP-CoPA/Cabana - ref: 0.6.1 + repository: aetx/Cabana + ref: cajita_loadbalancer path: cabana - name: Build Cabana working-directory: cabana @@ -106,7 +115,7 @@ jobs: cmake -B build \ -DCMAKE_INSTALL_PREFIX=$HOME/Cabana \ -DMPIEXEC_MAX_NUMPROCS=2 -DMPIEXEC_PREFLAGS="--oversubscribe" \ - -DCMAKE_PREFIX_PATH="$HOME/kokkos;$HOME/arborx" \ + -DCMAKE_PREFIX_PATH="$HOME/kokkos;$HOME/arborx;$HOME/ALL" \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ -DCabana_DISABLE_CAJITA_DEPRECATION_WARNINGS=ON \ -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} @@ -126,15 +135,6 @@ jobs: mkdir -p $HOME/n2p2/include mkdir -p $HOME/n2p2/lib make libnnpif INTERFACES=CabanaMD PROJECT_INCLUDE=$HOME/n2p2/include PROJECT_LIB=$HOME/n2p2/lib - - name: Checkout ALL - run: | - git clone --depth 1 --branch master https://gitlab.jsc.fz-juelich.de/SLMS/loadbalancing ALL - - name: Build ALL - working-directory: ALL - run: | - cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$HOME/ALL - cmake --build build --parallel 2 - cmake --install build - name: Checkout CabanaMD uses: actions/checkout@v3 - name: Build CabanaMD @@ -144,7 +144,7 @@ jobs: -DMPIEXEC_MAX_NUMPROCS=2 -DMPIEXEC_PREFLAGS="--oversubscribe" \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ -DCMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \ - -DCMAKE_PREFIX_PATH="$HOME/Cabana;$HOME/ALL" \ + -DCMAKE_PREFIX_PATH="$HOME/Cabana" \ -DCabanaMD_ENABLE_TESTING=ON \ -DCabanaMD_ENBALE_ALL=ON \ -DCabanaMD_LAYOUT=${{ matrix.layout }} \ From 7c7700fb16362f1a5b987d25dca92ed8bc97eea4 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 26 Aug 2021 21:20:15 +0200 Subject: [PATCH 34/50] remove old explicit dependency --- .github/workflows/CI.yml | 1 - CMakeLists.txt | 7 ++----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 7c4efea..28c02c0 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -146,7 +146,6 @@ jobs: -DCMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \ -DCMAKE_PREFIX_PATH="$HOME/Cabana" \ -DCabanaMD_ENABLE_TESTING=ON \ - -DCabanaMD_ENBALE_ALL=ON \ -DCabanaMD_LAYOUT=${{ matrix.layout }} \ -DCabanaMD_VECTORLENGTH=${{ matrix.vector }} \ -DCabanaMD_ENABLE_NNP=${{ matrix.nnp }} \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b6352e..2fb6417 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,11 +33,8 @@ if(NOT DEFINED N2P2_DIR) set(N2P2_DIR ~/n2p2/) endif() -option(CabanaMD_ENABLE_ALL "Build CabanaMD with ALL load balancing" OFF) -if(CabanaMD_ENABLE_ALL) - # todo(sschulz): All versions from 0.9.2 onwards should be fine. - find_package(ALL 0.9.2 REQUIRED) -endif() +# todo(sschulz): All versions from 0.9.2 onwards should be fine. +#CabanaMD_add_dependency( PACKAGE "ALL" VERSION 0.9.2 ) ##---------------------------------------------------------------------------## ## Print the Git revision number to stdout From 4475bf654212a5c394fddb987c3267ce6890f273 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Sun, 29 Aug 2021 12:21:08 +0200 Subject: [PATCH 35/50] use CabanaMD_ENABLE_LB to enable load balancing --- CMakeLists.txt | 11 ++++++++--- src/CMakeLists.txt | 5 ----- src/CabanaMD_config.hpp.cmakein | 1 + src/cabanamd.h | 5 +++++ src/cabanamd_impl.h | 6 ++++++ 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2fb6417..ee29491 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,14 @@ if( NOT Cabana_ENABLE_CAJITA ) message( FATAL_ERROR "Cabana must be compiled with Cajita" ) endif() +# todo(sschulz): This should detect lb capability automatically. +option(CabanaMD_ENABLE_LB "Utilize Cabana load balancer" ON) +if(CabanaMD_ENABLE_LB) + if( NOT Cabana_ENABLE_ALL ) + message( FATAL_ERROR "Cabana must be compiled with ALL library" ) + endif() +endif() + ##---------------------------------------------------------------------------## # Set up optional libraries ##---------------------------------------------------------------------------## @@ -33,9 +41,6 @@ if(NOT DEFINED N2P2_DIR) set(N2P2_DIR ~/n2p2/) endif() -# todo(sschulz): All versions from 0.9.2 onwards should be fine. -#CabanaMD_add_dependency( PACKAGE "ALL" VERSION 0.9.2 ) - ##---------------------------------------------------------------------------## ## Print the Git revision number to stdout ##---------------------------------------------------------------------------## diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6ecd4ec..e2cdaf1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -62,11 +62,6 @@ configure_file(CabanaMD_config.hpp.cmakein CabanaMD_config.hpp @ONLY) file(GLOB HEADERS_PUBLIC GLOB *.h force_types/*.h neighbor_types/*.h system_types/*.h ) -if(NOT CabanaMD_ENABLE_ALL) - list(REMOVE_ITEM HEADERS_PUBLIC - "load_balancer.h" - ) -endif() file(GLOB SOURCES *.cpp diff --git a/src/CabanaMD_config.hpp.cmakein b/src/CabanaMD_config.hpp.cmakein index f14e285..a12cec7 100644 --- a/src/CabanaMD_config.hpp.cmakein +++ b/src/CabanaMD_config.hpp.cmakein @@ -5,6 +5,7 @@ #define CabanaMD_GIT_COMMIT_HASH "@CabanaMD_GIT_COMMIT_HASH@" #cmakedefine CabanaMD_ENABLE_NNP +#cmakedefine CabanaMD_ENABLE_LB #cmakedefine CabanaMD_LAYOUT @CabanaMD_LAYOUT@ #cmakedefine CabanaMD_VECTORLENGTH "@CabanaMD_VECTORLENGTH@" diff --git a/src/cabanamd.h b/src/cabanamd.h index 099405b..545572f 100644 --- a/src/cabanamd.h +++ b/src/cabanamd.h @@ -49,6 +49,7 @@ #ifndef CABANAMD_H #define CABANAMD_H + #include #include #include @@ -57,8 +58,10 @@ #include #include +#ifdef CabanaMD_ENABLE_LB #include #include +#endif class CabanaMD { @@ -84,7 +87,9 @@ class CbnMD : public CabanaMD Comm *comm; Binning *binning; InputFile *input; +#ifdef CabanaMD_ENABLE_LB Cajita::LoadBalancer> *lb; +#endif void init( InputCL cl ) override; void run() override; diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index fd21bd8..f3f6a44 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -228,8 +228,10 @@ void CbnMD::init( InputCL commandline ) comm->update_force(); } +#ifdef CabanaMD_ENABLE_LB lb = new Cajita::LoadBalancer>( MPI_COMM_WORLD, system->global_grid ); +#endif // Initial output int step = 0; @@ -306,10 +308,12 @@ void CbnMD::run() { // Update domain decomposition lb_timer.reset(); +#ifdef CabanaMD_ENABLE_LB double work = system->N_local + system->N_ghost; auto new_global_grid = lb->createBalancedGlobalGrid( system->global_mesh, *system->partitioner, work ); system->update_global_grid( new_global_grid ); +#endif lb_time += lb_timer.seconds(); // Exchange atoms across MPI ranks @@ -393,6 +397,7 @@ void CbnMD::run() " ", T, " ", PE, " ", PE + KE, " ", time ); last_time = time; } +#ifdef CabanaMD_ENABLE_LB double work = system->N_local + system->N_ghost; std::array vertices; vertices = lb->getVertices(); @@ -401,6 +406,7 @@ void CbnMD::run() vertices = lb->getInternalVertices(); VTKWriter::writeDomain( MPI_COMM_WORLD, step, vertices, work, vtk_lb_domain_basename ); +#endif } if ( step % input->vtk_rate == 0 ) From 7151057d0448db5e87d529df08e0adc3ed38f38f Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Sun, 29 Aug 2021 12:21:47 +0200 Subject: [PATCH 36/50] force LB on in CI --- .github/workflows/CI.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 28c02c0..f13a387 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -149,6 +149,7 @@ jobs: -DCabanaMD_LAYOUT=${{ matrix.layout }} \ -DCabanaMD_VECTORLENGTH=${{ matrix.vector }} \ -DCabanaMD_ENABLE_NNP=${{ matrix.nnp }} \ + -DCabanaMD_ENABLE_LB=ON \ -DN2P2_DIR=$HOME/n2p2 \ -DCabanaMD_LAYOUT_NNP=${{ matrix.layout_nnp }} \ -DCabanaMD_VECTORLENGTH_NNP=${{ matrix.vector_nnp }} From 8085e2d68893f734a04166eaea5d89d11788e700 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Mon, 30 Aug 2021 16:31:16 +0200 Subject: [PATCH 37/50] use fixed number of subdivisions per rank for global mesh --- src/cabanamd.h | 1 - src/system.h | 33 +++++++++++++-------------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/cabanamd.h b/src/cabanamd.h index 545572f..123562c 100644 --- a/src/cabanamd.h +++ b/src/cabanamd.h @@ -49,7 +49,6 @@ #ifndef CABANAMD_H #define CABANAMD_H - #include #include #include diff --git a/src/system.h b/src/system.h index 67b7859..f1a89b6 100644 --- a/src/system.h +++ b/src/system.h @@ -87,8 +87,6 @@ class SystemCommon // Simulation total domain T_X_FLOAT global_mesh_x, global_mesh_y, global_mesh_z; - T_X_FLOAT grid_cell_size; - int grid_num_cells; // Simulation sub domain (single MPI rank) T_X_FLOAT local_mesh_x, local_mesh_y, local_mesh_z; @@ -123,9 +121,6 @@ class SystemCommon mass = t_mass( "System::mass", ntypes ); - // todo(sschulz): Choose a suitably small grid cell automatically - grid_cell_size = 0.0; - grid_num_cells = 2000; global_mesh_x = global_mesh_y = global_mesh_z = 0.0; local_mesh_lo_x = local_mesh_lo_y = local_mesh_lo_z = 0.0; local_mesh_hi_x = local_mesh_hi_y = local_mesh_hi_z = 0.0; @@ -156,24 +151,19 @@ class SystemCommon // Create the MPI partitions. partitioner = std::make_shared(); ranks_per_dim = partitioner->ranksPerDimension( MPI_COMM_WORLD, {} ); + int cells_per_dim_per_rank = 1; - // todo(sschulz): Generalize, so non cubic dimensions are allowed - // estimate suitable number of grid cells and use that constructor - // instead - grid_cell_size = ( high_corner[0] - low_corner[0] ) / grid_num_cells; - if ( std::abs( ( high_corner[0] - low_corner[0] ) - - ( high_corner[1] - low_corner[1] ) ) > - T_X_FLOAT( 100.0 ) * std::numeric_limits::epsilon() ) - throw std::logic_error( "Dimensions must be cubic" ); - else if ( std::abs( ( high_corner[1] - low_corner[1] ) - - ( high_corner[2] - low_corner[2] ) ) > - T_X_FLOAT( 100.0 ) * - std::numeric_limits::epsilon() ) - throw std::logic_error( "Dimensions must be cubic" ); + // The load balancing will be able to change the local domains with a + // resolution of 1/cells_per_dim_per_rank + cells_per_dim_per_rank = 100; + std::array cells_per_rank = { + cells_per_dim_per_rank * ranks_per_dim[0], + cells_per_dim_per_rank * ranks_per_dim[1], + cells_per_dim_per_rank * ranks_per_dim[2] }; // Create global mesh of MPI partitions. global_mesh = Cajita::createUniformGlobalMesh( low_corner, high_corner, - grid_cell_size ); + cells_per_rank ); global_mesh_x = global_mesh->extent( 0 ); global_mesh_y = global_mesh->extent( 1 ); @@ -190,7 +180,10 @@ class SystemCommon } // Create a local mesh - halo_width = std::ceil( ghost_cutoff / grid_cell_size ); + double minimum_cell_size = std::min( + std::min( global_mesh->cellSize( 0 ), global_mesh->cellSize( 1 ) ), + global_mesh->cellSize( 2 ) ); + halo_width = std::ceil( ghost_cutoff / minimum_cell_size ); // Update local_mesh_* and ghost_mesh_* info as well as create // local_grid. update_global_grid( global_grid ); From 89f539ba61f3a543fc7d616cc92f3447085dee5c Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 2 Sep 2021 18:35:54 +0200 Subject: [PATCH 38/50] use cmake_dependent_option for CabanaMD_ENABLE_LB --- CMakeLists.txt | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ee29491..9c7c296 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,7 @@ endif(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CXX_FLAGS) option(CMAKE_VERBOSE_MAKEFILE "Generate verbose Makefiles" OFF) include(GNUInstallDirs) +include(CMakeDependentOption) set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) @@ -25,13 +26,7 @@ if( NOT Cabana_ENABLE_CAJITA ) message( FATAL_ERROR "Cabana must be compiled with Cajita" ) endif() -# todo(sschulz): This should detect lb capability automatically. -option(CabanaMD_ENABLE_LB "Utilize Cabana load balancer" ON) -if(CabanaMD_ENABLE_LB) - if( NOT Cabana_ENABLE_ALL ) - message( FATAL_ERROR "Cabana must be compiled with ALL library" ) - endif() -endif() +cmake_dependent_option(CabanaMD_ENABLE_LB "Utilize Cabana load balancer" ON Cabana_ENABLE_ALL OFF) ##---------------------------------------------------------------------------## # Set up optional libraries From dd0f07b1767fc94a5253cd8515e54fd400eccbdf Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 23 Sep 2021 13:35:01 +0200 Subject: [PATCH 39/50] output lb quality as well --- src/cabanamd_impl.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index f3f6a44..1e0761c 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -245,10 +245,18 @@ void CbnMD::init( InputCL commandline ) auto KE = kine.compute( system ) / system->N; if ( !_print_lammps ) { +#ifdef CabanaMD_ENABLE_LB + log( out, "\n", std::fixed, std::setprecision( 6 ), + "#Timestep Temperature PotE ETot Time Atomsteps/s LBQuality\n", + step, " ", T, " ", PE, " ", PE + KE, " ", + std::setprecision( 2 ), 0.0, " ", std::scientific, 0.0, + std::setprecision( 2 ), 0.0 ); +#else log( out, "\n", std::fixed, std::setprecision( 6 ), "#Timestep Temperature PotE ETot Time Atomsteps/s\n", step, " ", T, " ", PE, " ", PE + KE, " ", std::setprecision( 2 ), 0.0, " ", std::scientific, 0.0 ); +#endif } else { @@ -385,9 +393,16 @@ void CbnMD::run() double time = timer.seconds(); double rate = 1.0 * system->N * input->thermo_rate / ( time - last_time ); +#ifdef CabanaMD_ENABLE_LB + log( out, std::fixed, std::setprecision( 6 ), step, " ", T, " ", + PE, " ", PE + KE, " ", std::setprecision( 2 ), time, " ", + std::scientific, rate, std::setprecision( 2 ), + lb->getQuality() ); +#else log( out, std::fixed, std::setprecision( 6 ), step, " ", T, " ", PE, " ", PE + KE, " ", std::setprecision( 2 ), time, " ", std::scientific, rate ); +#endif last_time = time; } else From 0de3bde8a1faf759b1bbd7363970f73ad2fde2e0 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 23 Sep 2021 13:42:54 +0200 Subject: [PATCH 40/50] missing space in output --- src/cabanamd_impl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index 1e0761c..eefc0e7 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -249,7 +249,7 @@ void CbnMD::init( InputCL commandline ) log( out, "\n", std::fixed, std::setprecision( 6 ), "#Timestep Temperature PotE ETot Time Atomsteps/s LBQuality\n", step, " ", T, " ", PE, " ", PE + KE, " ", - std::setprecision( 2 ), 0.0, " ", std::scientific, 0.0, + std::setprecision( 2 ), 0.0, " ", std::scientific, 0.0, " ", std::setprecision( 2 ), 0.0 ); #else log( out, "\n", std::fixed, std::setprecision( 6 ), @@ -396,7 +396,7 @@ void CbnMD::run() #ifdef CabanaMD_ENABLE_LB log( out, std::fixed, std::setprecision( 6 ), step, " ", T, " ", PE, " ", PE + KE, " ", std::setprecision( 2 ), time, " ", - std::scientific, rate, std::setprecision( 2 ), + std::scientific, rate, " ", std::setprecision( 2 ), lb->getQuality() ); #else log( out, std::fixed, std::setprecision( 6 ), step, " ", T, " ", From 451439f7fc80dd01975e31a451540666ad678c54 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 7 Oct 2021 09:48:37 +0200 Subject: [PATCH 41/50] rename getQuality->getImbalance --- src/cabanamd_impl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index eefc0e7..d38e97b 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -247,7 +247,7 @@ void CbnMD::init( InputCL commandline ) { #ifdef CabanaMD_ENABLE_LB log( out, "\n", std::fixed, std::setprecision( 6 ), - "#Timestep Temperature PotE ETot Time Atomsteps/s LBQuality\n", + "#Timestep Temperature PotE ETot Time Atomsteps/s LBImbalance\n", step, " ", T, " ", PE, " ", PE + KE, " ", std::setprecision( 2 ), 0.0, " ", std::scientific, 0.0, " ", std::setprecision( 2 ), 0.0 ); @@ -397,7 +397,7 @@ void CbnMD::run() log( out, std::fixed, std::setprecision( 6 ), step, " ", T, " ", PE, " ", PE + KE, " ", std::setprecision( 2 ), time, " ", std::scientific, rate, " ", std::setprecision( 2 ), - lb->getQuality() ); + lb->getImbalance() ); #else log( out, std::fixed, std::setprecision( 6 ), step, " ", T, " ", PE, " ", PE + KE, " ", std::setprecision( 2 ), time, " ", From 71a6478be8744a16c76a022f62ea119d57281ec2 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Thu, 21 Oct 2021 13:15:08 +0200 Subject: [PATCH 42/50] move LoadBalancer into Experimental namespace --- src/cabanamd.h | 2 +- src/cabanamd_impl.h | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cabanamd.h b/src/cabanamd.h index 123562c..75a87b2 100644 --- a/src/cabanamd.h +++ b/src/cabanamd.h @@ -87,7 +87,7 @@ class CbnMD : public CabanaMD Binning *binning; InputFile *input; #ifdef CabanaMD_ENABLE_LB - Cajita::LoadBalancer> *lb; + Cajita::Experimental::LoadBalancer> *lb; #endif void init( InputCL cl ) override; diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index d38e97b..367e5a7 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -229,7 +229,7 @@ void CbnMD::init( InputCL commandline ) } #ifdef CabanaMD_ENABLE_LB - lb = new Cajita::LoadBalancer>( + lb = new Cajita::Experimental::LoadBalancer>( MPI_COMM_WORLD, system->global_grid ); #endif @@ -247,7 +247,8 @@ void CbnMD::init( InputCL commandline ) { #ifdef CabanaMD_ENABLE_LB log( out, "\n", std::fixed, std::setprecision( 6 ), - "#Timestep Temperature PotE ETot Time Atomsteps/s LBImbalance\n", + "#Timestep Temperature PotE ETot Time Atomsteps/s " + "LBImbalance\n", step, " ", T, " ", PE, " ", PE + KE, " ", std::setprecision( 2 ), 0.0, " ", std::scientific, 0.0, " ", std::setprecision( 2 ), 0.0 ); From ed124921bdadfe8bc3466cf16b2011d96a7beb8d Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Fri, 12 Nov 2021 15:29:59 +0100 Subject: [PATCH 43/50] fix vacuum initialisation of particles. now corresponds to bulk system --- src/inputFile_impl.h | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/inputFile_impl.h b/src/inputFile_impl.h index e861469..61266b8 100644 --- a/src/inputFile_impl.h +++ b/src/inputFile_impl.h @@ -550,9 +550,10 @@ void InputFile::create_lattice( Comm *comm ) T_X_FLOAT max_z = lattice_constant * lattice_nz; std::array global_low = { 0.0, 0.0, 0.0 }; std::array global_high = { max_x, max_y, max_z }; - global_high[0] *= 2; - global_high[1] *= 2; - global_high[2] *= 2; + // Uncomment the following to create a vacuum for an unbalanced system. + // global_high[0] *= 2; + // global_high[1] *= 2; + // global_high[2] *= 2; system->create_domain( global_low, global_high, comm_ghost_cutoff ); s = *system; @@ -563,13 +564,6 @@ void InputFile::create_lattice( Comm *comm ) auto local_mesh_hi_y = s.local_mesh_hi_y; auto local_mesh_hi_z = s.local_mesh_hi_z; - // T_INT ix_start = local_mesh_lo_x / s.global_mesh_x * lattice_nx - 0.5; - // T_INT iy_start = local_mesh_lo_y / s.global_mesh_y * lattice_ny - 0.5; - // T_INT iz_start = local_mesh_lo_z / s.global_mesh_z * lattice_nz - 0.5; - // T_INT ix_end = local_mesh_hi_x / s.global_mesh_x * lattice_nx + 0.5; - // T_INT iy_end = local_mesh_hi_y / s.global_mesh_y * lattice_ny + 0.5; - // T_INT iz_end = local_mesh_hi_z / s.global_mesh_z * lattice_nz + 0.5; - // todo(sschulz): This should be checked for missing boundary cells. T_INT ix_start = local_mesh_lo_x / lattice_constant - 0.5; T_INT iy_start = local_mesh_lo_y / lattice_constant - 0.5; T_INT iz_start = local_mesh_lo_z / lattice_constant - 0.5; @@ -611,7 +605,8 @@ void InputFile::create_lattice( Comm *comm ) ( ztmp >= local_mesh_lo_z ) && ( xtmp < local_mesh_hi_x ) && ( ytmp < local_mesh_hi_y ) && - ( ztmp < local_mesh_hi_z ) ) + ( ztmp < local_mesh_hi_z ) && ( xtmp < max_x ) && + ( ytmp < max_y ) && ( ztmp < max_z ) ) { n++; } @@ -644,7 +639,8 @@ void InputFile::create_lattice( Comm *comm ) ( ztmp >= local_mesh_lo_z ) && ( xtmp < local_mesh_hi_x ) && ( ytmp < local_mesh_hi_y ) && - ( ztmp < local_mesh_hi_z ) ) + ( ztmp < local_mesh_hi_z ) && ( xtmp < max_x ) && + ( ytmp < max_y ) && ( ztmp < max_z ) ) { x( n, 0 ) = xtmp; x( n, 1 ) = ytmp; @@ -710,7 +706,8 @@ void InputFile::create_lattice( Comm *comm ) ( ztmp >= local_mesh_lo_z ) && ( xtmp < local_mesh_hi_x ) && ( ytmp < local_mesh_hi_y ) && - ( ztmp < local_mesh_hi_z ) ) + ( ztmp < local_mesh_hi_z ) && ( xtmp < max_x ) && + ( ytmp < max_y ) && ( ztmp < max_z ) ) { n++; } @@ -750,7 +747,8 @@ void InputFile::create_lattice( Comm *comm ) ( ztmp >= local_mesh_lo_z ) && ( xtmp < local_mesh_hi_x ) && ( ytmp < local_mesh_hi_y ) && - ( ztmp < local_mesh_hi_z ) ) + ( ztmp < local_mesh_hi_z ) && ( xtmp < max_x ) && + ( ytmp < max_y ) && ( ztmp < max_z ) ) { h_x( n, 0 ) = xtmp; h_x( n, 1 ) = ytmp; From 45ba7616f8e34fc2d05daf1b06515262c895e239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Wed, 19 Jun 2024 23:18:31 +0200 Subject: [PATCH 44/50] Fix deprecated code and remaining warnings --- CMakeLists.txt | 10 +-------- src/cabanamd.h | 6 +++--- src/cabanamd_impl.h | 2 +- src/system.h | 52 ++++++++++++++++++++++----------------------- 4 files changed, 31 insertions(+), 39 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c7c296..8ad0e9f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,6 @@ endif(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CXX_FLAGS) option(CMAKE_VERBOSE_MAKEFILE "Generate verbose Makefiles" OFF) include(GNUInstallDirs) -include(CMakeDependentOption) set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) @@ -19,14 +18,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) ##---------------------------------------------------------------------------## find_package(Cabana REQUIRED) -if( NOT Cabana_ENABLE_MPI ) - message( FATAL_ERROR "Cabana must be compiled with MPI" ) -endif() -if( NOT Cabana_ENABLE_CAJITA ) - message( FATAL_ERROR "Cabana must be compiled with Cajita" ) -endif() - -cmake_dependent_option(CabanaMD_ENABLE_LB "Utilize Cabana load balancer" ON Cabana_ENABLE_ALL OFF) +option(CabanaMD_ENABLE_LB "Utilize Cabana load balancer" OFF) ##---------------------------------------------------------------------------## # Set up optional libraries diff --git a/src/cabanamd.h b/src/cabanamd.h index 75a87b2..b69ae9a 100644 --- a/src/cabanamd.h +++ b/src/cabanamd.h @@ -58,8 +58,8 @@ #include #ifdef CabanaMD_ENABLE_LB -#include -#include +#include +#include #endif class CabanaMD @@ -87,7 +87,7 @@ class CbnMD : public CabanaMD Binning *binning; InputFile *input; #ifdef CabanaMD_ENABLE_LB - Cajita::Experimental::LoadBalancer> *lb; + Cabana::Grid::Experimental::LoadBalancer> *lb; #endif void init( InputCL cl ) override; diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index 367e5a7..beba2d2 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -229,7 +229,7 @@ void CbnMD::init( InputCL commandline ) } #ifdef CabanaMD_ENABLE_LB - lb = new Cajita::Experimental::LoadBalancer>( + lb = new Cabana::Grid::Experimental::LoadBalancer>( MPI_COMM_WORLD, system->global_grid ); #endif diff --git a/src/system.h b/src/system.h index f1a89b6..f5ab153 100644 --- a/src/system.h +++ b/src/system.h @@ -95,12 +95,12 @@ class SystemCommon T_X_FLOAT ghost_mesh_lo_x, ghost_mesh_lo_y, ghost_mesh_lo_z; T_X_FLOAT ghost_mesh_hi_x, ghost_mesh_hi_y, ghost_mesh_hi_z; T_X_FLOAT halo_width; - std::shared_ptr partitioner; - std::shared_ptr>> + std::shared_ptr> partitioner; + std::shared_ptr>> global_mesh; - std::shared_ptr>> + std::shared_ptr>> local_grid; - std::shared_ptr>> + std::shared_ptr>> global_grid; // Only needed for current comm @@ -149,7 +149,7 @@ class SystemCommon { halo_width = ghost_cutoff; // Create the MPI partitions. - partitioner = std::make_shared(); + partitioner = std::make_shared>(); ranks_per_dim = partitioner->ranksPerDimension( MPI_COMM_WORLD, {} ); int cells_per_dim_per_rank = 1; @@ -162,7 +162,7 @@ class SystemCommon cells_per_dim_per_rank * ranks_per_dim[2] }; // Create global mesh of MPI partitions. - global_mesh = Cajita::createUniformGlobalMesh( low_corner, high_corner, + global_mesh = Cabana::Grid::createUniformGlobalMesh( low_corner, high_corner, cells_per_rank ); global_mesh_x = global_mesh->extent( 0 ); @@ -171,7 +171,7 @@ class SystemCommon // Create the global grid. std::array is_periodic = { true, true, true }; - global_grid = Cajita::createGlobalGrid( MPI_COMM_WORLD, global_mesh, + global_grid = Cabana::Grid::createGlobalGrid( MPI_COMM_WORLD, global_mesh, is_periodic, *partitioner ); for ( int d = 0; d < 3; d++ ) @@ -193,11 +193,11 @@ class SystemCommon // number of ranks (per dim) does not change. We also assume that the // position of this rank in the cartesian grid of ranks does not change. void update_global_grid( const std::shared_ptr< - Cajita::GlobalGrid>> + Cabana::Grid::GlobalGrid>> &new_global_grid ) { global_grid = new_global_grid; - local_grid = Cajita::createLocalGrid( global_grid, halo_width ); + local_grid = Cabana::Grid::createLocalGrid( global_grid, halo_width ); update_mesh_info(); } @@ -247,23 +247,23 @@ class SystemCommon // Update local_mesh_* and ghost_mesh* info from global grid void update_mesh_info() { - auto local_mesh = Cajita::createLocalMesh( *local_grid ); - - local_mesh_lo_x = local_mesh.lowCorner( Cajita::Own(), 0 ); - local_mesh_lo_y = local_mesh.lowCorner( Cajita::Own(), 1 ); - local_mesh_lo_z = local_mesh.lowCorner( Cajita::Own(), 2 ); - local_mesh_hi_x = local_mesh.highCorner( Cajita::Own(), 0 ); - local_mesh_hi_y = local_mesh.highCorner( Cajita::Own(), 1 ); - local_mesh_hi_z = local_mesh.highCorner( Cajita::Own(), 2 ); - ghost_mesh_lo_x = local_mesh.lowCorner( Cajita::Ghost(), 0 ); - ghost_mesh_lo_y = local_mesh.lowCorner( Cajita::Ghost(), 1 ); - ghost_mesh_lo_z = local_mesh.lowCorner( Cajita::Ghost(), 2 ); - ghost_mesh_hi_x = local_mesh.highCorner( Cajita::Ghost(), 0 ); - ghost_mesh_hi_y = local_mesh.highCorner( Cajita::Ghost(), 1 ); - ghost_mesh_hi_z = local_mesh.highCorner( Cajita::Ghost(), 2 ); - local_mesh_x = local_mesh.extent( Cajita::Own(), 0 ); - local_mesh_y = local_mesh.extent( Cajita::Own(), 1 ); - local_mesh_z = local_mesh.extent( Cajita::Own(), 2 ); + auto local_mesh = Cabana::Grid::createLocalMesh( *local_grid ); + + local_mesh_lo_x = local_mesh.lowCorner( Cabana::Grid::Own(), 0 ); + local_mesh_lo_y = local_mesh.lowCorner( Cabana::Grid::Own(), 1 ); + local_mesh_lo_z = local_mesh.lowCorner( Cabana::Grid::Own(), 2 ); + local_mesh_hi_x = local_mesh.highCorner( Cabana::Grid::Own(), 0 ); + local_mesh_hi_y = local_mesh.highCorner( Cabana::Grid::Own(), 1 ); + local_mesh_hi_z = local_mesh.highCorner( Cabana::Grid::Own(), 2 ); + ghost_mesh_lo_x = local_mesh.lowCorner( Cabana::Grid::Ghost(), 0 ); + ghost_mesh_lo_y = local_mesh.lowCorner( Cabana::Grid::Ghost(), 1 ); + ghost_mesh_lo_z = local_mesh.lowCorner( Cabana::Grid::Ghost(), 2 ); + ghost_mesh_hi_x = local_mesh.highCorner( Cabana::Grid::Ghost(), 0 ); + ghost_mesh_hi_y = local_mesh.highCorner( Cabana::Grid::Ghost(), 1 ); + ghost_mesh_hi_z = local_mesh.highCorner( Cabana::Grid::Ghost(), 2 ); + local_mesh_x = local_mesh.extent( Cabana::Grid::Own(), 0 ); + local_mesh_y = local_mesh.extent( Cabana::Grid::Own(), 1 ); + local_mesh_z = local_mesh.extent( Cabana::Grid::Own(), 2 ); } }; From 087baee3cec4f01a11559c8a71c21395bb7843aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Thu, 20 Jun 2024 19:58:52 +0200 Subject: [PATCH 45/50] Apply clang-format --- src/cabanamd.h | 3 ++- src/cabanamd_impl.h | 5 +++-- src/system.h | 27 ++++++++++++++++----------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/cabanamd.h b/src/cabanamd.h index b69ae9a..d5d6222 100644 --- a/src/cabanamd.h +++ b/src/cabanamd.h @@ -87,7 +87,8 @@ class CbnMD : public CabanaMD Binning *binning; InputFile *input; #ifdef CabanaMD_ENABLE_LB - Cabana::Grid::Experimental::LoadBalancer> *lb; + Cabana::Grid::Experimental::LoadBalancer> + *lb; #endif void init( InputCL cl ) override; diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index beba2d2..8b047ab 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -229,8 +229,9 @@ void CbnMD::init( InputCL commandline ) } #ifdef CabanaMD_ENABLE_LB - lb = new Cabana::Grid::Experimental::LoadBalancer>( - MPI_COMM_WORLD, system->global_grid ); + lb = new Cabana::Grid::Experimental::LoadBalancer< + Cabana::Grid::UniformMesh>( MPI_COMM_WORLD, + system->global_grid ); #endif // Initial output diff --git a/src/system.h b/src/system.h index f5ab153..9a81285 100644 --- a/src/system.h +++ b/src/system.h @@ -96,11 +96,14 @@ class SystemCommon T_X_FLOAT ghost_mesh_hi_x, ghost_mesh_hi_y, ghost_mesh_hi_z; T_X_FLOAT halo_width; std::shared_ptr> partitioner; - std::shared_ptr>> + std::shared_ptr< + Cabana::Grid::GlobalMesh>> global_mesh; - std::shared_ptr>> + std::shared_ptr< + Cabana::Grid::LocalGrid>> local_grid; - std::shared_ptr>> + std::shared_ptr< + Cabana::Grid::GlobalGrid>> global_grid; // Only needed for current comm @@ -162,8 +165,8 @@ class SystemCommon cells_per_dim_per_rank * ranks_per_dim[2] }; // Create global mesh of MPI partitions. - global_mesh = Cabana::Grid::createUniformGlobalMesh( low_corner, high_corner, - cells_per_rank ); + global_mesh = Cabana::Grid::createUniformGlobalMesh( + low_corner, high_corner, cells_per_rank ); global_mesh_x = global_mesh->extent( 0 ); global_mesh_y = global_mesh->extent( 1 ); @@ -171,8 +174,8 @@ class SystemCommon // Create the global grid. std::array is_periodic = { true, true, true }; - global_grid = Cabana::Grid::createGlobalGrid( MPI_COMM_WORLD, global_mesh, - is_periodic, *partitioner ); + global_grid = Cabana::Grid::createGlobalGrid( + MPI_COMM_WORLD, global_mesh, is_periodic, *partitioner ); for ( int d = 0; d < 3; d++ ) { @@ -192,9 +195,10 @@ class SystemCommon // Update domain info according to new global grid. We assume that the // number of ranks (per dim) does not change. We also assume that the // position of this rank in the cartesian grid of ranks does not change. - void update_global_grid( const std::shared_ptr< - Cabana::Grid::GlobalGrid>> - &new_global_grid ) + void update_global_grid( + const std::shared_ptr< + Cabana::Grid::GlobalGrid>> + &new_global_grid ) { global_grid = new_global_grid; local_grid = Cabana::Grid::createLocalGrid( global_grid, halo_width ); @@ -247,7 +251,8 @@ class SystemCommon // Update local_mesh_* and ghost_mesh* info from global grid void update_mesh_info() { - auto local_mesh = Cabana::Grid::createLocalMesh( *local_grid ); + auto local_mesh = + Cabana::Grid::createLocalMesh( *local_grid ); local_mesh_lo_x = local_mesh.lowCorner( Cabana::Grid::Own(), 0 ); local_mesh_lo_y = local_mesh.lowCorner( Cabana::Grid::Own(), 1 ); From 8504ea2ee749f39fa30485f5dbde5c9a22f82722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Tue, 25 Jun 2024 14:15:51 +0200 Subject: [PATCH 46/50] Add a CLI flag for creating an unbalanced system --- src/inputCL.cpp | 7 +++++++ src/inputCL.h | 1 + src/inputFile_impl.h | 11 +++++++---- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/inputCL.cpp b/src/inputCL.cpp index b975b29..df96366 100644 --- a/src/inputCL.cpp +++ b/src/inputCL.cpp @@ -116,6 +116,8 @@ void InputCL::read_args( int argc, char *argv[] ) " (N = positive integer)\n", " (PATH = location of ", "directory)\n" ); + log( std::cout, " --sparse : " + "create a vacuum for an unbalanced system\n" ); } // Read Lammps input deck @@ -233,6 +235,11 @@ void InputCL::read_args( int argc, char *argv[] ) i += 3; } + else if ( ( strcmp( argv[i], "--sparse" ) == 0 ) ) + { + sparse = true; + } + else if ( ( strstr( argv[i], "--kokkos-" ) == NULL ) ) { log_err( std::cout, "Unknown command line argument: ", argv[i] ); diff --git a/src/inputCL.h b/src/inputCL.h index c39d3c0..15a2c64 100644 --- a/src/inputCL.h +++ b/src/inputCL.h @@ -63,6 +63,7 @@ class InputCL int layout_type; int nnp_layout_type; int device_type; + bool sparse = false; int dumpbinary_rate, correctness_rate; bool dumpbinaryflag, correctnessflag; diff --git a/src/inputFile_impl.h b/src/inputFile_impl.h index 61266b8..b417260 100644 --- a/src/inputFile_impl.h +++ b/src/inputFile_impl.h @@ -550,10 +550,13 @@ void InputFile::create_lattice( Comm *comm ) T_X_FLOAT max_z = lattice_constant * lattice_nz; std::array global_low = { 0.0, 0.0, 0.0 }; std::array global_high = { max_x, max_y, max_z }; - // Uncomment the following to create a vacuum for an unbalanced system. - // global_high[0] *= 2; - // global_high[1] *= 2; - // global_high[2] *= 2; + if ( commandline.sparse ) + { + // Create a vacuum for an unbalanced system. + global_high[0] *= 2; + global_high[1] *= 2; + global_high[2] *= 2; + } system->create_domain( global_low, global_high, comm_ghost_cutoff ); s = *system; From ed3bcd8c136bd85acaba7e3a1254ae77d8c6bbda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Fri, 9 Aug 2024 22:34:41 +0200 Subject: [PATCH 47/50] Remove unused file --- src/load_balancer.h | 107 -------------------------------------------- 1 file changed, 107 deletions(-) delete mode 100644 src/load_balancer.h diff --git a/src/load_balancer.h b/src/load_balancer.h deleted file mode 100644 index 0596cfa..0000000 --- a/src/load_balancer.h +++ /dev/null @@ -1,107 +0,0 @@ -/**************************************************************************** - * Copyright (c) 2018-2021 by the Cabana authors * - * All rights reserved. * - * * - * This file is part of the Cabana library. Cabana is distributed under a * - * BSD 3-clause license. For the licensing terms see the LICENSE file in * - * the top-level directory. * - * * - * SPDX-License-Identifier: BSD-3-Clause * - ****************************************************************************/ - -#ifndef LOADBALANCER_H -#define LOADBALANCER_H - -#include -#include - -#include - -#include - -#include - -template -class LoadBalancer -{ - public: - LoadBalancer( t_System *system ) - : _comm( MPI_COMM_WORLD ) - , _system( system ) - { - MPI_Comm_rank( _comm, &_rank ); - _liball = - std::make_shared>( ALL::TENSOR, 3, 0 ); - std::vector rank_dim_pos( _system->rank_dim_pos.begin(), - _system->rank_dim_pos.end() ); - std::vector ranks_per_dim( _system->ranks_per_dim.begin(), - _system->ranks_per_dim.end() ); - _liball->setProcGridParams( rank_dim_pos, ranks_per_dim ); - // todo(sschulz): Do we need a minimum domain size? - // std::vector min_size( 3, 15 ); - //_liball->setMinDomainSize( min_size ); - _liball->setCommunicator( _comm ); - _liball->setProcTag( _rank ); - _liball->setup(); - - std::vector> lb_vertices( 2, - ALL::Point( 3 ) ); - lb_vertices.at( 0 )[0] = _system->local_mesh_lo_x; - lb_vertices.at( 0 )[1] = _system->local_mesh_lo_y; - lb_vertices.at( 0 )[2] = _system->local_mesh_lo_z; - lb_vertices.at( 1 )[0] = _system->local_mesh_hi_x; - lb_vertices.at( 1 )[1] = _system->local_mesh_hi_y; - lb_vertices.at( 1 )[2] = _system->local_mesh_hi_z; - _liball->setVertices( lb_vertices ); - } - - void balance() - { - int rank; - MPI_Comm_rank( _comm, &rank ); - double work = ( _system->N_local + _system->N_ghost ); - _liball->setWork( work ); - _liball->balance(); - std::vector> updated_vertices = - _liball->getVertices(); - std::array low_corner = { updated_vertices.at( 0 )[0], - updated_vertices.at( 0 )[1], - updated_vertices.at( 0 )[2] }; - std::array high_corner = { updated_vertices.at( 1 )[0], - updated_vertices.at( 1 )[1], - updated_vertices.at( 1 )[2] }; - _system->update_domain( low_corner, high_corner ); - } - - void output( const int t ) const - { - std::string vtk_actual_domain_basename( "domain_act" ); - std::string vtk_lb_domain_basename( "domain_lb" ); - // _liball->printVTKoutlines( t ); - double work = ( _system->N_local + _system->N_ghost ); - - std::array vertices = { - _system->local_mesh_lo_x, _system->local_mesh_lo_y, - _system->local_mesh_lo_z, _system->local_mesh_hi_x, - _system->local_mesh_hi_y, _system->local_mesh_hi_z }; - VTKWriter::writeDomain( _comm, t, vertices, work, - vtk_actual_domain_basename ); - - std::vector> updated_vertices = - _liball->getVertices(); - for ( std::size_t d = 0; d < 3; ++d ) - vertices[d] = updated_vertices.at( 0 )[d]; - for ( std::size_t d = 3; d < 6; ++d ) - vertices[d] = updated_vertices.at( 1 )[d - 3]; - VTKWriter::writeDomain( _comm, t, vertices, work, - vtk_lb_domain_basename ); - } - - private: - MPI_Comm _comm; - t_System *_system; - std::shared_ptr> _liball; - int _rank; -}; - -#endif From e7eb3da0532f40920a04bbee27875ef1191005e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Tue, 10 Sep 2024 14:27:24 +0200 Subject: [PATCH 48/50] Use upstream Cabana repository --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f13a387..ed6f5fe 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -106,8 +106,8 @@ jobs: - name: Checkout Cabana uses: actions/checkout@v3 with: - repository: aetx/Cabana - ref: cajita_loadbalancer + repository: ECP-CoPA/Cabana + ref: 0.6.1 path: cabana - name: Build Cabana working-directory: cabana From 2980a482567e013de17efa319c1f5598096da7b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Tue, 10 Sep 2024 14:29:33 +0200 Subject: [PATCH 49/50] Revert unnecessary changes --- .github/workflows/CI.yml | 1 - .gitignore | 1 - src/CMakeLists.txt | 3 --- src/inputFile.h | 4 ++-- src/inputFile_impl.h | 38 -------------------------------------- 5 files changed, 2 insertions(+), 45 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ed6f5fe..3d8c352 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -117,7 +117,6 @@ jobs: -DMPIEXEC_MAX_NUMPROCS=2 -DMPIEXEC_PREFLAGS="--oversubscribe" \ -DCMAKE_PREFIX_PATH="$HOME/kokkos;$HOME/arborx;$HOME/ALL" \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ - -DCabana_DISABLE_CAJITA_DEPRECATION_WARNINGS=ON \ -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} cmake --build build --parallel 2 cmake --install build diff --git a/.gitignore b/.gitignore index c429bde..40e1f1a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,3 @@ build .cproject .project -*.swp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e2cdaf1..4805039 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,9 +89,6 @@ target_include_directories(CabanaMD PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/system_types $ $) -if(CabanaMD_ENBALE_ALL) - target_link_libraries(CabanaMD ALL::ALL) -endif() #------------------------------------------------------------ diff --git a/src/inputFile.h b/src/inputFile.h index 25b5d79..d70cd54 100644 --- a/src/inputFile.h +++ b/src/inputFile.h @@ -179,8 +179,8 @@ class InputFile int binning_type = BINNING_LINKEDCELL; - int comm_type; - int comm_exchange_rate; + int comm_type = COMM_MPI; + int comm_exchange_rate = 20; double comm_ghost_cutoff; int force_type = FORCE_LJ; diff --git a/src/inputFile_impl.h b/src/inputFile_impl.h index b417260..9213a71 100644 --- a/src/inputFile_impl.h +++ b/src/inputFile_impl.h @@ -79,46 +79,8 @@ InputFile::InputFile( InputCL commandline_, t_System *system_ ) output_file = commandline.output_file; error_file = commandline.error_file; - // set defaults (matches ExaMiniMD LJ example) - - nsteps = 0; - - thermo_rate = 0; - dumpbinary_rate = 0; - correctness_rate = 0; - vtk_rate = 0; - dumpbinaryflag = false; - correctnessflag = false; - timestepflag = false; - - lattice_offset_x = 0.0; - lattice_offset_y = 0.0; - lattice_offset_z = 0.0; - box[0] = 0; - box[2] = 0; - box[4] = 0; - box[1] = 40; - box[3] = 40; - box[5] = 40; - - units_style = UNITS_LJ; - lattice_style = LATTICE_FCC; - lattice_constant = 0.8442; - - temperature_target = 1.4; - temperature_seed = 87287; - - nsteps = 100; - thermo_rate = 10; - - neighbor_skin = 0.3; - neighbor_skin = 0.0; // for metal and real units - max_neigh_guess = 50; - comm_exchange_rate = 20; comm_ghost_cutoff = std::pow( ( 4.0 / lattice_constant ), ( 1.0 / 3.0 ) ) * 20.0; // 20 lattice constants - - force_cutoff = 2.5; } template From 71929413ce5e4270b9d1ab5099c016e43dbfde44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Wed, 26 Jun 2024 22:56:59 +0200 Subject: [PATCH 50/50] Change file naming scheme Place rank before timestep and add leading zeroes in the file names. This helps ParaView deduce the order correctly. --- input/in.lj | 2 +- src/vtk_writer.h | 45 ++++++++++++++++++++++++++++----------------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/input/in.lj b/input/in.lj index 6d804d1..20198eb 100644 --- a/input/in.lj +++ b/input/in.lj @@ -21,6 +21,6 @@ comm_modify cutoff * 20 fix 1 all nve thermo 10 -dump dmpvtk all vtk 10 dump_*%.vtu +dump dmpvtk all vtk 10 dump%_*.vtu run 100 diff --git a/src/vtk_writer.h b/src/vtk_writer.h index 7335975..88d54a6 100644 --- a/src/vtk_writer.h +++ b/src/vtk_writer.h @@ -13,13 +13,26 @@ #define VTK_DOMAIN_WRITER_H #include +#include #include #include +namespace +{ + +std::string set_width( const int value, const unsigned width = 3 ) +{ + std::ostringstream oss; + oss << std::setw( width ) << std::setfill( '0' ) << value; + return oss.str(); +} + +} // namespace + namespace VTKWriter { // Write PVTU -void writeDomainParallelFile( MPI_Comm comm, int time_step, +void writeDomainParallelFile( MPI_Comm comm, const std::string &time_step, std::string &basename ) { // Should only be called from a single rank @@ -41,8 +54,8 @@ void writeDomainParallelFile( MPI_Comm comm, int time_step, "NumberOfComponents=\"3\"/>\n" ); fprintf( file, "\t\n" ); for ( int i = 0; i < size; ++i ) - fprintf( file, "\t\n", basename.c_str(), - time_step, i ); + fprintf( file, "\t\n", basename.c_str(), + i, time_step.c_str() ); fprintf( file, "\n" ); fprintf( file, "\n" ); fclose( file ); @@ -57,11 +70,13 @@ void writeDomain( MPI_Comm comm, int time_step, { int rank; MPI_Comm_rank( comm, &rank ); + + auto time_step_fixed = set_width( time_step ); if ( rank == 1 ) - writeDomainParallelFile( comm, time_step, basename ); + writeDomainParallelFile( comm, time_step_fixed, basename ); std::stringstream filename; // todo(sschulz): properly format, according to max rank - filename << basename << "_" << time_step << "_" << rank << ".vtu"; + filename << basename << "_" << rank << "_" << time_step_fixed << ".vtu"; FILE *file = fopen( filename.str().c_str(), "w" ); fprintf( file, "\n" ); fprintf( file, " -void writeParticles( MPI_Comm comm, const int step, t_System *system, +void writeParticles( MPI_Comm comm, const int time_step, t_System *system, std::string filename, std::ofstream &err ) { int rank; MPI_Comm_rank( comm, &rank ); + auto time_step_fixed = set_width( time_step ); // Write parallel file if ( rank == 1 ) - writeParticlesParallelFile( comm, step, filename ); + writeParticlesParallelFile( comm, time_step_fixed, filename ); // Prepare actual filename // todo(sschulz): Separate filename construction into function size_t pos = 0; pos = filename.find( "*", pos ); if ( std::string::npos == pos ) log_err( err, "VTK output file does not contain required '*'" ); - std::stringstream time_string; - time_string << step; - filename.replace( pos, 1, time_string.str() ); + filename.replace( pos, 1, time_step_fixed ); pos = 0; pos = filename.find( "%", pos ); if ( std::string::npos == pos )