From dad6be655b5c7d541d9b348dc68a3be572a8a72a Mon Sep 17 00:00:00 2001 From: Waqar Butt Date: Mon, 1 Jul 2024 11:57:47 +0100 Subject: [PATCH] fixed tests and cleaned up aegis main function --- .vscode/launch.json | 25 ++++++++++ CMakeLists.txt | 9 +--- inres1/aegis_settings.json | 2 +- src/aegis/aegis.cpp | 51 ++++++++++++------- src/aegis_lib/Integrator.cpp | 34 ++++++++----- src/aegis_lib/Integrator.h | 3 +- src/aegis_lib/ParticleSimulation.cpp | 41 +++++++--------- src/aegis_lib/ParticleSimulation.h | 9 ++-- test/CMakeLists.txt | 1 + test/data/aegis_settings.json | 71 +++++++++++++++------------ test/regression/inres1_regression.cpp | 5 +- test/unit/CMakeLists.txt | 2 + test/unit/aegis_unit.cpp | 8 +-- 13 files changed, 163 insertions(+), 98 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 0dcdefd..3f04732 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -145,5 +145,30 @@ ] }, + { + "name": "AEGIS-regression-test-dbg", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceRoot}/bin/regression_tests", + "args": ["aegis_settings.json"], + "stopAtEntry": false, + "cwd": "${workspaceRoot}/test/data", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true + } + ] + }, + ], } \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a38aab..731ca67 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,13 +28,8 @@ include_directories(externals/json/include) #include(${CMAKE_SOURCE_DIR}/cmake/SetBuildType.cmake) # Default to cpmpile with debugging symbols -set(CMAKE_BUILD_TYPE RELEASE) -# set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -fsanitize=address") -# set (CMAKE_LINKER_FLAGS_DEBUG "${CMAKE_LINKER_FLAGS_DEBUG} -fno-omit-frame-pointer -fsanitize=address") -# set(BUILD_TYPE "PROFILING") -# set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pg") -# set (CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_LINKER_FLAGS_DEBUG} -pg") -#set(CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE}) +set(CMAKE_BUILD_TYPE DEBUG) + # Import VTK find_package(VTK COMPONENTS REQUIRED diff --git a/inres1/aegis_settings.json b/inres1/aegis_settings.json index 34b08d6..e283854 100644 --- a/inres1/aegis_settings.json +++ b/inres1/aegis_settings.json @@ -1,7 +1,7 @@ { "description": "inres1 test case to compare against SMARDDA", "aegis_params":{ - "DAGMC": "fine_inres1.h5m", + "DAGMC": "inres1_shad.h5m", "step_size": 0.01, "max_steps": 10000, "launch_pos": "fixed", diff --git a/src/aegis/aegis.cpp b/src/aegis/aegis.cpp index 6dd2027..6a890a9 100644 --- a/src/aegis/aegis.cpp +++ b/src/aegis/aegis.cpp @@ -2,21 +2,24 @@ #include #include #include +#include #include #include +#include "Integrator.h" int main(int argc, char ** argv) { - + // Setup MPI int rank, nprocs; MPI_Init(&argc, &argv); - double startTime = MPI_Wtime(); // get start time on across all processes - MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + double startTime = MPI_Wtime(); // get start time across all processes + // --------------------------------------------------------------------------------- + // Get Config file std::string configFileName; if (argc > 1) { @@ -32,9 +35,10 @@ main(int argc, char ** argv) } configFileName = "aegis_settings.json"; } + // --------------------------------------------------------------------------------- + // equilibrium setup auto configFile = std::make_shared(configFileName); - double equilibriumInstantiationStart = MPI_Wtime(); auto equilibrium = std::make_shared(configFile); equilibrium->move(); @@ -42,23 +46,33 @@ main(int argc, char ** argv) equilibrium->init_interp_splines(); equilibrium->centre(1); double equilibriumInstantionTime = MPI_Wtime(); + // --------------------------------------------------------------------------------- - ParticleSimulation simulation(configFile, equilibrium); + // Integrator setup + auto integrator = std::make_shared(); + // --------------------------------------------------------------------------------- + + // Main Particle simulation + ParticleSimulation simulation(configFile, equilibrium, integrator); simulation.Execute(); + // --------------------------------------------------------------------------------- - // print wall times for each process - // for (int i = 1; i < nprocs; ++i) - // { - // if (rank == i) - // { - // double endTime = MPI_Wtime(); - // double totalTime = endTime - startTime; - // std::cout << "Elapsed wall Time on process " << i << " = " << totalTime << std::endl; - // std::cout << "----------------------------" << std::endl << std::endl; - // } - // } + // Print individual process stats + for (int i = 1; i < nprocs; ++i) + { + if (rank == i) + { + std::cout << "\nProcess " << i << " handled particles: \n"; + integrator->print_particle_stats(); + double endTime = MPI_Wtime(); + double totalTime = endTime - startTime; + std::cout << "Elapsed wall Time on process " << i << " = " << totalTime << "\n"; + } + } + std::cout << "\n"; + // --------------------------------------------------------------------------------- - MPI_Finalize(); + // Print wall time and other profiling times if (rank == 0) { std::map profilingTimes = simulation.get_profiling_times(); @@ -72,5 +86,8 @@ main(int argc, char ** argv) std::cout << "Total wall time = " << MPI_Wtime() - startTime << "\n"; std::cout << "------------------------------------" << std::endl; } + // --------------------------------------------------------------------------------- + + MPI_Finalize(); return 0; } diff --git a/src/aegis_lib/Integrator.cpp b/src/aegis_lib/Integrator.cpp index 437b99b..9daf24a 100644 --- a/src/aegis_lib/Integrator.cpp +++ b/src/aegis_lib/Integrator.cpp @@ -6,6 +6,8 @@ #include #include "SimpleLogger.h" +SurfaceIntegrator::SurfaceIntegrator() {} + // surface integrator class constructor // initialise list of EntityHandles and maps associated with SurfaceIntegrator::SurfaceIntegrator(moab::Range const & Facets) @@ -21,6 +23,18 @@ SurfaceIntegrator::SurfaceIntegrator(moab::Range const & Facets) } } +void +SurfaceIntegrator::set_facets(moab::Range const & Facets) +{ + nFacets = Facets.size(); + for (const auto i : Facets) + { + facetEntities.push_back(i); + particlesShadowedBy[i] = 0; + powFac[i] = 0; + } +} + // Overload for constructor use with STL vector SurfaceIntegrator::SurfaceIntegrator(std::vector const & Facets) { @@ -238,19 +252,17 @@ SurfaceIntegrator::piecewise_multilinear_out( } } +// return number of particles depositing, shadowed, lost etc. void SurfaceIntegrator::print_particle_stats() -{ // return number of particles depositing, shadowed, lost etc. - - if (rank == 0) - { - LOG_WARNING << "Number of particles launched = " << nFacets; - LOG_WARNING << "Number of particles depositing power from omp = " << nParticlesHeatDep; - LOG_WARNING << "Number of shadowed particle intersections = " << nParticlesShadowed; - LOG_WARNING << "Number of particles lost from magnetic domain = " << nParticlesLost; - LOG_WARNING << "Number of particles terminated upon reaching max tracking length = " - << nParticlesMaxLength; - } +{ + std::cout << "DEPOSITING - " << nParticlesHeatDep << "\n"; + std::cout << "SHADOWED - " << nParticlesShadowed << "\n"; + std::cout << "LOST - " << nParticlesLost << "\n"; + std::cout << "MAX LENGTH - " << nParticlesTotal << "\n"; + int totalParticlesHandled = + nParticlesHeatDep + nParticlesShadowed + nParticlesLost + nParticlesMaxLength; + std::cout << "TOTAL - " << totalParticlesHandled << "\n"; }; void diff --git a/src/aegis_lib/Integrator.h b/src/aegis_lib/Integrator.h index 29d9fdd..63b62a0 100644 --- a/src/aegis_lib/Integrator.h +++ b/src/aegis_lib/Integrator.h @@ -59,12 +59,13 @@ PADDED // padded null particle class SurfaceIntegrator : public AegisBase { public: + SurfaceIntegrator(); SurfaceIntegrator(moab::Range const &Facets); // constructor (with moab::Range) SurfaceIntegrator(std::vector const &Facets); // constructor (with std::vector) // void q_values(); // return list of qvalues for each triangle // void psi_values(); // return list of psi values for each triangle - + void set_facets(moab::Range const &Facets); void count_hit(EntityHandle facet_hit); // count hits belonging to each facet void count_lost_ray(); void store_heat_flux(EntityHandle facet, double heatflux); // store the power associated with a particular facet diff --git a/src/aegis_lib/ParticleSimulation.cpp b/src/aegis_lib/ParticleSimulation.cpp index eb24300..5370ef0 100644 --- a/src/aegis_lib/ParticleSimulation.cpp +++ b/src/aegis_lib/ParticleSimulation.cpp @@ -3,16 +3,17 @@ // setup AEGIS simulation ParticleSimulation::ParticleSimulation(std::shared_ptr configFile, - std::shared_ptr equil) + std::shared_ptr equilibirum, + std::shared_ptr integrator) { set_mpi_params(); - read_params(configFile); - equilibrium = equil; + equilibrium = equilibirum; DAG = std::make_unique(); vtkInterface = std::make_unique(configFile); - init_geometry(); + _integrator = integrator; + _integrator->set_facets(targetFacets); } // Call different execute functions depending on which execute type selected @@ -110,7 +111,7 @@ ParticleSimulation::Execute_dynamic_mpi() worker(); } - std::array particleStats = integrator->particle_stats(); + std::array particleStats = _integrator->particle_stats(); std::array totalParticleStats; if (rank != 0) @@ -119,7 +120,7 @@ ParticleSimulation::Execute_dynamic_mpi() } else { - totalParticleStats = integrator->particle_stats(); + totalParticleStats = _integrator->particle_stats(); } for (int i = 1; i < nprocs; ++i) @@ -151,8 +152,6 @@ ParticleSimulation::Execute_dynamic_mpi() aegisMeshWriteTime = MPI_Wtime() - aegisMeshWriteStart; } - mpi_particle_stats(); - print_particle_stats(totalParticleStats); } @@ -479,7 +478,6 @@ ParticleSimulation::init_geometry() } targetFacets = select_target_surface(); - integrator = std::make_unique(targetFacets); prepSurfacesTime = MPI_Wtime() - prepSurfacesStart; double setupArrayOfParticlesTimeStart = MPI_Wtime(); @@ -499,7 +497,7 @@ ParticleSimulation::loop_over_particles(int startIndex, int endIndex) for (int i = startIndex; i < endIndex; ++i) { auto particle = listOfParticles[i]; - integrator->set_launch_position(particle.parent_entity_handle(), particle.get_pos()); + _integrator->set_launch_position(particle.parent_entity_handle(), particle.get_pos()); terminationState particleState = cartesian_particle_track(particle); double heatflux = (particleState == terminationState::DEPOSITING) ? particle.heatflux() : 0.0; heatfluxVals.emplace_back(heatflux); @@ -524,7 +522,6 @@ ParticleSimulation::setup_sources() listOfParticles.reserve(num_particles_launched()); for (const auto & facetEH : targetFacets) { - heatfluxMap.insert(std::pair(facetEH, 0.0)); std::vector triangleNodes; DAG->moab_instance()->get_adjacencies(&facetEH, 1, 0, false, triangleNodes); DAG->moab_instance()->get_coords(&triangleNodes[0], triangleNodes.size(), @@ -622,8 +619,8 @@ ParticleSimulation::terminate_particle_depositing(ParticleBase & particle) std::stringstream terminationLogString; double heatflux = particle.heatflux(); vtkInterface->write_particle_track(branchDepositingPart, heatflux); - integrator->count_particle(particle.parent_entity_handle(), terminationState::DEPOSITING, - heatflux); + _integrator->count_particle(particle.parent_entity_handle(), terminationState::DEPOSITING, + heatflux); terminationLogString << "Midplane reached. Depositing power after travelling " << trackLength << " units"; log_string(LogLevel::INFO, terminationLogString.str()); @@ -636,7 +633,8 @@ ParticleSimulation::terminate_particle_shadow(ParticleBase & particle) std::stringstream terminationLogString; double heatflux = 0.0; vtkInterface->write_particle_track(branchShadowedPart, heatflux); - integrator->count_particle(particle.parent_entity_handle(), terminationState::SHADOWED, heatflux); + _integrator->count_particle(particle.parent_entity_handle(), terminationState::SHADOWED, + heatflux); terminationLogString << "Surface " << nextSurf << " hit after travelling " << trackLength << " units"; log_string(LogLevel::INFO, terminationLogString.str()); @@ -649,7 +647,7 @@ ParticleSimulation::terminate_particle_lost(ParticleBase & particle) std::stringstream terminationLogString; double heatflux = 0.0; vtkInterface->write_particle_track(branchLostPart, heatflux); - integrator->count_particle(particle.parent_entity_handle(), terminationState::LOST, heatflux); + _integrator->count_particle(particle.parent_entity_handle(), terminationState::LOST, heatflux); terminationLogString << "Particle leaving magnetic field after travelling " << trackLength << " units"; log_string(LogLevel::INFO, terminationLogString.str()); @@ -662,8 +660,8 @@ ParticleSimulation::terminate_particle_maxlength(ParticleBase & particle) std::stringstream terminationLogString; double heatflux = 0.0; vtkInterface->write_particle_track(branchMaxLengthPart, heatflux); - integrator->count_particle(particle.parent_entity_handle(), terminationState::MAXLENGTH, - heatflux); + _integrator->count_particle(particle.parent_entity_handle(), terminationState::MAXLENGTH, + heatflux); terminationLogString << "Fieldline trace reached maximum length before intersection"; } @@ -793,7 +791,7 @@ ParticleSimulation::mpi_particle_stats() { std::cout << std::endl << "process " << i << " has the following particle stats:" << std::endl; - std::array localRankParticleStats = integrator->particle_stats(); + std::array localRankParticleStats = _integrator->particle_stats(); std::cout << "DEPOSITING - " << localRankParticleStats[0] << std::endl; std::cout << "SHADOWED - " << localRankParticleStats[1] << std::endl; @@ -862,7 +860,7 @@ ParticleSimulation::Execute_serial() // write out data and print final - std::array particleStats = integrator->particle_stats(); + std::array particleStats = _integrator->particle_stats(); vtkInterface->write_multiBlockData("particle_tracks.vtm"); @@ -918,7 +916,7 @@ ParticleSimulation::Execute_mpi() qvalues = loop_over_particles(startIndex, endIndex); // perform main loop double endTime = MPI_Wtime(); - std::array particleStats = integrator->particle_stats(); + std::array particleStats = _integrator->particle_stats(); std::array totalParticleStats; if (rank != 0) { @@ -930,7 +928,7 @@ ParticleSimulation::Execute_mpi() } else { - totalParticleStats = integrator->particle_stats(); + totalParticleStats = _integrator->particle_stats(); MPI_Gatherv(qvalues.data(), qvalues.size(), MPI_DOUBLE, rootQvalues.data(), recieveCounts.data(), displacements.data(), MPI_DOUBLE, rootRank, MPI_COMM_WORLD); // MPI_Gather(qvalues.data(), qvalues.size(), MPI_DOUBLE, @@ -952,7 +950,6 @@ ParticleSimulation::Execute_mpi() mainParticleLoopTime = MPI_Wtime() - mainParticleLoopStart; } - mpi_particle_stats(); print_particle_stats(totalParticleStats); } diff --git a/src/aegis_lib/ParticleSimulation.h b/src/aegis_lib/ParticleSimulation.h index 38d0db3..89a4c9f 100644 --- a/src/aegis_lib/ParticleSimulation.h +++ b/src/aegis_lib/ParticleSimulation.h @@ -78,7 +78,11 @@ enum class ExecuteOptions class ParticleSimulation : public AegisBase { public: - ParticleSimulation(std::shared_ptr configFile, std::shared_ptr equil); + ParticleSimulation(std::shared_ptr configFile, std::shared_ptr equilibrium, std::shared_ptr integrator); + + + + void Execute(); // switch between runs void Execute_serial(); // serial void Execute_mpi(); // MPI_Gatherv @@ -148,7 +152,6 @@ class ParticleSimulation : public AegisBase bool writeParticleLaunchPos = false; std::vector listOfTriangles; - std::unordered_map heatfluxMap; int totalNumberOfFacets = 0; std::vector vectorOfTargetSurfs; @@ -183,7 +186,7 @@ class ParticleSimulation : public AegisBase std::shared_ptr equilibrium; - std::unique_ptr integrator; + std::shared_ptr _integrator; std::unique_ptr vtkInterface; const std::string branchShadowedPart = "Shadowed Particles"; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 025284d..bfd3409 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -7,6 +7,7 @@ set(Boost_PACKAGES system log log_setup) set(BOOST_REQUESTED_VERSION 1.71) find_package(Boost REQUIRED COMPONENTS ${Boost_PACKAGES}) +find_package(MPI REQUIRED) include(ExternalProject) diff --git a/test/data/aegis_settings.json b/test/data/aegis_settings.json index 62dd57b..e283854 100644 --- a/test/data/aegis_settings.json +++ b/test/data/aegis_settings.json @@ -1,32 +1,41 @@ { - "description": "inres1 test case to compare against SMARDDA", - "aegis_params":{ - "DAGMC": "inres1_shad.h5m", - "step_size": 0.01, - "max_steps": 100000, - "launch_pos": "fixed", - "target_surfs": [1,2,3,4,5,6], - "force_no_deposition": false, - "coordinate_system": "cart", - "execution_type": "dynamic", - "task_farm_params":{ - "dynamic_task_size": 16, - "worker_profiling_enabled": false - } - }, - "equil_params":{ - "eqdsk": "EQ3.eqdsk", - "cenopt": 2, - "P_sol": 7.5e+06, - "lambda_q": 0.012, - "psiref": -2.1628794, - "r_outrbdry": 8.07385, - "rmove": -0.006, - "draw_equil_rz": false, - "draw_equil_xyz": false, - "print_debug_info": false - }, - "vtk_params":{ - "draw_particle_tracks": false - } - } + "description": "inres1 test case to compare against SMARDDA", + "aegis_params":{ + "DAGMC": "inres1_shad.h5m", + "step_size": 0.01, + "max_steps": 10000, + "launch_pos": "fixed", + + "monte_carlo_params":{ + "number_of_particles_per_facet":1, + "write_particle_launch_positions":false + }, + + "target_surfs": [1,2,3,4,5,6], + "force_no_deposition": false, + "coordinate_system": "flux", + "execution_type": "dynamic", + + "dynamic_batch_params":{ + "batch_size":16, + "worker_profiling": false, + "debug":false + } + }, + "equil_params":{ + "eqdsk": "EQ3.eqdsk", + "power_sol": 7.5e+06, + "lambda_q": 0.012, + "r_outrbdry": 8.07385, + "cenopt": 2, + "psiref": -2.1628794, + "rmove": -0.006, + "draw_equil_rz": false, + "draw_equil_xyz": false, + "print_debug_info": false + }, + "vtk_params":{ + "draw_particle_tracks": false + } + } + diff --git a/test/regression/inres1_regression.cpp b/test/regression/inres1_regression.cpp index f1a4f25..a098653 100644 --- a/test/regression/inres1_regression.cpp +++ b/test/regression/inres1_regression.cpp @@ -82,13 +82,14 @@ double dot_product(std::vector vector_a, std::vector vector_b); auto configFile = std::make_shared(configFilename); auto equilibrium = std::make_shared(configFile); - equilibrium->move(); + equilibrium->move(); equilibrium->psiref_override(); equilibrium->init_interp_splines(); equilibrium->centre(1); equilibrium->write_bfield(); - ParticleSimulation aegis(configFile, equilibrium); + std::shared_ptr integrator; + ParticleSimulation aegis(configFile, equilibrium, integrator); if (std::filesystem::exists(configFilename)){ aegis.Execute_serial(); diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index f9ce5d2..078aaee 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -8,11 +8,13 @@ target_include_directories(unit_tests PUBLIC ${PROJECT_SOURCE_DIR}/src/${PROJECT target_include_directories(unit_tests PUBLIC ${DAGMC_INCLUDE_DIRS}) target_include_directories(unit_tests PUBLIC ${VTK_LIBRARIES}) + target_link_libraries(unit_tests ${Boost_LIBRARIES}) target_link_libraries(unit_tests ${GTEST_LIBRARY} ${GTEST_MAIN_LIBRARY} pthread) target_link_libraries(unit_tests ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/lib${PROJECT_NAME}.so) target_link_libraries(unit_tests ${DAGMC_LIBRARY} dagmc-shared uwuw-shared ) target_link_libraries(unit_tests ${VTK_LIBRARIES}) +target_link_libraries(unit_tests MPI::MPI_CXX) include(GoogleTest) gtest_discover_tests(unit_tests WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}/test/data") diff --git a/test/unit/aegis_unit.cpp b/test/unit/aegis_unit.cpp index 5d494d4..7965e02 100644 --- a/test/unit/aegis_unit.cpp +++ b/test/unit/aegis_unit.cpp @@ -12,6 +12,7 @@ #include "Source.h" #include "CoordTransform.h" #include "SimpleLogger.h" +#include "mpi.h" using namespace moab; @@ -289,7 +290,7 @@ TEST_F(aegisUnitTest, SMARDDA_comparison_test) { TEST_F(aegisUnitTest, eqdsk_read) { - + MPI_Init(NULL, NULL); auto equilibrium = std::make_shared(); equilibrium->read_eqdsk("test.eqdsk"); eqdskData eqdsk = equilibrium->get_eqdsk_struct(); @@ -332,6 +333,7 @@ TEST_F(aegisUnitTest, eqdsk_read) { EXPECT_FLOAT_EQ(eqdsk.zbdry[39], 0.986207476); EXPECT_FLOAT_EQ(eqdsk.rlim[27], 557.200115); EXPECT_FLOAT_EQ(eqdsk.zlim[1], -149.995359); + MPI_Finalize(); } @@ -362,7 +364,7 @@ TEST_F(aegisUnitTest, polar_to_cart_transform){ } TEST_F(aegisUnitTest, polar_to_flux_transform){ - + MPI_Init(NULL, NULL); auto equilibrium = std::make_shared(); equilibrium->read_eqdsk("test.eqdsk"); equilibrium->init_interp_splines(); @@ -378,7 +380,7 @@ TEST_F(aegisUnitTest, polar_to_flux_transform){ EXPECT_FLOAT_EQ(output[0], -2.5001357); EXPECT_FLOAT_EQ(output[1], 2.1932724); EXPECT_FLOAT_EQ(output[2], -2); - + MPI_Finalize(); } double * vecNorm(double vector[3]){