Skip to content

Commit

Permalink
fixed tests and cleaned up aegis main function
Browse files Browse the repository at this point in the history
  • Loading branch information
@Waqar-ukaea
Waqar-ukaea committed Jul 1, 2024
1 parent f2ff296 commit dad6be6
Show file tree
Hide file tree
Showing 13 changed files with 163 additions and 98 deletions.
25 changes: 25 additions & 0 deletions .vscode/launch.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,5 +145,30 @@
]
},

{
"name": "AEGIS-regression-test-dbg",
"type": "cppdbg",
"request": "launch",
"program": "${workspaceRoot}/bin/regression_tests",
"args": ["aegis_settings.json"],
"stopAtEntry": false,
"cwd": "${workspaceRoot}/test/data",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
"setupCommands": [
{
"description": "Enable pretty-printing for gdb",
"text": "-enable-pretty-printing",
"ignoreFailures": true
},
{
"description": "Set Disassembly Flavor to Intel",
"text": "-gdb-set disassembly-flavor intel",
"ignoreFailures": true
}
]
},

],
}
9 changes: 2 additions & 7 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,8 @@ include_directories(externals/json/include)
#include(${CMAKE_SOURCE_DIR}/cmake/SetBuildType.cmake)

# Default to cpmpile with debugging symbols
set(CMAKE_BUILD_TYPE RELEASE)
# set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -fsanitize=address")
# set (CMAKE_LINKER_FLAGS_DEBUG "${CMAKE_LINKER_FLAGS_DEBUG} -fno-omit-frame-pointer -fsanitize=address")
# set(BUILD_TYPE "PROFILING")
# set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pg")
# set (CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_LINKER_FLAGS_DEBUG} -pg")
#set(CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE})
set(CMAKE_BUILD_TYPE DEBUG)


# Import VTK
find_package(VTK COMPONENTS REQUIRED
Expand Down
2 changes: 1 addition & 1 deletion inres1/aegis_settings.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"description": "inres1 test case to compare against SMARDDA",
"aegis_params":{
"DAGMC": "fine_inres1.h5m",
"DAGMC": "inres1_shad.h5m",
"step_size": 0.01,
"max_steps": 10000,
"launch_pos": "fixed",
Expand Down
51 changes: 34 additions & 17 deletions src/aegis/aegis.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,24 @@
#include <cstdio>
#include <cstring>
#include <vector>
#include <array>
#include <mpi.h>
#include <unistd.h>
#include "Integrator.h"

int
main(int argc, char ** argv)
{

// Setup MPI
int rank, nprocs;
MPI_Init(&argc, &argv);

double startTime = MPI_Wtime(); // get start time on across all processes

MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
double startTime = MPI_Wtime(); // get start time across all processes
// ---------------------------------------------------------------------------------

// Get Config file
std::string configFileName;
if (argc > 1)
{
Expand All @@ -32,33 +35,44 @@ main(int argc, char ** argv)
}
configFileName = "aegis_settings.json";
}
// ---------------------------------------------------------------------------------

// equilibrium setup
auto configFile = std::make_shared<JsonHandler>(configFileName);

double equilibriumInstantiationStart = MPI_Wtime();
auto equilibrium = std::make_shared<EquilData>(configFile);
equilibrium->move();
equilibrium->psiref_override();
equilibrium->init_interp_splines();
equilibrium->centre(1);
double equilibriumInstantionTime = MPI_Wtime();
// ---------------------------------------------------------------------------------

ParticleSimulation simulation(configFile, equilibrium);
// Integrator setup
auto integrator = std::make_shared<SurfaceIntegrator>();
// ---------------------------------------------------------------------------------

// Main Particle simulation
ParticleSimulation simulation(configFile, equilibrium, integrator);
simulation.Execute();
// ---------------------------------------------------------------------------------

// print wall times for each process
// for (int i = 1; i < nprocs; ++i)
// {
// if (rank == i)
// {
// double endTime = MPI_Wtime();
// double totalTime = endTime - startTime;
// std::cout << "Elapsed wall Time on process " << i << " = " << totalTime << std::endl;
// std::cout << "----------------------------" << std::endl << std::endl;
// }
// }
// Print individual process stats
for (int i = 1; i < nprocs; ++i)
{
if (rank == i)
{
std::cout << "\nProcess " << i << " handled particles: \n";
integrator->print_particle_stats();
double endTime = MPI_Wtime();
double totalTime = endTime - startTime;
std::cout << "Elapsed wall Time on process " << i << " = " << totalTime << "\n";
}
}
std::cout << "\n";
// ---------------------------------------------------------------------------------

MPI_Finalize();
// Print wall time and other profiling times
if (rank == 0)
{
std::map<std::string, double> profilingTimes = simulation.get_profiling_times();
Expand All @@ -72,5 +86,8 @@ main(int argc, char ** argv)
std::cout << "Total wall time = " << MPI_Wtime() - startTime << "\n";
std::cout << "------------------------------------" << std::endl;
}
// ---------------------------------------------------------------------------------

MPI_Finalize();
return 0;
}
34 changes: 23 additions & 11 deletions src/aegis_lib/Integrator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
#include <moab/OrientedBoxTreeTool.hpp>
#include "SimpleLogger.h"

SurfaceIntegrator::SurfaceIntegrator() {}

// surface integrator class constructor
// initialise list of EntityHandles and maps associated with
SurfaceIntegrator::SurfaceIntegrator(moab::Range const & Facets)
Expand All @@ -21,6 +23,18 @@ SurfaceIntegrator::SurfaceIntegrator(moab::Range const & Facets)
}
}

void
SurfaceIntegrator::set_facets(moab::Range const & Facets)
{
nFacets = Facets.size();
for (const auto i : Facets)
{
facetEntities.push_back(i);
particlesShadowedBy[i] = 0;
powFac[i] = 0;
}
}

// Overload for constructor use with STL vector
SurfaceIntegrator::SurfaceIntegrator(std::vector<EntityHandle> const & Facets)
{
Expand Down Expand Up @@ -238,19 +252,17 @@ SurfaceIntegrator::piecewise_multilinear_out(
}
}

// return number of particles depositing, shadowed, lost etc.
void
SurfaceIntegrator::print_particle_stats()
{ // return number of particles depositing, shadowed, lost etc.

if (rank == 0)
{
LOG_WARNING << "Number of particles launched = " << nFacets;
LOG_WARNING << "Number of particles depositing power from omp = " << nParticlesHeatDep;
LOG_WARNING << "Number of shadowed particle intersections = " << nParticlesShadowed;
LOG_WARNING << "Number of particles lost from magnetic domain = " << nParticlesLost;
LOG_WARNING << "Number of particles terminated upon reaching max tracking length = "
<< nParticlesMaxLength;
}
{
std::cout << "DEPOSITING - " << nParticlesHeatDep << "\n";
std::cout << "SHADOWED - " << nParticlesShadowed << "\n";
std::cout << "LOST - " << nParticlesLost << "\n";
std::cout << "MAX LENGTH - " << nParticlesTotal << "\n";
int totalParticlesHandled =
nParticlesHeatDep + nParticlesShadowed + nParticlesLost + nParticlesMaxLength;
std::cout << "TOTAL - " << totalParticlesHandled << "\n";
};

void
Expand Down
3 changes: 2 additions & 1 deletion src/aegis_lib/Integrator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,13 @@ PADDED // padded null particle
class SurfaceIntegrator : public AegisBase
{
public:
SurfaceIntegrator();
SurfaceIntegrator(moab::Range const &Facets); // constructor (with moab::Range)
SurfaceIntegrator(std::vector<EntityHandle> const &Facets); // constructor (with std::vector<EntityHandle>)

// void q_values(); // return list of qvalues for each triangle
// void psi_values(); // return list of psi values for each triangle

void set_facets(moab::Range const &Facets);
void count_hit(EntityHandle facet_hit); // count hits belonging to each facet
void count_lost_ray();
void store_heat_flux(EntityHandle facet, double heatflux); // store the power associated with a particular facet
Expand Down
41 changes: 19 additions & 22 deletions src/aegis_lib/ParticleSimulation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,17 @@

// setup AEGIS simulation
ParticleSimulation::ParticleSimulation(std::shared_ptr<JsonHandler> configFile,
std::shared_ptr<EquilData> equil)
std::shared_ptr<EquilData> equilibirum,
std::shared_ptr<SurfaceIntegrator> integrator)
{
set_mpi_params();

read_params(configFile);
equilibrium = equil;
equilibrium = equilibirum;
DAG = std::make_unique<moab::DagMC>();
vtkInterface = std::make_unique<VtkInterface>(configFile);

init_geometry();
_integrator = integrator;
_integrator->set_facets(targetFacets);
}

// Call different execute functions depending on which execute type selected
Expand Down Expand Up @@ -110,7 +111,7 @@ ParticleSimulation::Execute_dynamic_mpi()
worker();
}

std::array<int, 4> particleStats = integrator->particle_stats();
std::array<int, 4> particleStats = _integrator->particle_stats();
std::array<int, 4> totalParticleStats;

if (rank != 0)
Expand All @@ -119,7 +120,7 @@ ParticleSimulation::Execute_dynamic_mpi()
}
else
{
totalParticleStats = integrator->particle_stats();
totalParticleStats = _integrator->particle_stats();
}

for (int i = 1; i < nprocs; ++i)
Expand Down Expand Up @@ -151,8 +152,6 @@ ParticleSimulation::Execute_dynamic_mpi()
aegisMeshWriteTime = MPI_Wtime() - aegisMeshWriteStart;
}

mpi_particle_stats();

print_particle_stats(totalParticleStats);
}

Expand Down Expand Up @@ -479,7 +478,6 @@ ParticleSimulation::init_geometry()
}

targetFacets = select_target_surface();
integrator = std::make_unique<SurfaceIntegrator>(targetFacets);
prepSurfacesTime = MPI_Wtime() - prepSurfacesStart;

double setupArrayOfParticlesTimeStart = MPI_Wtime();
Expand All @@ -499,7 +497,7 @@ ParticleSimulation::loop_over_particles(int startIndex, int endIndex)
for (int i = startIndex; i < endIndex; ++i)
{
auto particle = listOfParticles[i];
integrator->set_launch_position(particle.parent_entity_handle(), particle.get_pos());
_integrator->set_launch_position(particle.parent_entity_handle(), particle.get_pos());
terminationState particleState = cartesian_particle_track(particle);
double heatflux = (particleState == terminationState::DEPOSITING) ? particle.heatflux() : 0.0;
heatfluxVals.emplace_back(heatflux);
Expand All @@ -524,7 +522,6 @@ ParticleSimulation::setup_sources()
listOfParticles.reserve(num_particles_launched());
for (const auto & facetEH : targetFacets)
{
heatfluxMap.insert(std::pair(facetEH, 0.0));
std::vector<moab::EntityHandle> triangleNodes;
DAG->moab_instance()->get_adjacencies(&facetEH, 1, 0, false, triangleNodes);
DAG->moab_instance()->get_coords(&triangleNodes[0], triangleNodes.size(),
Expand Down Expand Up @@ -622,8 +619,8 @@ ParticleSimulation::terminate_particle_depositing(ParticleBase & particle)
std::stringstream terminationLogString;
double heatflux = particle.heatflux();
vtkInterface->write_particle_track(branchDepositingPart, heatflux);
integrator->count_particle(particle.parent_entity_handle(), terminationState::DEPOSITING,
heatflux);
_integrator->count_particle(particle.parent_entity_handle(), terminationState::DEPOSITING,
heatflux);
terminationLogString << "Midplane reached. Depositing power after travelling " << trackLength
<< " units";
log_string(LogLevel::INFO, terminationLogString.str());
Expand All @@ -636,7 +633,8 @@ ParticleSimulation::terminate_particle_shadow(ParticleBase & particle)
std::stringstream terminationLogString;
double heatflux = 0.0;
vtkInterface->write_particle_track(branchShadowedPart, heatflux);
integrator->count_particle(particle.parent_entity_handle(), terminationState::SHADOWED, heatflux);
_integrator->count_particle(particle.parent_entity_handle(), terminationState::SHADOWED,
heatflux);
terminationLogString << "Surface " << nextSurf << " hit after travelling " << trackLength
<< " units";
log_string(LogLevel::INFO, terminationLogString.str());
Expand All @@ -649,7 +647,7 @@ ParticleSimulation::terminate_particle_lost(ParticleBase & particle)
std::stringstream terminationLogString;
double heatflux = 0.0;
vtkInterface->write_particle_track(branchLostPart, heatflux);
integrator->count_particle(particle.parent_entity_handle(), terminationState::LOST, heatflux);
_integrator->count_particle(particle.parent_entity_handle(), terminationState::LOST, heatflux);
terminationLogString << "Particle leaving magnetic field after travelling " << trackLength
<< " units";
log_string(LogLevel::INFO, terminationLogString.str());
Expand All @@ -662,8 +660,8 @@ ParticleSimulation::terminate_particle_maxlength(ParticleBase & particle)
std::stringstream terminationLogString;
double heatflux = 0.0;
vtkInterface->write_particle_track(branchMaxLengthPart, heatflux);
integrator->count_particle(particle.parent_entity_handle(), terminationState::MAXLENGTH,
heatflux);
_integrator->count_particle(particle.parent_entity_handle(), terminationState::MAXLENGTH,
heatflux);
terminationLogString << "Fieldline trace reached maximum length before intersection";
}

Expand Down Expand Up @@ -793,7 +791,7 @@ ParticleSimulation::mpi_particle_stats()
{
std::cout << std::endl
<< "process " << i << " has the following particle stats:" << std::endl;
std::array localRankParticleStats = integrator->particle_stats();
std::array localRankParticleStats = _integrator->particle_stats();

std::cout << "DEPOSITING - " << localRankParticleStats[0] << std::endl;
std::cout << "SHADOWED - " << localRankParticleStats[1] << std::endl;
Expand Down Expand Up @@ -862,7 +860,7 @@ ParticleSimulation::Execute_serial()

// write out data and print final

std::array<int, 4> particleStats = integrator->particle_stats();
std::array<int, 4> particleStats = _integrator->particle_stats();

vtkInterface->write_multiBlockData("particle_tracks.vtm");

Expand Down Expand Up @@ -918,7 +916,7 @@ ParticleSimulation::Execute_mpi()
qvalues = loop_over_particles(startIndex, endIndex); // perform main loop
double endTime = MPI_Wtime();

std::array<int, 4> particleStats = integrator->particle_stats();
std::array<int, 4> particleStats = _integrator->particle_stats();
std::array<int, 4> totalParticleStats;
if (rank != 0)
{
Expand All @@ -930,7 +928,7 @@ ParticleSimulation::Execute_mpi()
}
else
{
totalParticleStats = integrator->particle_stats();
totalParticleStats = _integrator->particle_stats();
MPI_Gatherv(qvalues.data(), qvalues.size(), MPI_DOUBLE, rootQvalues.data(),
recieveCounts.data(), displacements.data(), MPI_DOUBLE, rootRank, MPI_COMM_WORLD);
// MPI_Gather(qvalues.data(), qvalues.size(), MPI_DOUBLE,
Expand All @@ -952,7 +950,6 @@ ParticleSimulation::Execute_mpi()
mainParticleLoopTime = MPI_Wtime() - mainParticleLoopStart;
}

mpi_particle_stats();
print_particle_stats(totalParticleStats);
}

Expand Down
Loading

0 comments on commit dad6be6

Please sign in to comment.