From d22e51cdfab6711153beb346e665193cc839fe10 Mon Sep 17 00:00:00 2001 From: Waqar Butt Date: Mon, 29 Apr 2024 11:19:37 +0100 Subject: [PATCH 1/2] progress bar and minor QOL fixes --- inres1/aegis_settings.json | 4 +- src/aegis/aegis.cpp | 26 +++++++-- src/aegis_lib/ParticleSimulation.cpp | 83 +++++++++++++++++----------- src/aegis_lib/ParticleSimulation.h | 9 +-- 4 files changed, 80 insertions(+), 42 deletions(-) diff --git a/inres1/aegis_settings.json b/inres1/aegis_settings.json index dc42e81..64a8123 100644 --- a/inres1/aegis_settings.json +++ b/inres1/aegis_settings.json @@ -1,5 +1,6 @@ { "description": "inres1 test case to compare against SMARDDA", + "global_params": {"mpi_timings": false}, "aegis_params":{ "DAGMC": "inres1_shad.h5m", "step_size": 0.01, @@ -9,10 +10,11 @@ "force_no_deposition": false, "coordinate_system": "flux", "execution_type": "dynamic", + "print_mpi_particle_stats":false, "dynamic_batch_params":{ "batch_size":16, "worker_profiling": false, - "worker_debug":false + "worker_debug":false } }, "equil_params":{ diff --git a/src/aegis/aegis.cpp b/src/aegis/aegis.cpp index caded3c..e01d357 100644 --- a/src/aegis/aegis.cpp +++ b/src/aegis/aegis.cpp @@ -44,17 +44,31 @@ main(int argc, char ** argv) ParticleSimulation simulation(configFile, equilibrium); simulation.Execute(); - for (int i = 0; i < nprocs; ++i) + auto mpiTimings = configFile->data()["global_params"]["mpi_timings"]; + + if (mpiTimings) // print individual timings for each MPI rank { - if (rank == i) + for (int i = 1; i < nprocs; ++i) { - double endTime = MPI_Wtime(); - double totalTime = endTime - startTime; - std::cout << "Elapsed wall Time on process " << i << " = " << totalTime << std::endl; - std::cout << "----------------------------" << std::endl << std::endl; + if (rank == i) + { + double endTime = MPI_Wtime(); + double totalTime = endTime - startTime; + std::cout << "Elapsed wall Time on process " << i << " = " << totalTime << "\n"; + std::cout << "---------------------------- \n \n"; + } } } + if (rank == 0) // print final wall time once rank 0 is complete + { + double endTime = MPI_Wtime(); + double totalTime = endTime - startTime; + std::cout << "\n---------------------------- \n"; + std::cout << "Total elapsed wall Time = " << totalTime << "\n"; + std::cout << "---------------------------- \n" << std::endl; + } + MPI_Finalize(); return 0; diff --git a/src/aegis_lib/ParticleSimulation.cpp b/src/aegis_lib/ParticleSimulation.cpp index a67ba2c..4272271 100644 --- a/src/aegis_lib/ParticleSimulation.cpp +++ b/src/aegis_lib/ParticleSimulation.cpp @@ -66,36 +66,38 @@ ParticleSimulation::test_cyl_ray_fire(std::unique_ptr & particle) void ParticleSimulation::Execute() { - string_to_lowercase(exeType); + string_to_lowercase(executionType); - if (exeType == "serial" || nprocs == 1) + if (executionType == "serial" || nprocs == 1) { log_string(LogLevel::WARNING, "Executing in serial mode..."); if (nprocs > 1) { log_string(LogLevel::ERROR, "Aegis was invoked with mpirun but config suggests serial run. " "Defaulting to MPI with no load balancing..."); + executionType = "mpi"; Execute_mpi(); } else { + executionType = "serial"; Execute_serial(); } } - else if (exeType == "mpi") + else if (executionType == "mpi") { log_string(LogLevel::WARNING, "Executing MPI with no load balancing..."); Execute_mpi(); } - else if (exeType == "mpi_padded" || exeType == "padded") + else if (executionType == "mpi_padded" || executionType == "padded") { log_string(LogLevel::WARNING, "Executing padded MPI with no load balancing..."); Execute_padded_mpi(); } - else if (exeType == "mpi_dynamic" || exeType == "dynamic") + else if (executionType == "mpi_dynamic" || executionType == "dynamic") { log_string(LogLevel::WARNING, "Executing dynamic MPI load balancing..."); Execute_dynamic_mpi(); @@ -178,8 +180,10 @@ ParticleSimulation::Execute_dynamic_mpi() write_out_mesh(meshWriteOptions::BOTH, targetFacets); } - mpi_particle_stats(); - + if (printMpiParticleStats) + { + mpi_particle_stats(); + } print_particle_stats(totalParticleStats); } @@ -225,24 +229,11 @@ ParticleSimulation::handler(std::vector & handlerQVals) MPI_Recv(workerQVals.data(), workerQVals.size(), MPI_DOUBLE, MPI_ANY_SOURCE, mpiDataTag, MPI_COMM_WORLD, &status); MPI_Recv(&workerStartIndex, 1, MPI_INT, MPI_ANY_SOURCE, mpiIndexTag, MPI_COMM_WORLD, &status); - // printf("Time taken to recieve next data = %f \n", MPI_Wtime()); particlesHandled += dynamicBatchSize; batchesComplete += 1; - // progress indicator - if (batchesComplete == totalBatches / 4) - { - std::cout << "25% ... " << std::flush; - } - else if (batchesComplete == totalBatches / 2) - { - std::cout << "50% ... " << std::flush; - } - else if (batchesComplete == 3 * totalBatches / 4) - { - std::cout << "75% ... " << std::flush; - } + update_progress_indicator(batchesComplete, totalBatches); counter += workerQVals.size(); double * ptr = handlerQVals.data() + workerStartIndex; @@ -373,7 +364,10 @@ ParticleSimulation::read_params(const std::shared_ptr & configFile) coordinateConfig = aegisParams.get_optional("coordinate_system").value_or(coordinateConfig); - exeType = aegisParams.get_optional("execution_type").value_or(exeType); + executionType = aegisParams.get_optional("execution_type").value_or(executionType); + + printMpiParticleStats = + aegisParams.get_optional("print_mpi_particle_stats").value_or(printMpiParticleStats); if (aegisParamsData.contains("dynamic_batch_params")) { @@ -536,6 +530,10 @@ ParticleSimulation::loop_over_facets(int startFacet, int endFacet) } heatfluxVals.push_back(tri.heatflux()); + if (executionType == "serial") + { + update_progress_indicator(i, endFacet); + } } double endTime = MPI_Wtime(); @@ -829,19 +827,21 @@ ParticleSimulation::mpi_particle_stats() { if (rank == i) { - std::cout << std::endl - << "process " << i << " has the following particle stats:" << std::endl; + std::cout << "\n" + << "process " << i << " has the following particle stats:" + << "\n"; std::array localRankParticleStats = integrator->particle_stats(); - std::cout << "DEPOSITING - " << localRankParticleStats[0] << std::endl; - std::cout << "SHADOWED - " << localRankParticleStats[1] << std::endl; - std::cout << "LOST - " << localRankParticleStats[2] << std::endl; - std::cout << "MAX LENGTH - " << localRankParticleStats[3] << std::endl; + std::cout << "DEPOSITING - " << localRankParticleStats[0] << "\n"; + std::cout << "SHADOWED - " << localRankParticleStats[1] << "\n"; + std::cout << "LOST - " << localRankParticleStats[2] << "\n"; + std::cout << "MAX LENGTH - " << localRankParticleStats[3] << "\n"; int totalParticlesHandled = localRankParticleStats[0] + localRankParticleStats[1] + localRankParticleStats[2] + localRankParticleStats[3]; - std::cout << "TOTAL - " << totalParticlesHandled << std::endl; + std::cout << "TOTAL - " << totalParticlesHandled << "\n"; + std::cout << std::endl; } } } @@ -960,7 +960,10 @@ ParticleSimulation::Execute_padded_mpi() write_out_mesh(meshWriteOptions::BOTH, targetFacets); } - mpi_particle_stats(); + if (printMpiParticleStats) + { + mpi_particle_stats(); + } print_particle_stats(totalParticleStats); } @@ -1048,7 +1051,10 @@ ParticleSimulation::Execute_mpi() write_out_mesh(meshWriteOptions::BOTH, targetFacets); } - mpi_particle_stats(); + if (printMpiParticleStats) + { + mpi_particle_stats(); + } print_particle_stats(totalParticleStats); endTime = MPI_Wtime(); @@ -1147,6 +1153,8 @@ ParticleSimulation::write_out_mesh(meshWriteOptions option, moab::Range rangeofE { // mesh_coord_transform(coordinateSystem::CARTESIAN); // get target meshset for aegis_target outputs + std::cout << std::endl; + DAG->remove_graveyard(); EntityHandle targetMeshset; DAG->moab_instance()->create_meshset(MESHSET_SET, targetMeshset); @@ -1241,4 +1249,17 @@ ParticleSimulation::mesh_coord_transform(coordinateSystem coordSys) std::cout << "No coordinate system provided for mesh coordinate transform" << std::endl; break; } +} + +// progress indicator +void +ParticleSimulation::update_progress_indicator(int batchesComplete, int totalBatches) +{ + for (int i = 1; i <= 10; ++i) + { + if (batchesComplete == (i * totalBatches / 10)) + { + std::cout << "| " << i * 10 << "% " << std::flush; + } + } } \ No newline at end of file diff --git a/src/aegis_lib/ParticleSimulation.h b/src/aegis_lib/ParticleSimulation.h index 552c3c9..fed502a 100644 --- a/src/aegis_lib/ParticleSimulation.h +++ b/src/aegis_lib/ParticleSimulation.h @@ -122,11 +122,11 @@ class ParticleSimulation : public AegisBase void write_out_mesh(meshWriteOptions option, moab::Range rangeOfEntities = {}); void mesh_coord_transform(coordinateSystem coordSys); void select_coordinate_system(); - + void update_progress_indicator(int batchesComplete, int totalBatches); + // Simulation config parameters - std::string exeType = "dynamic"; - std::string dagmcInputFile; // no defaults because - std::string eqdskInputFile; + std::string executionType = "dynamic"; + std::string dagmcInputFile; // parameter required to be specified by user double powerSOL = 0.0; double lambdaQ = 0.0; double trackStepSize = 0.001; @@ -139,6 +139,7 @@ class ParticleSimulation : public AegisBase bool workerDebug = false; coordinateSystem coordSys = coordinateSystem::CARTESIAN; // default cartesian bool noMidplaneTermination = false; + bool printMpiParticleStats = false; std::vector listOfTriangles; int totalNumberOfFacets = 0; From 673007a3938ed5de0fccf8744c2797abe14475cd Mon Sep 17 00:00:00 2001 From: Waqar Butt Date: Mon, 29 Apr 2024 11:38:20 +0100 Subject: [PATCH 2/2] proper handling of json parameter mpiTimings --- src/aegis/aegis.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aegis/aegis.cpp b/src/aegis/aegis.cpp index e01d357..3378a72 100644 --- a/src/aegis/aegis.cpp +++ b/src/aegis/aegis.cpp @@ -44,7 +44,8 @@ main(int argc, char ** argv) ParticleSimulation simulation(configFile, equilibrium); simulation.Execute(); - auto mpiTimings = configFile->data()["global_params"]["mpi_timings"]; + JsonHandler globalParams(configFile->data()["global_params"]); + bool mpiTimings = globalParams.get_optional("mpi_timings").value_or(false); if (mpiTimings) // print individual timings for each MPI rank {