From 0bc6d4cc7ce761a82b196be9ec95c0f91e36f07d Mon Sep 17 00:00:00 2001 From: Dylan Copeland Date: Tue, 18 Mar 2025 12:24:42 -0700 Subject: [PATCH 1/5] Enabling BasisReader to load a serial basis on each rank independently of other world-communicator ranks. --- lib/linalg/BasisReader.cpp | 21 ++++++++++++--------- lib/linalg/BasisReader.h | 5 ++++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index 37d9cd591..12de46f7a 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -23,19 +23,21 @@ namespace CAROM { BasisReader::BasisReader( const std::string& base_file_name, Database::formats db_format, - const int dim) : + const int dim, + MPI_Comm comm) : d_dim(dim), full_file_name(""), base_file_name_(base_file_name), d_format(db_format) { CAROM_ASSERT(!base_file_name.empty()); + d_distributed = comm != MPI_COMM_NULL; int mpi_init; MPI_Initialized(&mpi_init); int rank; - if (mpi_init) { - MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (mpi_init && d_distributed) { + MPI_Comm_rank(comm, &rank); } else { rank = 0; @@ -55,7 +57,7 @@ BasisReader::BasisReader( We allow 0 local dimension. (global dimension still needs to be positive) */ std::vector tmp; - d_global_dim = get_global_offsets(d_dim, tmp, MPI_COMM_WORLD); + d_global_dim = get_global_offsets(d_dim, tmp, comm); CAROM_VERIFY(d_dim >= 0); CAROM_VERIFY(d_global_dim > 0); d_database = new HDFDatabaseMPIO(); @@ -63,7 +65,7 @@ BasisReader::BasisReader( else CAROM_ERROR("BasisWriter only supports HDF5/HDF5_MPIO data format!\n"); - d_database->open(full_file_name, "r", MPI_COMM_WORLD); + d_database->open(full_file_name, "r", comm); } BasisReader::~BasisReader() @@ -78,7 +80,7 @@ BasisReader::getSpatialBasis() int num_rows = getDim("basis"); int num_cols = getNumSamples("basis"); - Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols, true); + Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols, d_distributed); d_database->getDoubleArray("spatial_basis", &spatial_basis_vectors->item(0, 0), @@ -107,7 +109,8 @@ BasisReader::getSpatialBasis( CAROM_VERIFY(start_col <= end_col && end_col <= num_cols); int num_cols_to_read = end_col - start_col + 1; - Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols_to_read, true); + Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols_to_read, + d_distributed); sprintf(tmp, "spatial_basis"); d_database->getDoubleArray(tmp, &spatial_basis_vectors->item(0, 0), @@ -324,7 +327,7 @@ BasisReader::getSnapshotMatrix() int num_cols = getNumSamples("snapshot"); char tmp[100]; - Matrix* snapshots = new Matrix(num_rows, num_cols, true); + Matrix* snapshots = new Matrix(num_rows, num_cols, d_distributed); sprintf(tmp, "snapshot_matrix"); d_database->getDoubleArray(tmp, &snapshots->item(0, 0), @@ -353,7 +356,7 @@ BasisReader::getSnapshotMatrix( int num_cols_to_read = end_col - start_col + 1; char tmp[100]; - Matrix* snapshots = new Matrix(num_rows, num_cols_to_read, true); + Matrix* snapshots = new Matrix(num_rows, num_cols_to_read, d_distributed); sprintf(tmp, "snapshot_matrix"); d_database->getDoubleArray(tmp, &snapshots->item(0, 0), diff --git a/lib/linalg/BasisReader.h b/lib/linalg/BasisReader.h index 3488196dd..113e09ea1 100644 --- a/lib/linalg/BasisReader.h +++ b/lib/linalg/BasisReader.h @@ -51,7 +51,8 @@ class BasisReader { BasisReader( const std::string& base_file_name, Database::formats db_format = Database::formats::HDF5, - const int dim = -1); + const int dim = -1, + MPI_Comm comm = MPI_COMM_WORLD); /** * @brief Destructor. @@ -314,6 +315,8 @@ class BasisReader { * If negative, use the dimension from the rank-specific local file. */ int d_global_dim; + + bool d_distributed; }; } From df46dd3420c3998653f18d67fd9cf15ebc777689 Mon Sep 17 00:00:00 2001 From: Dylan Copeland Date: Wed, 3 Sep 2025 13:27:37 -0700 Subject: [PATCH 2/5] Adding distributed flag to database read. --- lib/linalg/BasisReader.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index 12de46f7a..22cd16df9 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -85,7 +85,7 @@ BasisReader::getSpatialBasis() d_database->getDoubleArray("spatial_basis", &spatial_basis_vectors->item(0, 0), num_rows*num_cols, - true); + d_distributed); return std::unique_ptr(spatial_basis_vectors); } @@ -118,7 +118,7 @@ BasisReader::getSpatialBasis( start_col - 1, num_cols_to_read, num_cols, - true); + d_distributed); return std::unique_ptr(spatial_basis_vectors); } @@ -159,7 +159,7 @@ BasisReader::getTemporalBasis() sprintf(tmp, "temporal_basis"); d_database->getDoubleArray(tmp, &temporal_basis_vectors->item(0, 0), - num_rows*num_cols); + num_rows*num_cols, d_distributed); return std::unique_ptr(temporal_basis_vectors); } @@ -190,7 +190,8 @@ BasisReader::getTemporalBasis( num_rows*num_cols_to_read, start_col - 1, num_cols_to_read, - num_cols); + num_cols, + d_distributed); return std::unique_ptr(temporal_basis_vectors); } @@ -232,7 +233,8 @@ BasisReader::getSingularValues() sprintf(tmp, "singular_value"); d_database->getDoubleArray(tmp, &singular_values->item(0), - size); + size, + d_distributed); return std::unique_ptr(singular_values); } @@ -332,7 +334,7 @@ BasisReader::getSnapshotMatrix() d_database->getDoubleArray(tmp, &snapshots->item(0, 0), num_rows*num_cols, - true); + d_distributed); return std::unique_ptr(snapshots); } @@ -364,7 +366,7 @@ BasisReader::getSnapshotMatrix( start_col - 1, num_cols_to_read, num_cols, - true); + d_distributed); return std::unique_ptr(snapshots); } } From b900a1bac78cff8a55b7fcc899558fe954171150 Mon Sep 17 00:00:00 2001 From: Dylan Copeland Date: Wed, 3 Sep 2025 17:38:17 -0700 Subject: [PATCH 3/5] Adding unit test to read a small serial file in parallel. --- unit_tests/basis_data/basis.000000 | Bin 0 -> 4816 bytes unit_tests/test_HDFDatabase.cpp | 28 ++++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 unit_tests/basis_data/basis.000000 diff --git a/unit_tests/basis_data/basis.000000 b/unit_tests/basis_data/basis.000000 new file mode 100644 index 0000000000000000000000000000000000000000..c5e2aa95bfb008b99ac3fa05309b3971158c4a49 GIT binary patch literal 4816 zcmeHK%}&BV5T3T66)`ECj3;~p4jjCJ2^bDWjdCy55N%R{Q1R~x^i4c^^rnyC&DZb^ zaA$YM0){rIM2I_0_M3h)JKN6Aw!8QB>dsnbBLgV54JmM>%FTe}HLn=HXM2+rgYb^< z!BmO?cmb>vpQHHWevQLg4TLAhXQXaWOexJ~LgT;6K)tqofG_+=Z`~zya-_@%3|N-+ zs~W6ORmp~-DxT|}(GvBIy0ZdeEuj=-n+99*rfEu7o8nl8Fi4>SS2vI+8l8J@PiI*u zN{az$$O>EIrrSaS$B_Yt{E=52YISUWO^){VtN08}iujU{|Ig8#_(#|x^W9RsUxCcm z>3Bc>P86(ozrV81zOGUWRaZ&*;Q3^<#QMP+&ux~e@+(kUh0=iLd-8e{KRGEC1=Rj@ zy+y%B<3{}4YWU%yzjgGC&+m~wtWDxEMw$9B4)@c#&VbH<&VbH<&VbIq{A3_$-^J$$ z#_7*D^@ZAp^2~%?@51*2_t*>l&~5k6-CpM^1o+CIbb@hS1PT52X+Q9K?xh#>TW;vz i%H!G?ADz=GJn)p5G=IFYWBmTUd6(vK^dL8H`{oaE1A0dQ literal 0 HcmV?d00001 diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index 7aff502c1..f6e58921a 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -284,6 +284,34 @@ TEST(HDF5, Test_parallel_reading) EXPECT_TRUE(data[d] == d + offsets[rank] * ncol); } +TEST(HDF5, Test_serial_file_parallel_reading) +{ + // Read the matrix on all ranks. + CAROM::BasisReader reader("./basis_data/basis.000000", + CAROM::Database::formats::HDF5, -1, + MPI_COMM_NULL); + std::unique_ptr B = reader.getSpatialBasis(); + + EXPECT_EQ(B->numRows(), 4); + EXPECT_EQ(B->numColumns(), 2); + + double error = 0.0; + for (int i=0; inumRows(); ++i) + for (int j=0; jnumColumns(); ++j) + { + const double B_ij = (i == 1 && j == 0) || (i == 2 && j == 1) ? -1.0 : 0.0; + error = std::max(error, std::abs(B_ij - (*B)(i, j))); + } + + std::unique_ptr sv = reader.getSingularValues(); + EXPECT_EQ(sv.dim(), 2); + + for (int i=0; i Date: Thu, 4 Sep 2025 11:35:25 -0700 Subject: [PATCH 4/5] Fixed new unit test. --- unit_tests/CMakeLists.txt | 2 ++ unit_tests/test_HDFDatabase.cpp | 57 +++++++++++++++++---------------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/unit_tests/CMakeLists.txt b/unit_tests/CMakeLists.txt index 0410447c0..fb242ebd2 100644 --- a/unit_tests/CMakeLists.txt +++ b/unit_tests/CMakeLists.txt @@ -88,6 +88,8 @@ endforeach() # Copy testing inputs and scripts to build/tests/ file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/baselines/basis_conversion/ DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/baselines/basis_conversion/") +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/basis_data/ + DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/basis_data/") file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/s_opt_data/ DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/s_opt_data/") diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index f6e58921a..7f18ec84f 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -284,34 +284,6 @@ TEST(HDF5, Test_parallel_reading) EXPECT_TRUE(data[d] == d + offsets[rank] * ncol); } -TEST(HDF5, Test_serial_file_parallel_reading) -{ - // Read the matrix on all ranks. - CAROM::BasisReader reader("./basis_data/basis.000000", - CAROM::Database::formats::HDF5, -1, - MPI_COMM_NULL); - std::unique_ptr B = reader.getSpatialBasis(); - - EXPECT_EQ(B->numRows(), 4); - EXPECT_EQ(B->numColumns(), 2); - - double error = 0.0; - for (int i=0; inumRows(); ++i) - for (int j=0; jnumColumns(); ++j) - { - const double B_ij = (i == 1 && j == 0) || (i == 2 && j == 1) ? -1.0 : 0.0; - error = std::max(error, std::abs(B_ij - (*B)(i, j))); - } - - std::unique_ptr sv = reader.getSingularValues(); - EXPECT_EQ(sv.dim(), 2); - - for (int i=0; i B = reader.getSpatialBasis(); + + EXPECT_EQ(B->numRows(), 4); + EXPECT_EQ(B->numColumns(), 2); + EXPECT_FALSE(B->distributed()); + + double error = 0.0; + for (int i=0; inumRows(); ++i) + for (int j=0; jnumColumns(); ++j) + { + const double B_ij = (i == 1 && j == 0) || (i == 2 && j == 1) ? -1.0 : 0.0; + error = std::max(error, std::abs(B_ij - (*B)(i, j))); + } + + std::unique_ptr sv = reader.getSingularValues(); + EXPECT_EQ(sv->dim(), 2); + + for (int i=0; idim(); ++i) + error = std::max(error, std::abs((*sv)(i) - 1.0)); + + EXPECT_NEAR(error, 0.0, threshold); +} + TEST(BasisGeneratorIO, HDFDatabase) { // Get the rank of this process, and the number of processors. From 47678e0b9509608294ee1633f1fc9ea73c7ae837 Mon Sep 17 00:00:00 2001 From: Dylan Copeland Date: Tue, 9 Sep 2025 09:59:04 -0700 Subject: [PATCH 5/5] Verify that BasisReader comm is NULL or WORLD. --- lib/linalg/BasisReader.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index 22cd16df9..a87a51b0c 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -30,7 +30,8 @@ BasisReader::BasisReader( base_file_name_(base_file_name), d_format(db_format) { - CAROM_ASSERT(!base_file_name.empty()); + CAROM_VERIFY(!base_file_name.empty()); + CAROM_VERIFY(comm == MPI_COMM_NULL || comm == MPI_COMM_WORLD); d_distributed = comm != MPI_COMM_NULL; int mpi_init;