diff --git a/CMakeLists.txt b/CMakeLists.txt index 1320f5c3e..812b0e318 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -321,7 +321,7 @@ SET(STATUS_PNETCDF ${PnetCDF_C_FOUND}) ### CHECK_C_SOURCE_COMPILES(" #include -#if !NC_HAS_NC4 +#if !NC_HAS_HDF5 && !NC_HAS_NC4 choke me #endif int main() {return 0;}" HAVE_NETCDF4) @@ -452,7 +452,7 @@ SET(LDFLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CMAKE_EXE_LINKER_FLAGS_${CMAKE_BUILD_TY is_disabled(BUILD_SHARED_LIBS enable_static) is_enabled(BUILD_SHARED_LIBS enable_shared) -is_enabled(HAVE_PAR_FILTERS have_par_filters) +is_enabled(PIO_HAS_PAR_FILTERS have_par_filters) is_enabled(USE_SZIP HAS_SZIP_WRITE) is_enabled(STATUS_PNETCDF HAS_PNETCDF) is_enabled(HAVE_H5Z_SZIP HAS_SZLIB) diff --git a/README.md b/README.md index eb523848f..662e4662c 100644 --- a/README.md +++ b/README.md @@ -1,117 +1,114 @@ # ParallelIO -The Parallel IO libraries (PIO) are high-level parallel I/O C and -Fortran libraries for applications that need to do netCDF I/O from -large numbers of processors on a HPC system. - -PIO provides a netCDF-like API, and allows users to designate some -subset of processors to perform IO. Computational code calls -netCDF-like functions to read and write data, and PIO uses the IO -processors to perform all necessary IO. - -## Intracomm Mode - -In Intracomm mode, PIO allows the user to designate some subset of -processors to do all I/O. The I/O processors also participate in -computational work. - -![I/O on Many Processors with Async - Mode](./doc/images/I_O_on_Many_Intracomm.png) - -## Async Mode - -PIO also supports the creation of multiple computation components, -each containing many processors, and one shared set of IO -processors. The computational components can perform write operation -asynchronously, and the IO processors will take care of all storage -interaction. - -![I/O on Many Processors with Async - Mode](./doc/images/I_O_on_Many_Async.png) - -## Website - -For complete documentation, see our website at -[http://ncar.github.io/ParallelIO/](http://ncar.github.io/ParallelIO/). - -## Mailing List - -The (low-traffic) PIO mailing list is at -https://groups.google.com/forum/#!forum/parallelio, send email to the -list at parallelio@googlegroups.com. - -## Testing - -The results of our continuous integration testing with GitHub actions -can be found on any of the Pull Requests on the GitHub site: -https://github.com/NCAR/ParallelIO. +## Overview +This repository provides a robust implementation of **Parallel I/O (PIO)** using **NetCDF** and **ParallelNetCDF** backends. It is optimized for **high-performance computing (HPC)** and integrates with parallel file systems to efficiently handle large-scale scientific datasets. + +## Features +- Parallel I/O support using **NetCDF** and **ParallelNetCDF**. +- Optimized **data rearrangement** for improved performance. +- **Asynchronous I/O** for reducing computational overhead. +- Compatible with **MPI-based distributed computing**. +- Extensive testing suite and continuous integration with **GitHub Actions**. + +## Installation +### Prerequisites +Ensure you have the following dependencies installed: +- C and Fortran compilers (e.g., GCC, Intel, Clang) +- **MPI library** (MPICH or OpenMPI) +- **NetCDF** and **ParallelNetCDF** (built with MPI support) +- Either **CMake** or **Autotools** + +### Build Instructions +#### Using CMake +```bash +mkdir build && cd build +cmake .. -DENABLE_FORTRAN=ON -DENABLE_NETCDF=ON +make -j$(nproc) +make install +``` -The results of our nightly tests on multiple platforms can be found on -our cdash site at -[http://my.cdash.org/index.php?project=PIO](http://my.cdash.org/index.php?project=PIO). +#### Using Autotools +```bash +autoreconf -i +./configure --enable-fortran --enable-netcdf-integration +make -j$(nproc) +make install +``` -## Dependencies +## Running Tests +### Unit and Integration Tests +To run the test suite, use the following commands: +```bash +mpiexec -n 4 make check # Run all unit tests +ctest -VV # Run CMake-based tests (must be run on a parallel-capable node) +``` -PIO can use NetCDF (version 4.6.1+) and/or PnetCDF (version 1.9.0+) -for I/O. NetCDF may be built with or without netCDF-4 features. NetCDF -is required for PIO, PnetCDF is optional. +For debugging failed tests: +```bash +ctest --rerun-failed --output-on-failure # Must be run on a parallel-capable node +``` -The NetCDF C library must be built with MPI, which requires that it be -linked with an MPI-enabled version of HDF5. Optionally, NetCDF can be -built with DAP support, which introduces a dependency on CURL. HDF5, -itself, introduces dependencies on LIBZ and (optionally) SZIP. +## Usage +### Simple Example (C) +```c +#include +MPI_Init(NULL, NULL); +PIO_Init(); +PIO_CreateFile("output.nc", PIO_WRITE, PIO_NETCDF); +PIO_CloseFile(); +PIO_Finalize(); +MPI_Finalize(); +``` -## Building PIO +### Simple Example (Fortran) +```fortran +program test_pio + use pio + call MPI_Init() + call PIO_Init() + call PIO_CreateFile("output.nc", PIO_WRITE, PIO_NETCDF) + call PIO_CloseFile() + call PIO_Finalize() + call MPI_Finalize() +end program test_pio +``` -To build PIO, unpack the distribution tarball and do: +## Contributing +We welcome contributions! To contribute: +1. Fork the repository. +2. Create a feature branch. +3. Run tests before committing changes. +4. Submit a pull request with a clear description. -``` -CC=mpicc FC=mpif90 ./configure --enable-fortran && make check install -``` +Refer to **doc/contributing_code.txt** for detailed guidelines. -For a full description of the available options and flags, try: -``` -./configure --help -``` +## Intracomm and Asynchronous Modes +ParallelIO supports two primary modes of operation: -Note that environment variables CC and FC may need to be set to the -MPI versions of the C and Fortran compiler. Also CPPFLAGS and LDFLAGS -may need to be set to indicate the locations of one or more of the -dependent libraries. (If using MPI compilers, the entire set of -dependent libraries should be built with the same compilers.) For -example: +### Intracomm Mode +- In **Intracomm mode**, a subset of MPI tasks participate in I/O operations while also performing computation tasks. This approach avoids the bottleneck that occurs when every task participates in I/O, improving overall performance. +- This mode ensures efficient parallel data writes and reads across distributed systems. +- Suitable for systems where dedicated I/O tasks are not needed. -``` -export CC=mpicc -export FC=mpifort -export CPPFLAGS='-I/usr/local/netcdf-fortran-4.4.5_c_4.6.3_mpich-3.2/include -I/usr/local/netcdf-c-4.6.3_hdf5-1.10.5/include -I/usr/local/pnetcdf-1.11.0_shared/include' -export LDFLAGS='-L/usr/local/netcdf-c-4.6.3_hdf5-1.10.5/lib -L/usr/local/pnetcdf-1.11.0_shared/lib' -./configure --prefix=/usr/local/pio-2.4.2 --enable-fortran -make check -make install -``` +#### Illustration: +![Intracomm Mode](./doc/images/I_O_on_Many_Intracomm.png) -## Building with CMake +### Asynchronous Mode +- In **Asynchronous mode**, a subset of tasks handle I/O while others focus on computation. +- Helps in overlapping computation and I/O for better performance. -The typical configuration with CMake can be done as follows: -``` -CC=mpicc FC=mpif90 cmake [-DOPTION1=value1 -DOPTION2=value2 ...] /path/to/pio/source -``` +#### Illustration: +![Asynchronous Mode](doc/images/I_O_on_Many_Async.png) -Full instructions for the cmake build can be found in the [installation -documentation](https://ncar.github.io/ParallelIO/install.html). +## Documentation +- [NetCDF Homepage](https://www.unidata.ucar.edu/software/netcdf/) +- [ParallelNetCDF Homepage](https://parallel-netcdf.github.io/) +- [User Guide](doc/users_guide.txt) +- [API Documentation](doc/api.txt) +- [Testing Guidelines](doc/Testing.txt) -# References +## License +This project is licensed under the **MIT License**. See the `LICENSE` file for details. -Hartnett, E., Edwards, J., "THE PARALLELIO (PIO) C/FORTRAN LIBRARIES -FOR SCALABLE HPC PERFORMANCE", 37th Conference on Environmental -Information Processing Technologies, American Meteorological Society -Annual Meeting, January, 2021. Retrieved on Feb 3, 2021, from -[https://www.researchgate.net/publication/348169990_THE_PARALLELIO_PIO_CFORTRAN_LIBRARIES_FOR_SCALABLE_HPC_PERFORMANCE]. -Hartnett, E., Edwards, J., "POSTER: THE PARALLELIO (PIO) C/FORTRAN LIBRARIES -FOR SCALABLE HPC PERFORMANCE", 37th Conference on Environmental -Information Processing Technologies, American Meteorological Society -Annual Meeting, January, 2021. Retrieved on Feb 3, 2021, from -[https://www.researchgate.net/publication/348170136_THE_PARALLELIO_PIO_CFORTRAN_LIBRARIES_FOR_SCALABLE_HPC_PERFORMANCE]. diff --git a/configure.ac b/configure.ac index 7f8aa1ba2..cc701edfb 100644 --- a/configure.ac +++ b/configure.ac @@ -261,7 +261,7 @@ fi # Do we have netCDF-4? AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include "netcdf_meta.h"], -[[#if !NC_HAS_NC4 +[[#if !NC_HAS_HDF5 && !NC_HAS_NC4 # error #endif] ])], [have_netcdf4=yes], [have_netcdf4=no]) diff --git a/examples/c/darray_no_async.c b/examples/c/darray_no_async.c index 6568f3f3c..07c5b6329 100644 --- a/examples/c/darray_no_async.c +++ b/examples/c/darray_no_async.c @@ -277,7 +277,7 @@ int main(int argc, char* argv[]) /* printf("rank: %d Creating decomposition, elements_per_pe %lld...\n", my_rank, */ /* elements_per_pe); */ if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe, - compdof, &ioid, PIO_REARR_SUBSET, NULL, NULL))) + compdof, &ioid, PIO_REARR_SUBSET, NULL, NULL, NULL))) ERR(ret); /* Write the decomposition file. */ diff --git a/examples/c/example1.c b/examples/c/example1.c index 6b0197624..f3988fd60 100644 --- a/examples/c/example1.c +++ b/examples/c/example1.c @@ -342,7 +342,7 @@ int check_file(int ntasks, char *filename) { if (verbose) printf("rank: %d Creating decomposition...\n", my_rank); if ((ret = PIOc_InitDecomp(iosysid, PIO_INT, NDIM, dim_len, (PIO_Offset)elements_per_pe, - compdof, &ioid, NULL, NULL, NULL))) + compdof, &ioid, NULL, NULL, NULL, NULL))) ERR(ret); free(compdof); diff --git a/examples/c/examplePio.c b/examples/c/examplePio.c index 3ec01748a..fa1b9be30 100644 --- a/examples/c/examplePio.c +++ b/examples/c/examplePio.c @@ -319,7 +319,7 @@ struct examplePioClass* epc_createDecomp( struct examplePioClass* this ) if (this->verbose) printf("rank: %d Creating decomposition...\n", this->myRank); PIOc_InitDecomp(this->pioIoSystem, PIO_INT, 1, this->dimLen, (PIO_Offset)(this->arrIdxPerPe), - this->compdof, &this->iodescNCells, NULL, NULL, NULL); + this->compdof, &this->iodescNCells, NULL, NULL, NULL, NULL); return this; } diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt index 35e188149..4b2196d11 100644 --- a/src/clib/CMakeLists.txt +++ b/src/clib/CMakeLists.txt @@ -143,7 +143,7 @@ endif () target_include_directories (pioc PUBLIC ${PIO_C_EXTRA_INCLUDE_DIRS}) target_link_libraries (pioc - PUBLIC ${PIO_C_EXTRA_LIBRARIES}) + PUBLIC ${PIO_C_EXTRA_LIBRARIES} ${CMAKE_DL_LIBS}) target_compile_options (pioc PRIVATE ${PIO_C_EXTRA_COMPILE_OPTIONS}) target_compile_definitions (pioc diff --git a/src/clib/pio.h b/src/clib/pio.h index b15d61b0b..1c2fc1ae7 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -802,23 +802,44 @@ extern "C" { int PIOc_set_log_level(int level); int PIOc_set_global_log_level(int iosysid, int level); + /** + * @brief Function type for custom partitioning strategies (subset partitioning) + * + * @param ios Pointer to the iosystem description + * @param iodesc Pointer to the IO description structure + * @param color Pointer to store the computed color value + * @param key Pointer to store the computed key value + * @return PIO_NOERR on success, error code otherwise + */ + typedef int (*pio_partition_fn)(int comprank, int iorank, int comptasks, int iotasks, + int *color, int *key); + + /* Decomposition. */ /* Init decomposition with 1-based compmap array. */ int PIOc_InitDecomp_ReadOnly(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, const PIO_Offset *compmap, int *ioidp, const int *rearr, - const PIO_Offset *iostart, const PIO_Offset *iocount); + const PIO_Offset *iostart, const PIO_Offset *iocount, pio_partition_fn partition_fn); int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, const PIO_Offset *compmap, int *ioidp, const int *rearr, - const PIO_Offset *iostart, const PIO_Offset *iocount); + const PIO_Offset *iostart, const PIO_Offset *iocount, pio_partition_fn partition_fn); int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *gdimlen, const long int *start, const long int *count, int *ioidp); /* Init decomposition with 0-based compmap array. */ int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, const PIO_Offset *compmap, int *ioidp, int rearranger, - const PIO_Offset *iostart, const PIO_Offset *iocount); - + const PIO_Offset *iostart, const PIO_Offset *iocount, pio_partition_fn partition_fn); + + int PIOc_InitDecomp_DynamicPartitioner(int iosysid, int basetype, int ndims, + const int *dims, int maplen, + const PIO_Offset *compmap, int *ioidp, + const int *rearranger, + const PIO_Offset *iostart, + const PIO_Offset *iocount, + const char *lib_path, + const char *func_name) ; /* Free resources associated with a decomposition. */ int PIOc_freedecomp(int iosysid, int ioid); diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 620ce4d18..30de3e3e1 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -17,6 +17,8 @@ #include #include #include +#include // used in PIOc_InitDecomp_DynamicPartitioner + #ifdef NC_HAS_PAR_FILTERS #include #include @@ -251,7 +253,7 @@ extern "C" { const PIO_Offset *dest_ioindex); /* Create the MPI communicators needed by the subset rearranger. */ - int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc); + int default_subset_partition(int comprank, int iorank, int comptasks, int iotasks, int *color, int *key); /* Like MPI_Alltoallw(), but with flow control. */ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendtypes, @@ -295,9 +297,10 @@ extern "C" { /* Free a region list. */ void free_region_list(io_region *top); + /* Create a subset rearranger. */ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, const int *gsize, - int ndim, io_desc_t *iodesc); + int ndim, io_desc_t *iodesc, pio_partition_fn partition_fn); /* Create a box rearranger. */ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *compmap, const int *gsize, diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 3302ae7a3..bc4c9a6d3 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -2513,9 +2513,10 @@ int initdecomp_dof_handler(iosystem_desc_t *ios) if (iocount_present) iocountp = iocount; + /* TODO: support for custom partition functions */ /* Call the function. */ PIOc_InitDecomp(iosysid, pio_type, ndims, dims, maplen, compmap, &ioid, rearrangerp, - iostartp, iocountp); + iostartp, iocountp, NULL); PLOG((1, "PIOc_InitDecomp returned")); @@ -2940,7 +2941,7 @@ int set_loglevel_handler(iosystem_desc_t *ios) { #if PIO_ENABLE_LOGGING int iosysid; - int level; + int level=0; int mpierr; #endif diff --git a/src/clib/pio_rearrange.c b/src/clib/pio_rearrange.c index 74627f646..e8c7e4708 100644 --- a/src/clib/pio_rearrange.c +++ b/src/clib/pio_rearrange.c @@ -12,6 +12,7 @@ #define MPI_Type_create_hvector MPI_Type_hvector #endif + /** * Convert a 1-D index into a coordinate value in an arbitrary * dimension space. E.g., for index 4 into a array defined as a[3][2], @@ -1914,252 +1915,208 @@ get_regions(int ndims, const int *gdimlen, int maplen, const PIO_Offset *map, } /** - * Create the MPI communicators needed by the subset rearranger. + * @brief Create MPI communicators for the subset rearranger. * - * The subset rearranger needs a mapping from compute tasks to IO - * task, the only requirement is that each compute task map to one and - * only one IO task. This mapping groups by mpi task id others are - * possible and may be better for certain decompositions + * Creates a mapping from compute tasks to IO tasks where each compute task + * maps to exactly one IO task. The current implementation groups by MPI task ID, + * though other strategies may be more optimal for certain decompositions. * - * The as yet unrealized vision here is that the user would be able to - * supply an alternative subset partitioning function. Requirements of - * this function are that there be exactly one io task per compute - * task group. + * @param[in] ios Pointer to the iosystem description structure + * @param[in,out] iodesc Pointer to the IO description structure + * @return PIO_NOERR on success, error code otherwise * - * @param ios pointer to the iosystem_desc_t struct. - * @param iodesc a pointer to the io_desc_t struct. - * @returns 0 on success, error code otherwise. - * @author Jim Edwards + * @note Future enhancement: Allow user-defined subset partitioning functions + * that maintain the one-to-one IO task per compute task group requirement. */ -int -default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc) +int default_subset_partition(int comprank,int iorank,int comptasks,int iotasks,int *color,int *key) { - int color; - int key; - int mpierr; /* Return value from MPI functions. */ + /* Input validation */ + pioassert(comprank >= -1 && comprank < comptasks, "invalid input", __FILE__, __LINE__); + pioassert(iorank >= -1 && iorank < iotasks, "invalid input", __FILE__, __LINE__); - pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__); - PLOG((1, "default_subset_partition ios->ioproc = %d ios->io_rank = %d " - "ios->comp_rank = %d", ios->ioproc, ios->io_rank, ios->comp_rank)); + PLOG((1, "default_subset_partition iorank = %d " + "comprank = %d", iorank, comprank)); - /* Create a new comm for each subset group with the io task in - rank 0 and only 1 io task per group */ - if (ios->ioproc) - { - key = 0; - color= ios->io_rank; - } - else - { - int taskratio = max(1,ios->num_comptasks / ios->num_iotasks); - key = max(1, ios->comp_rank % taskratio + 1); - color = min(ios->num_iotasks - 1, ios->comp_rank / taskratio); + /* Assign color and key values based on process type */ + if (iorank >=0 ) { + *key = 0; + *color = iorank; + } else { + int taskratio = max(1, comptasks / iotasks); + *key = max(1, comprank % taskratio + 1); + *color = min(iotasks - 1, comprank / taskratio); } - PLOG((3, "key = %d color = %d", key, color)); - - /* Create new communicators. */ - if ((mpierr = MPI_Comm_split(ios->union_comm, color, key, &iodesc->subset_comm))) - return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); + + PLOG((3, "key = %d color = %d", *key, *color)); return PIO_NOERR; } /** - * Create the subset rearranger. - * - * The subset rearranger computes a mapping between IO tasks and - * compute tasks such that each compute task communicates with one and - * only one IO task. + * @brief Create the subset rearranger for parallel I/O operations. * - * This function is called from PIOc_InitDecomp(). + * Establishes a mapping between IO and compute tasks where each compute task + * communicates with exactly one IO task. Supports both default and custom + * partitioning strategies. * - * This function: - *
    - *
  • Calls default_subset_partition() to create subset_comm. - *
  • For IO tasks, allocates iodesc->rcount array (length ntasks). - *
  • Allocates iodesc->scount array (length 1) - *
  • Determins value of iodesc->scount[0], the number of data - * elements on this compute task which are read/written. - *
  • Allocated and inits iodesc->sindex (length iodesc->scount[0]), - * init it to contain indicies to data. - *
  • Pass the reduced maplen (without holes) from each compute task - * to its associated IO task. - *
  • On IO tasks, determine llen. - *
  • Determine whether fill values will be needed. - *
  • Pass iodesc->sindex from each compute task to its associated IO - * task. - *
  • Create shrtmap, which is compmap without the holes. - *
  • Gather shrtmaps from each task into iomap. - *
  • On IO tasks, sort the mapping, this will transpose the data - * into IO order. - *
  • On IO tasks, allocate and init iodesc->rindex and iodesc->rfrom - * (length iodesc->llen). - *
  • On IO tasks, handle fill values, if needed. - *
  • On IO tasks, scatter values of srcindex to subset communicator. - *
  • On IO tasks, call get_regions() and distribute the max - * maxregions to all tasks in IO communicator. - *
  • On IO tasks, call compute_maxIObuffersize(). - *
+ * @param[in] ios Pointer to the iosystem description + * @param[in] maplen Length of the decomposition map + * @param[in] compmap 1-based array of offsets into file array (0 indicates no transfer) + * @param[in] gdimlen Array containing global dimension sizes + * @param[in] ndims Number of dimensions + * @param[in,out] iodesc IO description structure to be initialized + * @param[in] partition_fn Optional custom partitioning function (NULL for default) * - * @param ios pointer to the iosystem_desc_t struct. - * @param maplen the length of the map. - * @param compmap a 1 based array of offsets into the array record on - * file. A 0 in this array indicates a value which should not be - * transfered. - * @param gdimlen an array length ndims with the sizes of the global - * dimensions. - * @param ndims the number of dimensions. - * @param iodesc a pointer to the io_desc_t struct. - * @returns 0 on success, error code otherwise. - * @author Jim Edwards + * @return PIO_NOERR on success, error code otherwise */ -int -subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, - const int *gdimlen, int ndims, io_desc_t *iodesc) +int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, + const int *gdimlen, int ndims, io_desc_t *iodesc, + pio_partition_fn partition_fn) { - int i, j; - PIO_Offset *iomap = NULL; - mapsort *map = NULL; - PIO_Offset totalgridsize; - PIO_Offset *srcindex = NULL; - PIO_Offset *myfillgrid = NULL; - int maxregions; - int rank, ntasks; - int rcnt = 0; - int mpierr; /* Return call from MPI function calls. */ - int ret; - - /* Check inputs. */ + /* Validate input parameters */ pioassert(ios && maplen >= 0 && compmap && gdimlen && ndims >= 0 && iodesc, - "invalid input", __FILE__, __LINE__); + "Invalid input parameters", __FILE__, __LINE__); PLOG((2, "subset_rearrange_create maplen = %d ndims = %d", maplen, ndims)); - /* subset partitions each have exactly 1 io task which is task 0 - * of that subset_comm */ - /* TODO: introduce a mechanism for users to define partitions */ - if ((ret = default_subset_partition(ios, iodesc))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); + int ret = PIO_NOERR; + int mpierr = PIO_NOERR; + int color, key; + + /* Apply partitioning strategy or default if no strategy provided */ + if (partition_fn == NULL) { + partition_fn = default_subset_partition; + } + + if ((ret = partition_fn(ios->comp_rank, ios->io_rank, ios->num_comptasks, ios->num_iotasks, &color, &key)) != PIO_NOERR) { + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + } + + /* Create communicator using computed color and key */ + if ((mpierr = MPI_Comm_split(ios->union_comm, color, key, + &iodesc->subset_comm))) { + return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); + } + + /* Set rearranger type */ iodesc->rearranger = PIO_REARR_SUBSET; - /* Get size of this subset communicator and rank of this task in it. */ - if ((mpierr = MPI_Comm_rank(iodesc->subset_comm, &rank))) + /* Rest of the existing subset_rearrange_create implementation... */ + + int rank = 0; + int ntasks = 0; + int i, j; + PIO_Offset *iomap = NULL; + mapsort *map = NULL; + PIO_Offset totalgridsize = 1; + PIO_Offset *srcindex = NULL; + PIO_Offset *myfillgrid = NULL; + int maxregions; + + /* Get communicator size and rank */ + if ((mpierr = MPI_Comm_rank(iodesc->subset_comm, &rank))) { return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Comm_size(iodesc->subset_comm, &ntasks))) + } + if ((mpierr = MPI_Comm_size(iodesc->subset_comm, &ntasks))) { return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + } - /* Check rank for correctness. */ - if (ios->ioproc) - pioassert(rank == 0, "Bad io rank in subset create", __FILE__, __LINE__); - else + /* Validate rank assignments */ + if (ios->ioproc) { + pioassert(rank == 0, "Bad IO rank in subset create", __FILE__, __LINE__); + } else { pioassert(rank > 0 && rank < ntasks, "Bad comp rank in subset create", __FILE__, __LINE__); + } - /* Remember the maplen for this computation task. */ + /* Calculate total grid size */ + for (int i = 0; i < ndims; i++) { + totalgridsize *= gdimlen[i]; + } + + /* Initialize counts and memory for IO processes */ iodesc->ndof = maplen; - - if (ios->ioproc) - { - /* Allocate space to hold count of data to be received in pio_swapm(). */ - if (!(iodesc->rcount = malloc(ntasks * sizeof(int)))) + if (ios->ioproc) { + if (!(iodesc->rcount = calloc(ntasks, sizeof(int)))) { return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - rcnt = 1; - - if(ios->async) + } + if (ios->async) { iodesc->ndof = 0; - + } } - /* Allocate space to hold count of data to be sent in pio_swapm(). */ - if (!(iodesc->scount = malloc(sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - + /* Allocate send counts array */ + if (!(iodesc->scount = calloc(1, sizeof(int)))) { + ret = pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + goto cleanup; + } iodesc->scount[0] = 0; - /* Find the total size of the global data array. */ - totalgridsize = 1; - for (i = 0; i < ndims; i++) - totalgridsize *= gdimlen[i]; - - /* Determine scount[0], the number of data elements in the - * computation task that are to be written, by looking at - * compmap. */ -// int compmax = -1; -// int compmin = 5000; - for (i = 0; i < iodesc->ndof; i++) - { - // This is allowed in some cases - // pioassert(compmap[i]>=-1 && compmap[i]<=totalgridsize, "Compmap value out of bounds", - // __FILE__,__LINE__); - if (compmap[i] > 0) + /* Count valid elements in compmap */ + for (int i = 0; i < iodesc->ndof; i++) { + if (compmap[i] > 0) { (iodesc->scount[0])++; -// if (compmap[i] > compmax) -// compmax = compmap[i]; -// if (compmap[i] > 0 && compmap[i]scount[0] > 0) - if (!(iodesc->sindex = calloc(iodesc->scount[0], sizeof(PIO_Offset)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - j = 0; - for (i = 0; i < iodesc->ndof; i++){ - PLOG((4,"compmap[%d] = %d ",i, compmap[i])); - if (compmap[i] > 0){ - PLOG((4,"sindex[%d] = %d ",j, i)); - iodesc->sindex[j++] = i; + /* Allocate and initialize sindex array for valid elements */ + if (iodesc->scount[0] > 0) { + if (!(iodesc->sindex = calloc(iodesc->scount[0], sizeof(PIO_Offset)))) { + ret = pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + goto cleanup; + } + + int j = 0; + for (int i = 0; i < iodesc->ndof; i++) { + if (compmap[i] > 0) { + iodesc->sindex[j++] = i; + } } } PLOG((2,"At line %d scount[0]=%d",__LINE__,iodesc->scount[0])); - /* Pass the reduced maplen (without holes) from each compute task - * to its associated IO task. */ + /* Gather send counts to root of subset communicator */ + int rcnt = (ios->ioproc) ? 1 : 0; if ((mpierr = MPI_Gather(iodesc->scount, 1, MPI_INT, iodesc->rcount, rcnt, - MPI_INT, 0, iodesc->subset_comm))) - return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); - + MPI_INT, 0, iodesc->subset_comm))) { + ret = check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); + goto cleanup; + } + /* Initialize displacement arrays for gather operations */ + int *rdispls = calloc(sizeof(int), ntasks); + int *recvcounts = calloc(sizeof(int), ntasks); iodesc->llen = 0; - int rdispls[ntasks]; - int recvcounts[ntasks]; - - /* On IO tasks determine llen. */ - if (ios->ioproc) - { - for (i = 0; i < ntasks; i++) - { + if (ios->ioproc) { + /* Calculate displacements and total length for IO tasks */ + for (int i = 0; i < ntasks; i++) { iodesc->llen += iodesc->rcount[i]; - rdispls[i] = 0; + rdispls[i] = (i > 0) ? rdispls[i-1] + iodesc->rcount[i-1] : 0; recvcounts[i] = iodesc->rcount[i]; - if (i > 0) - rdispls[i] = rdispls[i - 1] + iodesc->rcount[i - 1]; } - if (iodesc->llen > 0) - { - if (!(srcindex = calloc(iodesc->llen, sizeof(PIO_Offset)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + /* Allocate source index array if needed */ + if (iodesc->llen > 0) { + if (!(srcindex = calloc(iodesc->llen, sizeof(PIO_Offset)))) { + ret = pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + goto cleanup; + } } - } - else - { - for (i = 0; i < ntasks; i++) - { + } else { + /* Initialize arrays for non-IO tasks */ + for (int i = 0; i < ntasks; i++) { recvcounts[i] = 0; rdispls[i] = 0; } } -// PLOG((2,"At line %d rdispls[%d]=%d rcount=%d",__LINE__,1,rdispls[0], iodesc->rcount[0])); - /* Determine whether fill values will be needed. */ - if(! iodesc->readonly) - if ((ret = determine_fill(ios, iodesc, gdimlen, compmap))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); + + /* Handle fill values if needed */ + if (!iodesc->readonly) { + if ((ret = determine_fill(ios, iodesc, gdimlen, compmap))) { + goto cleanup; + } + } /* Pass the sindex from each compute task to its associated IO task. */ if ((mpierr = MPI_Gatherv(iodesc->sindex, iodesc->scount[0], PIO_OFFSET, @@ -2237,7 +2194,7 @@ subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); } - int cnt[ntasks]; + int *cnt = calloc(sizeof(int), ntasks); for (i = 0; i < ntasks; i++) { cnt[i] = rdispls[i]; @@ -2255,34 +2212,35 @@ subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, // PIO_Offset previomap[ntasks]; // for (i = 0; i < ntasks; i++) // previomap[i] = -1; - if(iodesc->llen > 0){ - mapsort *mptr = &map[0]; - iomap[0] = mptr->iomap; - soffset = mptr->soffset; - int increment; - - for (i = 0, rllen=0; i < iodesc->llen; i++) - { - mptr = &map[i]; - increment = 0; - iodesc->rfrom[i] = mptr->rfrom; + if(iodesc->llen > 0) + { + mapsort *mptr = &map[0]; + iomap[0] = mptr->iomap; + soffset = mptr->soffset; + int increment; + + for (i = 0, rllen=0; i < iodesc->llen; i++) + { + mptr = &map[i]; + increment = 0; + iodesc->rfrom[i] = mptr->rfrom; // if(mptr->iomap > previomap[mptr->rfrom]) // { - if(i==iodesc->llen-1 || mptr->iomap < map[i+1].iomap){ - iomap[rllen] = mptr->iomap; - increment = 1; - } - soffset = mptr->soffset; - + if(i==iodesc->llen-1 || mptr->iomap < map[i+1].iomap){ + iomap[rllen] = mptr->iomap; + increment = 1; + } + soffset = mptr->soffset; + // } // previomap[mptr->rfrom]=iomap[rllen]; - srcindex[(cnt[mptr->rfrom])++] = soffset; - iodesc->rindex[i] = rllen; - rllen = rllen + increment; - iodesc->rllen = rllen; - } + srcindex[(cnt[mptr->rfrom])++] = soffset; + iodesc->rindex[i] = rllen; + rllen = rllen + increment; + iodesc->rllen = rllen; + } } - + free(cnt); /* Handle fill values if needed. */ PLOG((3, "ios->ioproc %d iodesc->needsfill %d iodesc->rllen %d", ios->ioproc, iodesc->needsfill, iodesc->rllen)); if (ios->ioproc && iodesc->needsfill) @@ -2300,7 +2258,7 @@ subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, thisgridmax[0] = thisgridsize[0]; int xtra = totalgridsize - thisgridsize[0] * ios->num_iotasks; - PLOG((4, "xtra %d", xtra)); + PLOG((3, "xtra %d", xtra)); for (nio = 0; nio < ios->num_iotasks; nio++) { @@ -2313,7 +2271,7 @@ subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, thisgridsize[nio]++; thisgridmin[nio] = thisgridmax[nio - 1] + 1; thisgridmax[nio] = thisgridmin[nio] + thisgridsize[nio] - 1; - PLOG((4, "nio %d thisgridsize[nio] %d thisgridmin[nio] %d thisgridmax[nio] %d", + PLOG((3, "nio %d thisgridsize[nio] %d thisgridmin[nio] %d thisgridmax[nio] %d", nio, thisgridsize[nio], thisgridmin[nio], thisgridmax[nio])); } for (int i = 0; i < iodesc->rllen; i++) @@ -2435,7 +2393,8 @@ subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, (void *)iodesc->sindex, iodesc->scount[0], PIO_OFFSET, 0, iodesc->subset_comm))) return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); - + free(recvcounts); + free(rdispls); if (ios->ioproc) { iodesc->maxregions = 0; @@ -2466,11 +2425,19 @@ subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, iodesc->nrecvs = ntasks; } -// PLOG((2, "At line %d sindex[20] = %d",__LINE__,iodesc->sindex[20])); - return PIO_NOERR; +cleanup: + /* Free allocated memory in case of error */ + if (ret != PIO_NOERR) { + if (ios->ioproc) { + free(iodesc->rcount); + free(srcindex); + } + free(iodesc->scount); + free(iodesc->sindex); + } + return ret; } - /** * Performance tuning rearranger. * diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 1be592345..e36cf5f9f 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -492,6 +492,8 @@ compare( const void* a, const void* b) * decompositions for the SUBSET rearranger. Ignored if block * rearranger is used. If NULL and SUBSET rearranger is used, the * iostarts are generated. + * @param partition_fn Optional function for custom partitioning (NULL for default) + * * @returns 0 on success, error code otherwise * @ingroup PIO_initdecomp_c * @author Jim Edwards, Ed Hartnett @@ -499,7 +501,8 @@ compare( const void* a, const void* b) int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, const PIO_Offset *compmap, int *ioidp, const int *rearranger, - const PIO_Offset *iostart, const PIO_Offset *iocount) + const PIO_Offset *iostart, const PIO_Offset *iocount, + pio_partition_fn partition_fn) { iosystem_desc_t *ios; /* Pointer to io system information. */ io_desc_t *iodesc; /* The IO description. */ @@ -674,7 +677,7 @@ PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int ma PLOG((2, "creating subset rearranger iodesc->num_aiotasks = %d readonly = %d", iodesc->num_aiotasks, iodesc->readonly)); if ((ierr = subset_rearrange_create(ios, maplen, (PIO_Offset *)iodesc->map, gdimlen, - ndims, iodesc))) + ndims, iodesc, partition_fn))) return pio_err(ios, NULL, ierr, __FILE__, __LINE__); } @@ -765,6 +768,48 @@ PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int ma return PIO_NOERR; } + +int PIOc_InitDecomp_DynamicPartitioner(int iosysid, int basetype, int ndims, + const int *dims, int maplen, + const PIO_Offset *compmap, int *ioidp, + const int *rearranger, + const PIO_Offset *iostart, + const PIO_Offset *iocount, + const char *lib_path, + const char *func_name) { + + void *handle; + pio_partition_fn partition_fn; + + // Open the library + + handle = dlopen(lib_path, RTLD_LAZY); + if (!handle) { + fprintf(stderr, "Error loading library >%s<: %s\n", lib_path, dlerror()); + return -1; + } + + // Get the function + partition_fn = (pio_partition_fn)dlsym(handle, func_name); + if (!partition_fn) { + fprintf(stderr, "Error getting function %s: %s\n", func_name, dlerror()); + dlclose(handle); + return -2; + } + + // Call the original PIOc_InitDecomp with our loaded function + int ret = PIOc_InitDecomp(iosysid, basetype, ndims, dims, maplen, + compmap, ioidp, rearranger, iostart, iocount, + partition_fn); + + // Close the library handle + dlclose(handle); + return ret; +} + + + + /** * Initialize the decomposition used with distributed arrays. The * decomposition describes how the data will be distributed between @@ -814,7 +859,8 @@ PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int ma int PIOc_InitDecomp_ReadOnly(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, const PIO_Offset *compmap, int *ioidp, const int *rearranger, - const PIO_Offset *iostart, const PIO_Offset *iocount) + const PIO_Offset *iostart, const PIO_Offset *iocount, + pio_partition_fn partition_fn) { iosystem_desc_t *ios; /* Pointer to io system information. */ io_desc_t *iodesc; /* The IO description. */ @@ -993,7 +1039,7 @@ PIOc_InitDecomp_ReadOnly(int iosysid, int pio_type, int ndims, const int *gdimle PLOG((2, "creating subset rearranger iodesc->num_aiotasks = %d readonly = %d", iodesc->num_aiotasks, iodesc->readonly)); if ((ierr = subset_rearrange_create(ios, maplen, (PIO_Offset *)iodesc->map, gdimlen, - ndims, iodesc))) + ndims, iodesc, partition_fn))) return pio_err(ios, NULL, ierr, __FILE__, __LINE__); } else /* box rearranger */ @@ -1113,7 +1159,8 @@ PIOc_InitDecomp_ReadOnly(int iosysid, int pio_type, int ndims, const int *gdimle int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, const PIO_Offset *compmap, int *ioidp, int rearranger, - const PIO_Offset *iostart, const PIO_Offset *iocount) + const PIO_Offset *iostart, const PIO_Offset *iocount, + pio_partition_fn partition_fn) { PIO_Offset *compmap_1_based; int *rearrangerp = NULL; @@ -1139,7 +1186,7 @@ PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int m /* Call the legacy version of the function. */ ret = PIOc_InitDecomp(iosysid, pio_type, ndims, gdimlen, maplen, compmap_1_based, - ioidp, rearrangerp, iostart, iocount); + ioidp, rearrangerp, iostart, iocount, partition_fn); free(compmap_1_based); @@ -1220,7 +1267,7 @@ PIOc_InitDecomp_bc(int iosysid, int pio_type, int ndims, const int *gdimlen, } return PIOc_InitDecomp(iosysid, pio_type, ndims, gdimlen, maplen, compmap, ioidp, - &rearr, NULL, NULL); + &rearr, NULL, NULL, NULL); } /** diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 7c1c349e4..685c14c0f 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -1439,7 +1439,7 @@ PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm comm /* Initialize the decomposition. */ ret = PIOc_InitDecomp(iosysid, pio_type, ndims, global_dimlen, task_maplen[my_rank], - compmap, ioidp, NULL, NULL, NULL); + compmap, ioidp, NULL, NULL, NULL, NULL); free(compmap); } @@ -3126,7 +3126,7 @@ iotype_is_valid(int iotype) /* Some builds include netCDF-4. */ /* as of netcdf 4.9.3 NC_HAS_NC4 is no longer defined */ -#if NC_HAS_NC4 || (NC_VERSION_PATCH > 2) +#if NC_HAS_NC4 || NC_HAS_HDF5 if (iotype == PIO_IOTYPE_NETCDF4C || iotype == PIO_IOTYPE_NETCDF4P) ret++; #endif /* _NETCDF4 */ diff --git a/src/flib/piolib_mod.F90 b/src/flib/piolib_mod.F90 index 265a0e102..610731622 100644 --- a/src/flib/piolib_mod.F90 +++ b/src/flib/piolib_mod.F90 @@ -854,7 +854,8 @@ subroutine PIO_initdecomp_dof_i4(iosystem, basepiotype, dims, compdof, iodesc, r end subroutine PIO_initdecomp_dof_i4 - subroutine PIO_initdecomp_internal(iosystem,basepiotype,dims,maplen, compdof, iodesc, rearr, iostart, iocount) + subroutine PIO_initdecomp_internal(iosystem,basepiotype,dims,maplen, compdof, iodesc, rearr, & + iostart, iocount, lib_path, func_name) type (iosystem_desc_t), intent(in) :: iosystem integer(i4), intent(in) :: basepiotype integer(i4), intent(in) :: dims(:) @@ -862,16 +863,17 @@ subroutine PIO_initdecomp_internal(iosystem,basepiotype,dims,maplen, compdof, io integer (PIO_OFFSET_KIND), intent(in) :: compdof(maplen) ! global degrees of freedom for computational decomposition integer, optional, target :: rearr integer (PIO_OFFSET_KIND), optional :: iostart(:), iocount(:) + character(len=*), optional :: lib_path, func_name type (io_desc_t), intent(inout) :: iodesc integer(c_int) :: ndims integer(c_int), dimension(:), allocatable, target :: cdims integer(PIO_OFFSET_KIND), dimension(:), allocatable, target :: cstart, ccount - type(C_PTR) :: crearr + interface integer(C_INT) function PIOc_InitDecomp(iosysid,basetype,ndims,dims, & - maplen, compmap, ioidp, rearr, iostart, iocount) & + maplen, compmap, ioidp, rearr, iostart, iocount, partition_fn) & bind(C,name="PIOc_InitDecomp") use iso_c_binding integer(C_INT), value :: iosysid @@ -884,7 +886,26 @@ integer(C_INT) function PIOc_InitDecomp(iosysid,basetype,ndims,dims, & type(C_PTR), value :: rearr type(C_PTR), value :: iostart type(C_PTR), value :: iocount + type(C_PTR), value :: partition_fn end function PIOc_InitDecomp + + integer(C_INT) function PIOc_InitDecomp_DynamicPartitioner(iosysid,basetype,ndims,dims, & + maplen, compmap, ioidp, rearranger, iostart, iocount, & + lib_path, func_name) bind(C, name="PIOc_InitDecomp_DynamicPartitioner") + use iso_c_binding + integer(C_INT), value :: iosysid + integer(C_INT), value :: basetype + integer(C_INT), value :: ndims + integer(C_INT) :: dims(*) + integer(C_INT), value :: maplen + integer(C_SIZE_T) :: compmap(*) + integer(C_INT) :: ioidp + type(C_PTR), value :: rearranger + type(C_PTR), value :: iostart + type(C_PTR), value :: iocount + character(kind=c_char), intent(in) :: lib_path(*), func_name(*) + end function PIOc_InitDecomp_DynamicPartitioner + end interface integer :: ierr,i @@ -899,22 +920,39 @@ end function PIOc_InitDecomp else crearr = C_NULL_PTR endif - - if(present(iostart) .and. present(iocount)) then - allocate(cstart(ndims), ccount(ndims)) - do i=1,ndims - cstart(i) = iostart(ndims-i+1)-1 - ccount(i) = iocount(ndims-i+1) - end do - - ierr = PIOc_InitDecomp(iosystem%iosysid, basepiotype, ndims, cdims, & - maplen, compdof, iodesc%ioid, crearr, C_LOC(cstart), C_LOC(ccount)) - deallocate(cstart, ccount) - else - ierr = PIOc_InitDecomp(iosystem%iosysid, basepiotype, ndims, cdims, & - maplen, compdof, iodesc%ioid, crearr, C_NULL_PTR, C_NULL_PTR) - end if - + if(present(lib_path) .and. present(func_name) .and. len_trim(lib_path) > 0 .and. len_trim(func_name) > 0) then + if(present(iostart) .and. present(iocount)) then + allocate(cstart(ndims), ccount(ndims)) + do i=1,ndims + cstart(i) = iostart(ndims-i+1)-1 + ccount(i) = iocount(ndims-i+1) + end do + + ierr = PIOc_InitDecomp_DynamicPartitioner(iosystem%iosysid, basepiotype, ndims, cdims, & + maplen, compdof, iodesc%ioid, crearr, C_LOC(cstart), C_LOC(ccount), & + trim(lib_path)//C_NULL_CHAR, trim(func_name)//C_NULL_CHAR) + deallocate(cstart, ccount) + else + ierr = PIOc_InitDecomp_DynamicPartitioner(iosystem%iosysid, basepiotype, ndims, cdims, & + maplen, compdof, iodesc%ioid, crearr, C_NULL_PTR, C_NULL_PTR, & + trim(lib_path)//C_NULL_CHAR, trim(func_name)//C_NULL_CHAR) + end if + else + if(present(iostart) .and. present(iocount)) then + allocate(cstart(ndims), ccount(ndims)) + do i=1,ndims + cstart(i) = iostart(ndims-i+1)-1 + ccount(i) = iocount(ndims-i+1) + end do + + ierr = PIOc_InitDecomp(iosystem%iosysid, basepiotype, ndims, cdims, & + maplen, compdof, iodesc%ioid, crearr, C_LOC(cstart), C_LOC(ccount), C_NULL_PTR) + deallocate(cstart, ccount) + else + ierr = PIOc_InitDecomp(iosystem%iosysid, basepiotype, ndims, cdims, & + maplen, compdof, iodesc%ioid, crearr, C_NULL_PTR, C_NULL_PTR, C_NULL_PTR) + end if + endif deallocate(cdims) end subroutine PIO_initdecomp_internal @@ -967,7 +1005,7 @@ subroutine PIO_initdecomp_readonly(iosystem,basepiotype,dims, compdof, iodesc, r type(C_PTR) :: crearr interface integer(C_INT) function PIOc_InitDecomp_ReadOnly(iosysid,basetype,ndims,dims, & - maplen, compmap, ioidp, rearr, iostart, iocount) & + maplen, compmap, ioidp, rearr, iostart, iocount, partition_fn) & bind(C,name="PIOc_InitDecomp_ReadOnly") use iso_c_binding integer(C_INT), value :: iosysid @@ -980,6 +1018,7 @@ integer(C_INT) function PIOc_InitDecomp_ReadOnly(iosysid,basetype,ndims,dims, & type(C_PTR), value :: rearr type(C_PTR), value :: iostart type(C_PTR), value :: iocount + type(C_PTR), value :: partition_fn end function PIOc_InitDecomp_ReadOnly end interface integer :: ierr,i @@ -1004,11 +1043,11 @@ end function PIOc_InitDecomp_ReadOnly end do ierr = PIOc_InitDecomp_ReadOnly(iosystem%iosysid, basepiotype, ndims, cdims, & - maplen, compdof, iodesc%ioid, crearr, C_LOC(cstart), C_LOC(ccount)) + maplen, compdof, iodesc%ioid, crearr, C_LOC(cstart), C_LOC(ccount), C_NULL_PTR) deallocate(cstart, ccount) else ierr = PIOc_InitDecomp_ReadOnly(iosystem%iosysid, basepiotype, ndims, cdims, & - maplen, compdof, iodesc%ioid, crearr, C_NULL_PTR, C_NULL_PTR) + maplen, compdof, iodesc%ioid, crearr, C_NULL_PTR, C_NULL_PTR, C_NULL_PTR) end if deallocate(cdims) @@ -1021,7 +1060,7 @@ end subroutine PIO_initdecomp_readonly !! I8 version of PIO_initdecomp_dof_i4. !! @author Jim Edwards subroutine PIO_initdecomp_dof_i8(iosystem, basepiotype, dims, compdof, & - iodesc, rearr, iostart, iocount) + iodesc, rearr, iostart, iocount, lib_path, func_name) type (iosystem_desc_t), intent(in) :: iosystem integer(i4), intent(in) :: basepiotype integer(i4), intent(in) :: dims(:) @@ -1030,7 +1069,7 @@ subroutine PIO_initdecomp_dof_i8(iosystem, basepiotype, dims, compdof, & integer (PIO_OFFSET_KIND), optional :: iostart(:), iocount(:) type (io_desc_t), intent(inout) :: iodesc integer :: maplen - + character(len=*), optional, intent(in) :: lib_path, func_name #ifdef TIMING call t_startf("PIO:initdecomp_dof") #endif @@ -1038,7 +1077,7 @@ subroutine PIO_initdecomp_dof_i8(iosystem, basepiotype, dims, compdof, & maplen = size(compdof) call PIO_initdecomp_internal(iosystem, basepiotype, dims, maplen, & - compdof, iodesc, rearr, iostart, iocount) + compdof, iodesc, rearr, iostart, iocount, lib_path, func_name) #ifdef TIMING call t_stopf("PIO:initdecomp_dof") diff --git a/tests/cperf/piodecomptest.c b/tests/cperf/piodecomptest.c index 5efedb026..25cacfd01 100644 --- a/tests/cperf/piodecomptest.c +++ b/tests/cperf/piodecomptest.c @@ -141,7 +141,7 @@ int test_write_darray(int iosys, const char decomp_file[], int rank, const char /* allocated in pioc_read_nc_decomp_int */ free(full_map); ierr = PIOc_InitDecomp(iosys, PIO_DOUBLE, ndims, global_dimlen, maplen[rank], - dofmap, &ioid, NULL, NULL, NULL); + dofmap, &ioid, NULL, NULL, NULL, NULL); free(global_dimlen); double dsum=0; @@ -235,7 +235,7 @@ int test_read_darray(int iosys,const char decomp_file[], int rank, const char my free(full_map); // PIOc_set_log_level(3); ierr = PIOc_InitDecomp(iosys, pio_type, ndims, global_dimlen, maplen[rank], - dofmap, &ioid, NULL, NULL, NULL); + dofmap, &ioid, NULL, NULL, NULL, NULL); free(dofmap); free(global_dimlen); switch(pio_type) diff --git a/tests/cunit/pio_tests.h b/tests/cunit/pio_tests.h index db4fb599f..a4a2a3ddd 100644 --- a/tests/cunit/pio_tests.h +++ b/tests/cunit/pio_tests.h @@ -122,5 +122,5 @@ int run_test_main(int argc, char **argv, int min_ntasks, int max_ntasks, /* Create a 2D decomposition used in some tests. */ int create_decomposition_2d(int ntasks, int my_rank, int iosysid, int *dim_len_2d, int *ioid, - int pio_type); + int pio_type, int rearranger); #endif /* _PIO_TESTS_H */ diff --git a/tests/cunit/test_async_multicomp.c b/tests/cunit/test_async_multicomp.c index ceb7daa1e..6a96360cc 100644 --- a/tests/cunit/test_async_multicomp.c +++ b/tests/cunit/test_async_multicomp.c @@ -90,7 +90,7 @@ int main(int argc, char **argv) int ioid = 0; if ((ret = create_decomposition_2d(NUM_COMP_PROCS, my_rank, iosysid[my_comp_idx], dim_len_2d, - &ioid, PIO_SHORT))) + &ioid, PIO_SHORT, rearranger[rearr]))) AERR2(ret, iosysid[my_comp_idx]); #ifndef USE_MPE /* For some reason MPE logging breaks this test! */ diff --git a/tests/cunit/test_async_perf.c b/tests/cunit/test_async_perf.c index 744b63ae2..758252e0b 100644 --- a/tests/cunit/test_async_perf.c +++ b/tests/cunit/test_async_perf.c @@ -113,7 +113,7 @@ create_decomposition_3d(int ntasks, int my_rank, int rearr, int iosysid, int *io if(rearr==PIO_REARR_SUBSET) PIOc_set_global_log_level(iosysid, 2); /* Create the PIO decomposition for this test. */ if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3, dim_len_3d, my_elem_per_pe, - compdof, ioid, rearr, NULL, NULL))) + compdof, ioid, rearr, NULL, NULL, NULL))) AERR(ret); /* Free the mapping. */ diff --git a/tests/cunit/test_common.c b/tests/cunit/test_common.c index cc9f15c31..188b17bb7 100644 --- a/tests/cunit/test_common.c +++ b/tests/cunit/test_common.c @@ -995,6 +995,24 @@ check_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *nci return ret; } +/* round_robin_partition is an example of a user provided subset partitioning method +**/ + +int round_robin_partition(int comprank, int iorank, int comptasks, int iotasks, int *color, int *key) +{ + if(!color || !key || iorank < -1 || iorank >= iotasks || comprank < -1 || comprank >= comptasks) { + return PIO_EINVAL; + } + if (iorank > -1){ + *key = 0; + *color = iorank; + }else{ + *key = 1; + *color = comprank % iotasks; + } + return PIO_NOERR; +} + /* Create the decomposition to divide the 3-dimensional sample data * between tasks. For the purposes of decomposition we are only * concerned with 2 dimensions - we ignore the unlimited dimension. @@ -1008,12 +1026,13 @@ check_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *nci * @returns 0 for success, error code otherwise. **/ int create_decomposition_2d(int ntasks, int my_rank, int iosysid, int *dim_len_2d, - int *ioid, int pio_type) + int *ioid, int pio_type, int rearranger) { PIO_Offset elements_per_pe; /* Array elements per processing unit. */ PIO_Offset *compdof; /* The decomposition mapping. */ int ret; - + pio_partition_fn partition_fn; + /* How many data elements per task? In this example we will end up * with 4. */ elements_per_pe = dim_len_2d[0] * dim_len_2d[1] / ntasks; @@ -1026,9 +1045,17 @@ int create_decomposition_2d(int ntasks, int my_rank, int iosysid, int *dim_len_2 for (int i = 0; i < elements_per_pe; i++) compdof[i] = my_rank * elements_per_pe + i + 1; + if(rearranger == PIO_REARR_SUBSET){ + partition_fn = default_subset_partition; + }else if(rearranger == -PIO_REARR_SUBSET){ + rearranger = PIO_REARR_SUBSET; + partition_fn = round_robin_partition; + } + + /* Create the PIO decomposition for this test. */ if ((ret = PIOc_InitDecomp(iosysid, pio_type, NDIM2, dim_len_2d, elements_per_pe, - compdof, ioid, NULL, NULL, NULL))) + compdof, ioid, &rearranger, NULL, NULL, partition_fn))) ERR(ret); diff --git a/tests/cunit/test_darray.c b/tests/cunit/test_darray.c index ee16c525d..c1f351dd0 100644 --- a/tests/cunit/test_darray.c +++ b/tests/cunit/test_darray.c @@ -337,7 +337,7 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank * @returns 0 for success, error code otherwise. */ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, - MPI_Comm test_comm) + MPI_Comm test_comm, int rearranger) { #define NUM_TYPES_TO_TEST 3 int ioid; @@ -354,7 +354,7 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, /* Decompose the data over the tasks. */ if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, - &ioid, pio_type[t]))) + &ioid, pio_type[t], rearranger))) return ret; /* Run a simple darray test. */ @@ -372,8 +372,8 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, /* Run tests for darray functions. */ int main(int argc, char **argv) { -#define NUM_REARRANGERS_TO_TEST 2 - int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; +#define NUM_REARRANGERS_TO_TEST 3 + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET, -PIO_REARR_SUBSET}; int my_rank; int ntasks; int num_flavors; /* Number of PIO netCDF flavors in this build. */ @@ -406,11 +406,11 @@ int main(int argc, char **argv) /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, - ioproc_start, rearranger[r], &iosysid))) + ioproc_start, abs(rearranger[r]), &iosysid))) return ret; /* Run tests. */ - if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm))) + if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm, rearranger[r]))) return ret; /* Finalize PIO system. */ diff --git a/tests/cunit/test_darray_1d.c b/tests/cunit/test_darray_1d.c index d725bfb8d..fbc1f0040 100644 --- a/tests/cunit/test_darray_1d.c +++ b/tests/cunit/test_darray_1d.c @@ -75,7 +75,7 @@ int create_decomposition_1d(int ntasks, int my_rank, int iosysid, int pio_type, /* Create the PIO decomposition for this test. */ if ((ret = PIOc_InitDecomp(iosysid, pio_type, NDIM, dim_len_1d, elements_per_pe, - compdof, ioid, NULL, NULL, NULL))) + compdof, ioid, NULL, NULL, NULL, NULL))) ERR(ret); return 0; diff --git a/tests/cunit/test_darray_2sync.c b/tests/cunit/test_darray_2sync.c index c09316510..ec755ec54 100644 --- a/tests/cunit/test_darray_2sync.c +++ b/tests/cunit/test_darray_2sync.c @@ -217,7 +217,7 @@ int darray_fill_test(int iosysid, int my_rank, int num_iotypes, int *iotype, /* Initialize the decomposition. Only the subset * decomposition uses the fill value. */ if ((ret = PIOc_init_decomp(iosysid, test_type[t], NDIM1, &gdimlen, elements_per_pe, - compdof, &ioid, PIO_REARR_BOX, NULL, NULL))) + compdof, &ioid, PIO_REARR_BOX, NULL, NULL, NULL))) AERR(ret); /* Set the record number for the unlimited dimension. */ @@ -424,7 +424,7 @@ int darray_simple_test(int iosysid, int my_rank, int num_iotypes, int *iotype, /* Initialize the decomposition. */ if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM1, &gdimlen, elements_per_pe, - compdof, &ioid, PIO_REARR_BOX, NULL, NULL))) + compdof, &ioid, PIO_REARR_BOX, NULL, NULL, NULL))) AERR(ret); /* Set the record number for the unlimited dimension. */ diff --git a/tests/cunit/test_darray_3d.c b/tests/cunit/test_darray_3d.c index 476dffbb4..51c651f64 100644 --- a/tests/cunit/test_darray_3d.c +++ b/tests/cunit/test_darray_3d.c @@ -93,7 +93,7 @@ int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid) /* Create the PIO decomposition for this test. */ if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3, dim_len_3d, elements_per_pe, - compdof, ioid, 0, NULL, NULL))) + compdof, ioid, 0, NULL, NULL, NULL))) ERR(ret); /* Free the mapping. */ diff --git a/tests/cunit/test_darray_append.c b/tests/cunit/test_darray_append.c index d2e8b2113..94c826d5a 100644 --- a/tests/cunit/test_darray_append.c +++ b/tests/cunit/test_darray_append.c @@ -313,7 +313,7 @@ int test_darray_append(int iosysid, int ioid, int num_flavors, int *flavor, int * @returns 0 for success, error code otherwise. */ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, - MPI_Comm test_comm) + MPI_Comm test_comm, int rearranger) { #define NUM_TYPES_TO_TEST 3 int ioid; @@ -330,7 +330,7 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, /* Decompose the data over the tasks. */ if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, - &ioid, pio_type[t]))) + &ioid, pio_type[t], rearranger))) return ret; /* Run a simple darray test. */ @@ -348,8 +348,8 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, /* Run tests for darray functions. */ int main(int argc, char **argv) { -#define NUM_REARRANGERS_TO_TEST 2 - int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; +#define NUM_REARRANGERS_TO_TEST 3 + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET, -PIO_REARR_SUBSET}; int my_rank; int ntasks; int num_flavors; /* Number of PIO netCDF flavors in this build. */ @@ -382,11 +382,11 @@ int main(int argc, char **argv) /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, - ioproc_start, rearranger[r], &iosysid))) + ioproc_start, abs(rearranger[r]), &iosysid))) return ret; /* Run tests. */ - if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm))) + if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm, rearranger[r]))) return ret; /* Finalize PIO system. */ diff --git a/tests/cunit/test_darray_async.c b/tests/cunit/test_darray_async.c index 86384dc1b..e91fc7047 100644 --- a/tests/cunit/test_darray_async.c +++ b/tests/cunit/test_darray_async.c @@ -241,7 +241,7 @@ int run_darray_async_test(int iosysid, int my_rank, MPI_Comm test_comm, MPI_Comm /* Create the PIO decomposition for this test. */ if ((ret = PIOc_init_decomp(iosysid, piotype, NDIM2, &dim_len[1], elements_per_pe, - compdof, &ioid, PIO_REARR_BOX, NULL, NULL))) + compdof, &ioid, PIO_REARR_BOX, NULL, NULL, NULL))) PBAIL(ret); /* Write the decomp file (on appropriate tasks). */ diff --git a/tests/cunit/test_darray_async_many.c b/tests/cunit/test_darray_async_many.c index 36cb7c856..420840558 100644 --- a/tests/cunit/test_darray_async_many.c +++ b/tests/cunit/test_darray_async_many.c @@ -382,47 +382,47 @@ int run_darray_async_test(int iosysid, int my_rank, MPI_Comm test_comm, sprintf(decomp_filename, "decomp_%s_%d.nc", TEST_NAME, rearr); /* Create the PIO decompositions for this test. */ if ((ret = PIOc_init_decomp(iosysid, PIO_BYTE, NDIM2, &dim_len[2], elements_per_pe, - compdof, &ioid_byte, rearr, NULL, NULL))) + compdof, &ioid_byte, rearr, NULL, NULL, NULL))) AERR(ret); if ((ret = PIOc_init_decomp(iosysid, PIO_CHAR, NDIM2, &dim_len[2], elements_per_pe, - compdof, &ioid_char, rearr, NULL, NULL))) + compdof, &ioid_char, rearr, NULL, NULL, NULL))) AERR(ret); if ((ret = PIOc_init_decomp(iosysid, PIO_SHORT, NDIM2, &dim_len[2], elements_per_pe, - compdof, &ioid_short, rearr, NULL, NULL))) + compdof, &ioid_short, rearr, NULL, NULL, NULL))) AERR(ret); if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM2, &dim_len[2], elements_per_pe, - compdof, &ioid_int, rearr, NULL, NULL))) + compdof, &ioid_int, rearr, NULL, NULL, NULL))) AERR(ret); if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM2, &dim_len[2], elements_per_pe, - compdof, &ioid_float, rearr, NULL, NULL))) + compdof, &ioid_float, rearr, NULL, NULL, NULL))) AERR(ret); if ((ret = PIOc_init_decomp(iosysid, PIO_DOUBLE, NDIM2, &dim_len[2], elements_per_pe, - compdof, &ioid_double, rearr, NULL, NULL))) + compdof, &ioid_double, rearr, NULL, NULL, NULL))) AERR(ret); #ifdef _NETCDF4 if ((ret = PIOc_init_decomp(iosysid, PIO_UBYTE, NDIM2, &dim_len[2], elements_per_pe, - compdof, &ioid_ubyte, rearr, NULL, NULL))) + compdof, &ioid_ubyte, rearr, NULL, NULL, NULL))) AERR(ret); if ((ret = PIOc_init_decomp(iosysid, PIO_USHORT, NDIM2, &dim_len[2], elements_per_pe, - compdof, &ioid_ushort, rearr, NULL, NULL))) + compdof, &ioid_ushort, rearr, NULL, NULL, NULL))) AERR(ret); if ((ret = PIOc_init_decomp(iosysid, PIO_UINT, NDIM2, &dim_len[2], elements_per_pe, - compdof, &ioid_uint, rearr, NULL, NULL))) + compdof, &ioid_uint, rearr, NULL, NULL, NULL))) AERR(ret); if ((ret = PIOc_init_decomp(iosysid, PIO_INT64, NDIM2, &dim_len[2], elements_per_pe, - compdof, &ioid_int64, rearr, NULL, NULL))) + compdof, &ioid_int64, rearr, NULL, NULL, NULL))) AERR(ret); if ((ret = PIOc_init_decomp(iosysid, PIO_UINT64, NDIM2, &dim_len[2], elements_per_pe, - compdof, &ioid_uint64, rearr, NULL, NULL))) + compdof, &ioid_uint64, rearr, NULL, NULL, NULL))) AERR(ret); #endif if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3, &dim_len[1], elements_per_pe_3d, - compdof_3d, &ioid_4d_int, rearr, NULL, NULL))) + compdof_3d, &ioid_4d_int, rearr, NULL, NULL, NULL))) AERR(ret); if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM3, &dim_len[1], elements_per_pe_3d, - compdof_3d, &ioid_4d_float, rearr, NULL, NULL))) + compdof_3d, &ioid_4d_float, rearr, NULL, NULL, NULL))) AERR(ret); /* These are the decompositions associated with each type. */ diff --git a/tests/cunit/test_darray_async_simple.c b/tests/cunit/test_darray_async_simple.c index 35f853b67..dc8b5742f 100644 --- a/tests/cunit/test_darray_async_simple.c +++ b/tests/cunit/test_darray_async_simple.c @@ -86,7 +86,7 @@ int run_darray_async_test(int iosysid, int my_rank, MPI_Comm test_comm, /* Create the PIO decomposition for this test. */ if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM1, &dim_len, elements_per_pe, - compdof, &ioid, PIO_REARR_BOX, NULL, NULL))) + compdof, &ioid, PIO_REARR_BOX, NULL, NULL, NULL))) ERR(ret); /* Write the decomp file (on appropriate tasks). */ diff --git a/tests/cunit/test_darray_fill.c b/tests/cunit/test_darray_fill.c index f4d6e64a5..bc7bbcb51 100644 --- a/tests/cunit/test_darray_fill.c +++ b/tests/cunit/test_darray_fill.c @@ -282,10 +282,10 @@ int main(int argc, char **argv) /* Initialize decompositions. */ if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, wcompmap, - &wioid, &rearranger[r], NULL, NULL))) + &wioid, &rearranger[r], NULL, NULL, NULL))) return ret; if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, rcompmap, - &rioid, &rearranger[r], NULL, NULL))) + &rioid, &rearranger[r], NULL, NULL, NULL))) return ret; /* Create the test file in each of the available iotypes. */ diff --git a/tests/cunit/test_darray_frame.c b/tests/cunit/test_darray_frame.c index 41f434523..ba4eba087 100644 --- a/tests/cunit/test_darray_frame.c +++ b/tests/cunit/test_darray_frame.c @@ -77,7 +77,7 @@ int test_frame_simple(int iosysid, int num_iotypes, int *iotype, int my_rank, /* Create the PIO decomposition for this test. */ printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe); if ((ret = PIOc_InitDecomp(iosysid, PIO_INT, NDIM2, dim_len_2d, elements_per_pe, - compdof, &ioid, NULL, NULL, NULL))) + compdof, &ioid, NULL, NULL, NULL, NULL))) ERR(ret); printf("%d decomposition initialized.\n", my_rank); diff --git a/tests/cunit/test_darray_lossycompress.c b/tests/cunit/test_darray_lossycompress.c index 041aea633..2c1c87d92 100644 --- a/tests/cunit/test_darray_lossycompress.c +++ b/tests/cunit/test_darray_lossycompress.c @@ -406,7 +406,7 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank * @returns 0 for success, error code otherwise. */ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, - MPI_Comm test_comm) + MPI_Comm test_comm, int rearranger) { #define NUM_TYPES_TO_TEST 2 int ioid; @@ -423,7 +423,7 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, /* Decompose the data over the tasks. */ if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, - &ioid, pio_type[t]))) + &ioid, pio_type[t], rearranger))) return ret; /* Run a simple darray test. */ @@ -441,8 +441,8 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, /* Run tests for darray functions. */ int main(int argc, char **argv) { -#define NUM_REARRANGERS_TO_TEST 2 - int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; +#define NUM_REARRANGERS_TO_TEST 3 + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET, -PIO_REARR_SUBSET}; int my_rank; int ntasks; int num_flavors; /* Number of PIO netCDF flavors in this build. */ @@ -475,11 +475,11 @@ int main(int argc, char **argv) /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, - ioproc_start, rearranger[r], &iosysid))) + ioproc_start, abs(rearranger[r]), &iosysid))) return ret; /* Run tests. */ - if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm))) + if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm, rearranger[r]))) return ret; /* Finalize PIO system. */ diff --git a/tests/cunit/test_darray_multi.c b/tests/cunit/test_darray_multi.c index 2784ee0c5..0a32994eb 100644 --- a/tests/cunit/test_darray_multi.c +++ b/tests/cunit/test_darray_multi.c @@ -381,7 +381,7 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank * @returns 0 for success, error code otherwise. */ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, - MPI_Comm test_comm) + MPI_Comm test_comm, int rearranger) { #ifdef _NETCDF4 #define NUM_TYPES_TO_TEST 11 @@ -404,7 +404,7 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, /* Decompose the data over the tasks. */ if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, - &ioid, pio_type[t]))) + &ioid, pio_type[t], rearranger))) return ret; /* Run a simple darray test. */ @@ -422,8 +422,8 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, /* Run tests for darray functions. */ int main(int argc, char **argv) { -#define NUM_REARRANGERS_TO_TEST 2 - int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; +#define NUM_REARRANGERS_TO_TEST 3 + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET, -PIO_REARR_SUBSET}; int my_rank; int ntasks; int num_flavors; /* Number of PIO netCDF flavors in this build. */ @@ -456,11 +456,11 @@ int main(int argc, char **argv) /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, - ioproc_start, rearranger[r], &iosysid))) + ioproc_start, abs(rearranger[r]), &iosysid))) return ret; /* Run tests. */ - if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm))) + if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm, rearranger[r]))) return ret; /* Finalize PIO system. */ diff --git a/tests/cunit/test_darray_multivar.c b/tests/cunit/test_darray_multivar.c index ab68195d3..3338aee30 100644 --- a/tests/cunit/test_darray_multivar.c +++ b/tests/cunit/test_darray_multivar.c @@ -513,7 +513,7 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, /* Decompose the data over the tasks. */ if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, - &ioid, test_type[t]))) + &ioid, test_type[t], rearranger))) return ret; /* Run the different combinations of use_fill and use_default. */ @@ -542,8 +542,8 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, /* Run tests for darray functions. */ int main(int argc, char **argv) { -#define NUM_REARRANGERS 2 - int rearranger[NUM_REARRANGERS] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; +#define NUM_REARRANGERS_TO_TEST 3 + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET, -PIO_REARR_SUBSET}; int my_rank; int ntasks; int num_flavors; /* Number of PIO netCDF flavors in this build. */ @@ -572,13 +572,13 @@ int main(int argc, char **argv) ERR(ret); /* Test for both arrangers. */ - for (int r = 0; r < NUM_REARRANGERS; r++) + for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++) { /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, - ioproc_start, rearranger[r], &iosysid))) + ioproc_start, abs(rearranger[r]), &iosysid))) return ret; /* printf("test Rearranger %d\n",rearranger[r]); */ /* Run tests. */ diff --git a/tests/cunit/test_darray_multivar2.c b/tests/cunit/test_darray_multivar2.c index 377f77a1b..72222fda7 100644 --- a/tests/cunit/test_darray_multivar2.c +++ b/tests/cunit/test_darray_multivar2.c @@ -191,7 +191,7 @@ int create_decomposition_2d_2(int ntasks, int my_rank, int iosysid, int *dim_len /* Create the PIO decomposition for this test. */ if ((ret = PIOc_InitDecomp(iosysid, pio_type, NDIM2, dim_len_2d, elements_per_pe, - compdof, ioid, NULL, NULL, NULL))) + compdof, ioid, NULL, NULL, NULL, NULL))) ERR(ret); /* Free the mapping. */ diff --git a/tests/cunit/test_darray_multivar3.c b/tests/cunit/test_darray_multivar3.c index f44e38976..fdfbf7f9c 100644 --- a/tests/cunit/test_darray_multivar3.c +++ b/tests/cunit/test_darray_multivar3.c @@ -267,7 +267,7 @@ int create_dcomp_gaps(int ntasks, int my_rank, int iosysid, int *dim_len_2d, /* Create the PIO decomposition for this test. */ if ((ret = PIOc_InitDecomp(iosysid, pio_type, NDIM2, dim_len_2d, elements_per_pe, - compdof, ioid, NULL, NULL, NULL))) + compdof, ioid, NULL, NULL, NULL, NULL))) ERR(ret); /* Free the mapping. */ diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index 1a94ecdf3..d1e77c5e7 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -407,7 +407,7 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, * @returns 0 for success, error code otherwise. */ int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor, - int my_rank, MPI_Comm test_comm) + int my_rank, MPI_Comm test_comm, int rearranger) { int ioid; char filename[PIO_MAX_NAME + 1]; @@ -440,7 +440,7 @@ int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor, /* Decompose the data over the tasks. */ if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, - dim_len_2d, &ioid, type_to_use))) + dim_len_2d, &ioid, type_to_use, rearranger))) return ret; /* Run a simple darray test. */ @@ -459,8 +459,8 @@ int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor, /* Run tests for darray functions. */ int main(int argc, char **argv) { -#define NUM_REARRANGERS_TO_TEST 2 - int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; +#define NUM_REARRANGERS_TO_TEST 3 + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET, -PIO_REARR_SUBSET}; int my_rank; int ntasks; int num_flavors; /* Number of PIO netCDF flavors in this build. */ @@ -499,12 +499,12 @@ int main(int argc, char **argv) * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, ioproc_start, - rearranger[r], &iosysid))) + abs(rearranger[r]), &iosysid))) return ret; /* Run tests. */ if ((ret = test_all_darray(iosysid, fmt, num_flavors, flavor, - my_rank, test_comm))) + my_rank, test_comm, rearranger[r]))) return ret; /* Finalize PIO system. */ diff --git a/tests/cunit/test_decomp_frame.c b/tests/cunit/test_decomp_frame.c index 3aa893d6e..5a7c1f242 100644 --- a/tests/cunit/test_decomp_frame.c +++ b/tests/cunit/test_decomp_frame.c @@ -283,10 +283,10 @@ int main(int argc, char **argv) /* Initialize decompositions. */ if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, wcompmap, - &wioid, &rearranger[r], NULL, NULL))) + &wioid, &rearranger[r], NULL, NULL, NULL))) return ret; if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, rcompmap, - &rioid, &rearranger[r], NULL, NULL))) + &rioid, &rearranger[r], NULL, NULL, NULL))) return ret; /* Create the test file in each of the available iotypes. */ diff --git a/tests/cunit/test_decomp_uneven.c b/tests/cunit/test_decomp_uneven.c index c916d3abd..d7d1f5d00 100644 --- a/tests/cunit/test_decomp_uneven.c +++ b/tests/cunit/test_decomp_uneven.c @@ -76,7 +76,7 @@ int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *dim_len, /* Create the PIO decomposition for this test. */ if ((ret = PIOc_init_decomp(iosysid, pio_type, NDIM3, dim_len, elements_per_pe, - compdof, ioid, 0, NULL, NULL))) + compdof, ioid, 0, NULL, NULL, NULL))) ERR(ret); /* Free the mapping. */ diff --git a/tests/cunit/test_decomps.c b/tests/cunit/test_decomps.c index 0c57eec91..c8be9e6ff 100644 --- a/tests/cunit/test_decomps.c +++ b/tests/cunit/test_decomps.c @@ -40,7 +40,7 @@ #define NUM_IO1 1 #define NUM_IO2 2 #define NUM_IO4 4 -#define NUM_REARRANGER 2 +#define NUM_REARRANGERS_TO_TEST 3 /** * Test some decomposition functions. @@ -81,19 +81,19 @@ int test_decomp1(int iosysid, int use_io, int my_rank, MPI_Comm test_comm) /* These should not work. */ bad_slice_dimlen[1] = 0; if (PIOc_InitDecomp(iosysid + TEST_VAL_42, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe, - compdof, &ioid, NULL, NULL, NULL) != PIO_EBADID) + compdof, &ioid, NULL, NULL, NULL, NULL) != PIO_EBADID) return ERR_WRONG; if (PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, bad_slice_dimlen, (PIO_Offset)elements_per_pe, - compdof, &ioid, NULL, NULL, NULL) != PIO_EINVAL) + compdof, &ioid, NULL, NULL, NULL, NULL) != PIO_EINVAL) return ERR_WRONG; if (PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, NULL, (PIO_Offset)elements_per_pe, - compdof, &ioid, NULL, NULL, NULL) != PIO_EINVAL) + compdof, &ioid, NULL, NULL, NULL, NULL) != PIO_EINVAL) return ERR_WRONG; if (PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe, - NULL, &ioid, NULL, NULL, NULL) != PIO_EINVAL) + NULL, &ioid, NULL, NULL, NULL, NULL) != PIO_EINVAL) return ERR_WRONG; if (PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe, - compdof, NULL, NULL, NULL, NULL) != PIO_EINVAL) + compdof, NULL, NULL, NULL, NULL, NULL) != PIO_EINVAL) return ERR_WRONG; /* Sometimes we will test with these arrays. */ @@ -113,7 +113,7 @@ int test_decomp1(int iosysid, int use_io, int my_rank, MPI_Comm test_comm) /* Create the PIO decomposition for this test. */ if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe, - compdof, &ioid, NULL, iostart, iocount))) + compdof, &ioid, NULL, iostart, iocount, NULL))) { if (iostart) free(iostart); @@ -365,7 +365,7 @@ int main(int argc, char **argv) int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; int ioid; - int rearranger[NUM_REARRANGERS] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET, -PIO_REARR_SUBSET}; int ret; /* Return code. */ /* Initialize test. */ @@ -383,14 +383,14 @@ int main(int argc, char **argv) /* Test for each rearranger. */ /* for (int r = 0; r < NUM_REARRANGERS; r++) */ - for (int r = 1; r < NUM_REARRANGERS; r++) + for (int r = 1; r < NUM_REARRANGERS_TO_TEST; r++) { int num_iotests = (rearranger[r] == PIO_REARR_BOX) ? 2 : 1; for (int io_test = 0; io_test < num_iotests; io_test++) { /* Initialize PIO system on world. */ - if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO4, STRIDE1, BASE0, rearranger[r], &iosysid))) + if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO4, STRIDE1, BASE0, abs(rearranger[r]), &iosysid))) ERR(ret); /* Set the error handler. */ @@ -407,11 +407,11 @@ int main(int argc, char **argv) /* Decompose the data over the tasks. */ if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, &ioid, - PIO_INT))) + PIO_INT, rearranger[r]))) return ret; /* Test decomposition read/write. */ - if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, rearranger[r], + if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, abs(rearranger[r]), my_rank, test_comm))) return ret; diff --git a/tests/cunit/test_perf2.c b/tests/cunit/test_perf2.c index ea77cf41d..f297685fe 100644 --- a/tests/cunit/test_perf2.c +++ b/tests/cunit/test_perf2.c @@ -95,7 +95,7 @@ create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid) /* Create the PIO decomposition for this test. */ if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3, dim_len_3d, elements_per_pe, - compdof, ioid, 0, NULL, NULL))) + compdof, ioid, 0, NULL, NULL, NULL))) ERR(ret); /* Free the mapping. */ diff --git a/tests/cunit/test_pioc.c b/tests/cunit/test_pioc.c index f50f6fbdd..97e8901c5 100644 --- a/tests/cunit/test_pioc.c +++ b/tests/cunit/test_pioc.c @@ -117,15 +117,15 @@ int create_decomposition(int ntasks, int my_rank, int iosysid, int dim1_len, int /* These should fail. */ if (PIOc_init_decomp(iosysid + TEST_VAL_42, PIO_FLOAT, NDIM1, dim_len, elements_per_pe, - compdof, ioid, 0, NULL, NULL) != PIO_EBADID) + compdof, ioid, 0, NULL, NULL, NULL) != PIO_EBADID) ERR(ERR_WRONG); if (PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM1, bad_dim_len, elements_per_pe, - compdof, ioid, 0, NULL, NULL) != PIO_EINVAL) + compdof, ioid, 0, NULL, NULL, NULL) != PIO_EINVAL) ERR(ERR_WRONG); /* Create the PIO decomposition for this test. */ if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM1, dim_len, elements_per_pe, - compdof, ioid, 0, NULL, NULL))) + compdof, ioid, 0, NULL, NULL, NULL))) ERR(ret); /* Free the mapping. */ @@ -2299,7 +2299,7 @@ int test_decomp_public_async(int my_test_size, int my_rank, int iosysid, MPI_Com /* Create the PIO decomposition for this test. */ if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM1, &dim_len, elements_per_pe, - compdof, &ioid, PIO_REARR_BOX, NULL, NULL))) + compdof, &ioid, PIO_REARR_BOX, NULL, NULL, NULL))) ERR(ret); /* Write the decomp file (on appropriate tasks). */ diff --git a/tests/cunit/test_pioc_unlim.c b/tests/cunit/test_pioc_unlim.c index be56ee1d8..e002bb7b7 100644 --- a/tests/cunit/test_pioc_unlim.c +++ b/tests/cunit/test_pioc_unlim.c @@ -83,7 +83,7 @@ int create_decomposition(int ntasks, int my_rank, int iosysid, int dim1_len, /* Create the PIO decomposition for this test. */ if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, NDIM - 1, &dim_len[1], elements_per_pe, - compdof, ioid, NULL, NULL, NULL))) + compdof, ioid, NULL, NULL, NULL, NULL))) ERR(ret); /* Free the mapping. */ diff --git a/tests/cunit/test_rearr.c b/tests/cunit/test_rearr.c index 27ed335a4..0b92fba00 100644 --- a/tests/cunit/test_rearr.c +++ b/tests/cunit/test_rearr.c @@ -29,6 +29,8 @@ /* Name of test var. (Name of a Welsh town.)*/ #define VAR_NAME "Llanfairpwllgwyngyllgogerychwyrndrobwllllantysiliogogogoch" +extern int round_robin_partition(int comprank, int iorank, int comptasks, int iotasks, int *color, int *key); + /* Test some of the rearranger utility functions. */ int test_rearranger_opts1(int iosysid) { @@ -684,7 +686,16 @@ int test_init_decomp(int iosysid, MPI_Comm test_comm, int my_rank) /* Initialize a decomposition. */ if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM1, gdimlen, MAPLEN2, - compmap, &ioid, PIO_REARR_BOX, NULL, NULL))) + compmap, &ioid, PIO_REARR_BOX, NULL, NULL, NULL))) + return ret; + + /* Free it. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + return ret; + + /* Initialize a decomposition. */ + if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM1, gdimlen, MAPLEN2, + compmap, &ioid, PIO_REARR_SUBSET, NULL, NULL, round_robin_partition))) return ret; /* Free it. */ @@ -923,41 +934,6 @@ int test_box_rearrange_create_2(MPI_Comm test_comm, int my_rank) return 0; } -/* Test function default_subset_partition. */ -int test_default_subset_partition(MPI_Comm test_comm, int my_rank) -{ - iosystem_desc_t *ios; - io_desc_t *iodesc; - int mpierr; - int ret; - - /* Allocate IO system info struct for this test. */ - if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) - return PIO_ENOMEM; - - /* Allocate IO desc struct for this test. */ - if (!(iodesc = calloc(1, sizeof(io_desc_t)))) - return PIO_ENOMEM; - - ios->ioproc = 1; - ios->io_rank = my_rank; - ios->union_comm = test_comm; - - /* Run the function to test. */ - if ((ret = default_subset_partition(ios, iodesc))) - return ret; - - /* Free the created communicator. */ - if ((mpierr = MPI_Comm_free(&iodesc->subset_comm))) - MPIERR(mpierr); - - /* Free resources from test. */ - free(iodesc); - free(ios); - - return 0; -} - /* Test function rearrange_comp2io. */ int test_rearrange_comp2io(MPI_Comm test_comm, int my_rank) { @@ -1288,9 +1264,6 @@ int run_no_iosys_tests(int my_rank, MPI_Comm test_comm) if ((ret = test_box_rearrange_create_2(test_comm, my_rank))) return ret; - if ((ret = test_default_subset_partition(test_comm, my_rank))) - return ret; - if ((ret = test_rearrange_comp2io(test_comm, my_rank))) return ret; diff --git a/tests/cunit/test_shared.c b/tests/cunit/test_shared.c index cd7308917..d07fdbbab 100644 --- a/tests/cunit/test_shared.c +++ b/tests/cunit/test_shared.c @@ -105,7 +105,7 @@ int test_no_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm /* Create the PIO decomposition for this test. */ if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe, - compdof, &ioid, 0, NULL, NULL))) + compdof, &ioid, 0, NULL, NULL, NULL))) return ret; free(compdof); diff --git a/tests/cunit/test_simple.c b/tests/cunit/test_simple.c index e18ca0d2a..22733051c 100644 --- a/tests/cunit/test_simple.c +++ b/tests/cunit/test_simple.c @@ -81,7 +81,7 @@ int main(int argc, char **argv) for (i = 0; i < elements_per_pe; i++) compmap[i] = my_rank + i; if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM1, &gdimlen, elements_per_pe, compmap, - &ioid, PIO_REARR_BOX, NULL, NULL))) + &ioid, PIO_REARR_BOX, NULL, NULL, NULL))) ERR(ret); free(compmap); diff --git a/tests/performance/pioperformance.F90 b/tests/performance/pioperformance.F90 index 7f09919ed..b5e69b5ed 100644 --- a/tests/performance/pioperformance.F90 +++ b/tests/performance/pioperformance.F90 @@ -29,8 +29,11 @@ program pioperformance integer :: vs, varsize(max_nvars) ! Local size of array for idealized decomps logical :: unlimdimindof integer :: log_level + integer, parameter :: pathlen=120 + character(len=pathlen) :: lib_path, func_name + namelist /pioperf/ decompfile, pio_typenames, rearrangers, niotasks, nframes, & - nvars, varsize, unlimdimindof, log_level + nvars, varsize, unlimdimindof, log_level, lib_path, func_name #ifdef BGQTRY external :: print_memusage #endif @@ -67,6 +70,8 @@ program pioperformance varsize(1) = 1 unlimdimindof=.false. log_level = -1 + func_name = "" + lib_path = "" if(mype==0) then open(unit=12,file='pioperf.nl',status='old') read(12,pioperf) @@ -97,6 +102,8 @@ program pioperformance call MPI_Bcast(nvars, max_nvars, MPI_INTEGER, 0, MPI_COMM_WORLD,ierr) call MPI_Bcast(varsize, max_nvars, MPI_INTEGER, 0, MPI_COMM_WORLD,ierr) call MPI_Bcast(log_level, 1, MPI_INTEGER, 0, MPI_COMM_WORLD,ierr) + call MPI_Bcast(lib_path, pathlen, MPI_CHARACTER, 0, MPI_COMM_WORLD,ierr) + call MPI_Bcast(func_name, pathlen, MPI_CHARACTER, 0, MPI_COMM_WORLD,ierr) call t_initf('pioperf.nl', LogPrint=.false., mpicom=MPI_COMM_WORLD, MasterTask=MasterTask) niotypes = 0 @@ -116,7 +123,7 @@ program pioperformance do nv=1,max_nvars if(nvars(nv)>0) then call pioperformancetest(decompfile(i), piotypes(1:niotypes), mype, npe, & - rearrangers, niotasks, nframes, nvars(nv), varsize(vs),unlimdimindof) + rearrangers, niotasks, nframes, nvars(nv), varsize(vs),unlimdimindof,lib_path,func_name) if(mype==0) print * ,' complete' endif enddo @@ -130,7 +137,7 @@ program pioperformance contains subroutine pioperformancetest(filename, piotypes, mype, npe_base, & - rearrangers, niotasks,nframes, nvars, varsize, unlimdimindof) + rearrangers, niotasks,nframes, nvars, varsize, unlimdimindof, lib_path, func_name ) use pio use pio_support, only : pio_readdof use perf_mod @@ -143,6 +150,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & integer, intent(in) :: nvars integer, intent(in) :: varsize logical, intent(in) :: unlimdimindof + character(len=*), intent(in) :: lib_path, func_name integer(kind=PIO_Offset_kind), pointer :: compmap(:) integer :: ntasks integer :: comm @@ -284,16 +292,21 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & call date_and_time(DATE=date, TIME=time) nvarmult= pio_put_var(File, rundate, date//' '//time(1:4)) endif - + if(mype == 0 .and. len_trim(lib_path) > 0) print *,'Using partition lib_path=',trim(lib_path),len_trim(lib_path) + if(mype == 0 .and. len_trim(func_name) > 0) print *,'Using partition function=',trim(func_name), len_trim(func_name) + if(.not. unlimdimindof) then #ifdef VARINT - call PIO_InitDecomp(iosystem, PIO_INT, gdims, compmap, iodesc_i4, rearr=rearr) + call PIO_InitDecomp(iosystem, PIO_INT, gdims, compmap, iodesc_i4, rearr=rearr, & + lib_path=trim(lib_path),func_name=trim(func_name)) #endif #ifdef VARREAL - call PIO_InitDecomp(iosystem, PIO_REAL, gdims, compmap, iodesc_r4, rearr=rearr) + call PIO_InitDecomp(iosystem, PIO_REAL, gdims, compmap, iodesc_r4, rearr=rearr, & + lib_path=trim(lib_path),func_name=trim(func_name)) #endif #ifdef VARDOUBLE - call PIO_InitDecomp(iosystem, PIO_DOUBLE, gdims, compmap, iodesc_r8, rearr=rearr) + call PIO_InitDecomp(iosystem, PIO_DOUBLE, gdims, compmap, iodesc_r8, rearr=rearr, & + lib_path=trim(lib_path),func_name=trim(func_name)) #endif endif @@ -306,13 +319,16 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & ! compmap = compmap2 + (frame-1)*gdims(ndims) ! print *,__FILE__,__LINE__,compmap #ifdef VARINT - call PIO_InitDecomp(iosystem, PIO_INT, gdims, compmap, iodesc_i4, rearr=rearr) + call PIO_InitDecomp(iosystem, PIO_INT, gdims, compmap, iodesc_i4, rearr=rearr, & + lib_path=trim(lib_path),func_name=trim(func_name)) #endif #ifdef VARREAL - call PIO_InitDecomp(iosystem, PIO_REAL, gdims, compmap, iodesc_r4, rearr=rearr) + call PIO_InitDecomp(iosystem, PIO_REAL, gdims, compmap, iodesc_r4, rearr=rearr, & + lib_path=trim(lib_path),func_name=trim(func_name)) #endif #ifdef VARDOUBLE - call PIO_InitDecomp(iosystem, PIO_DOUBLE, gdims, compmap, iodesc_r8, rearr=rearr) + call PIO_InitDecomp(iosystem, PIO_DOUBLE, gdims, compmap, iodesc_r8, rearr=rearr, & + lib_path=trim(lib_path),func_name=trim(func_name)) #endif endif !if(mype==0) print *,__FILE__,__LINE__,'Frame: ',recnum @@ -399,13 +415,16 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & if( unlimdimindof) then #ifdef VARINT - call PIO_InitDecomp(iosystem, PIO_INT, gdims, compmap, iodesc_i4, rearr=rearr) + call PIO_InitDecomp(iosystem, PIO_INT, gdims, compmap, iodesc_i4, rearr=rearr, & + lib_path=trim(lib_path),func_name=trim(func_name)) #endif #ifdef VARREAL - call PIO_InitDecomp(iosystem, PIO_REAL, gdims, compmap, iodesc_r4, rearr=rearr) + call PIO_InitDecomp(iosystem, PIO_REAL, gdims, compmap, iodesc_r4, rearr=rearr, & + lib_path=trim(lib_path),func_name=trim(func_name)) #endif #ifdef VARDOUBLE - call PIO_InitDecomp(iosystem, PIO_DOUBLE, gdims, compmap, iodesc_r8, rearr=rearr) + call PIO_InitDecomp(iosystem, PIO_DOUBLE, gdims, compmap, iodesc_r8, rearr=rearr, & + lib_path=trim(lib_path),func_name=trim(func_name)) #endif endif @@ -571,7 +590,7 @@ subroutine init_ideal_dof(doftype, mype, npe, ndims, gdims, compmap, varsize) enddo endif if(minval(compmap)< 1 .or. maxval(compmap) > gdims(1)) then - print *,__FILE__,__LINE__,trim(doftype),varsize,minval(compmap),maxval(compmap) + print *,__LINE__,trim(doftype),varsize,minval(compmap),maxval(compmap) call piodie(__FILE__,__LINE__,'Compmap out of bounds') endif end subroutine init_ideal_dof