NCAR · jedwards4b · Feb 20, 2025 · Feb 20, 2025 · Feb 20, 2025 · Feb 20, 2025
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -321,7 +321,7 @@ SET(STATUS_PNETCDF ${PnetCDF_C_FOUND})
 ###
 CHECK_C_SOURCE_COMPILES("
 #include <netcdf_meta.h>
-#if !NC_HAS_NC4
+#if !NC_HAS_HDF5 && !NC_HAS_NC4
       choke me
 #endif
 int main() {return 0;}" HAVE_NETCDF4)
@@ -452,7 +452,7 @@ SET(LDFLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CMAKE_EXE_LINKER_FLAGS_${CMAKE_BUILD_TY
 
 is_disabled(BUILD_SHARED_LIBS enable_static)
 is_enabled(BUILD_SHARED_LIBS enable_shared)
-is_enabled(HAVE_PAR_FILTERS have_par_filters)
+is_enabled(PIO_HAS_PAR_FILTERS have_par_filters)
 is_enabled(USE_SZIP HAS_SZIP_WRITE)
 is_enabled(STATUS_PNETCDF HAS_PNETCDF)
 is_enabled(HAVE_H5Z_SZIP HAS_SZLIB)

diff --git a/README.md b/README.md
@@ -1,117 +1,114 @@
 # ParallelIO
 
-The Parallel IO libraries (PIO) are high-level parallel I/O C and
-Fortran libraries for applications that need to do netCDF I/O from
-large numbers of processors on a HPC system.
-
-PIO provides a netCDF-like API, and allows users to designate some
-subset of processors to perform IO. Computational code calls
-netCDF-like functions to read and write data, and PIO uses the IO
-processors to perform all necessary IO.
-
-## Intracomm Mode
-
-In Intracomm mode, PIO allows the user to designate some subset of
-processors to do all I/O. The I/O processors also participate in
-computational work.
-
-![I/O on Many Processors with Async
- Mode](./doc/images/I_O_on_Many_Intracomm.png)
-
-## Async Mode
-
-PIO also supports the creation of multiple computation components,
-each containing many processors, and one shared set of IO
-processors. The computational components can perform write operation
-asynchronously, and the IO processors will take care of all storage
-interaction.
-
-![I/O on Many Processors with Async
- Mode](./doc/images/I_O_on_Many_Async.png)
-
-## Website
-
-For complete documentation, see our website at
-[http://ncar.github.io/ParallelIO/](http://ncar.github.io/ParallelIO/).
-
-## Mailing List
-
-The (low-traffic) PIO mailing list is at
-https://groups.google.com/forum/#!forum/parallelio, send email to the
-list at parallelio@googlegroups.com.
-
-## Testing
-
-The results of our continuous integration testing with GitHub actions
-can be found on any of the Pull Requests on the GitHub site:
-https://github.com/NCAR/ParallelIO.
+## Overview
+This repository provides a robust implementation of **Parallel I/O (PIO)** using **NetCDF** and **ParallelNetCDF** backends. It is optimized for **high-performance computing (HPC)** and integrates with parallel file systems to efficiently handle large-scale scientific datasets.
+
+## Features
+- Parallel I/O support using **NetCDF** and **ParallelNetCDF**.
+- Optimized **data rearrangement** for improved performance.
+- **Asynchronous I/O** for reducing computational overhead.
+- Compatible with **MPI-based distributed computing**.
+- Extensive testing suite and continuous integration with **GitHub Actions**.
+
+## Installation
+### Prerequisites
+Ensure you have the following dependencies installed:
+- C and Fortran compilers (e.g., GCC, Intel, Clang)
+- **MPI library** (MPICH or OpenMPI)
+- **NetCDF** and **ParallelNetCDF** (built with MPI support)
+- Either **CMake** or **Autotools**
+
+### Build Instructions
+#### Using CMake
+```bash
+mkdir build && cd build
+cmake .. -DENABLE_FORTRAN=ON -DENABLE_NETCDF=ON
+make -j$(nproc)
+make install
+```
 
-The results of our nightly tests on multiple platforms can be found on
-our cdash site at
-[http://my.cdash.org/index.php?project=PIO](http://my.cdash.org/index.php?project=PIO).
+#### Using Autotools
+```bash
+autoreconf -i
+./configure --enable-fortran --enable-netcdf-integration
+make -j$(nproc)
+make install
+```
 
-## Dependencies
+## Running Tests
+### Unit and Integration Tests
+To run the test suite, use the following commands:
+```bash
+mpiexec -n 4 make check  # Run all unit tests
+ctest -VV   # Run CMake-based tests (must be run on a parallel-capable node)
+```
 
-PIO can use NetCDF (version 4.6.1+) and/or PnetCDF (version 1.9.0+)
-for I/O. NetCDF may be built with or without netCDF-4 features. NetCDF
-is required for PIO, PnetCDF is optional.
+For debugging failed tests:
+```bash
+ctest --rerun-failed --output-on-failure  # Must be run on a parallel-capable node
+```
 
-The NetCDF C library must be built with MPI, which requires that it be
-linked with an MPI-enabled version of HDF5. Optionally, NetCDF can be
-built with DAP support, which introduces a dependency on CURL.  HDF5,
-itself, introduces dependencies on LIBZ and (optionally) SZIP.
+## Usage
+### Simple Example (C)
+```c
+#include <pio.h>
+MPI_Init(NULL, NULL);
+PIO_Init();
+PIO_CreateFile("output.nc", PIO_WRITE, PIO_NETCDF);
+PIO_CloseFile();
+PIO_Finalize();
+MPI_Finalize();
+```
 
-## Building PIO
+### Simple Example (Fortran)
+```fortran
+program test_pio
+  use pio
+  call MPI_Init()
+  call PIO_Init()
+  call PIO_CreateFile("output.nc", PIO_WRITE, PIO_NETCDF)
+  call PIO_CloseFile()
+  call PIO_Finalize()
+  call MPI_Finalize()
+end program test_pio
+```
 
-To build PIO, unpack the distribution tarball and do:
+## Contributing
+We welcome contributions! To contribute:
+1. Fork the repository.
+2. Create a feature branch.
+3. Run tests before committing changes.
+4. Submit a pull request with a clear description.
 
-```
-CC=mpicc FC=mpif90 ./configure --enable-fortran && make check install
-```
+Refer to **doc/contributing_code.txt** for detailed guidelines.
 
-For a full description of the available options and flags, try:
-```
-./configure --help
-```
+## Intracomm and Asynchronous Modes
+ParallelIO supports two primary modes of operation:
 
-Note that environment variables CC and FC may need to be set to the
-MPI versions of the C and Fortran compiler. Also CPPFLAGS and LDFLAGS
-may need to be set to indicate the locations of one or more of the
-dependent libraries. (If using MPI compilers, the entire set of
-dependent libraries should be built with the same compilers.) For
-example:
+### Intracomm Mode
+- In **Intracomm mode**, a subset of MPI tasks participate in I/O operations while also performing computation tasks. This approach avoids the bottleneck that occurs when every task participates in I/O, improving overall performance.
+- This mode ensures efficient parallel data writes and reads across distributed systems.
+- Suitable for systems where dedicated I/O tasks are not needed.
 
-```
-export CC=mpicc
-export FC=mpifort
-export CPPFLAGS='-I/usr/local/netcdf-fortran-4.4.5_c_4.6.3_mpich-3.2/include -I/usr/local/netcdf-c-4.6.3_hdf5-1.10.5/include -I/usr/local/pnetcdf-1.11.0_shared/include'
-export LDFLAGS='-L/usr/local/netcdf-c-4.6.3_hdf5-1.10.5/lib -L/usr/local/pnetcdf-1.11.0_shared/lib'
-./configure --prefix=/usr/local/pio-2.4.2 --enable-fortran
-make check
-make install
-```
+#### Illustration:
+![Intracomm Mode](./doc/images/I_O_on_Many_Intracomm.png)
 
-## Building with CMake
+### Asynchronous Mode
+- In **Asynchronous mode**, a subset of tasks handle I/O while others focus on computation.
+- Helps in overlapping computation and I/O for better performance.
 
-The typical configuration with CMake can be done as follows:
 
-```
-CC=mpicc FC=mpif90 cmake [-DOPTION1=value1 -DOPTION2=value2 ...] /path/to/pio/source
-```
+#### Illustration:
+![Asynchronous Mode](doc/images/I_O_on_Many_Async.png)
 
-Full instructions for the cmake build can be found in the [installation
-documentation](https://ncar.github.io/ParallelIO/install.html).
+## Documentation
+- [NetCDF Homepage](https://www.unidata.ucar.edu/software/netcdf/)
+- [ParallelNetCDF Homepage](https://parallel-netcdf.github.io/)
+- [User Guide](doc/users_guide.txt)
+- [API Documentation](doc/api.txt)
+- [Testing Guidelines](doc/Testing.txt)
 
-# References
+## License
+This project is licensed under the **MIT License**. See the `LICENSE` file for details.
 
-Hartnett, E., Edwards, J., "THE PARALLELIO (PIO) C/FORTRAN LIBRARIES
-FOR SCALABLE HPC PERFORMANCE", 37th Conference on Environmental
-Information Processing Technologies, American Meteorological Society
-Annual Meeting, January, 2021. Retrieved on Feb 3, 2021, from
-[https://www.researchgate.net/publication/348169990_THE_PARALLELIO_PIO_CFORTRAN_LIBRARIES_FOR_SCALABLE_HPC_PERFORMANCE].
 
-Hartnett, E., Edwards, J., "POSTER: THE PARALLELIO (PIO) C/FORTRAN LIBRARIES
-FOR SCALABLE HPC PERFORMANCE", 37th Conference on Environmental
-Information Processing Technologies, American Meteorological Society
-Annual Meeting, January, 2021. Retrieved on Feb 3, 2021, from
-[https://www.researchgate.net/publication/348170136_THE_PARALLELIO_PIO_CFORTRAN_LIBRARIES_FOR_SCALABLE_HPC_PERFORMANCE].
diff --git a/configure.ac b/configure.ac
@@ -261,7 +261,7 @@ fi
 
 # Do we have netCDF-4?
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include "netcdf_meta.h"],
-[[#if !NC_HAS_NC4
+[[#if !NC_HAS_HDF5 && !NC_HAS_NC4
 # error
 #endif]
 ])], [have_netcdf4=yes], [have_netcdf4=no])

diff --git a/examples/c/darray_no_async.c b/examples/c/darray_no_async.c
@@ -277,7 +277,7 @@ int main(int argc, char* argv[])
     /* printf("rank: %d Creating decomposition, elements_per_pe %lld...\n", my_rank, */
     /*        elements_per_pe); */
     if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe,
-                                compdof, &ioid, PIO_REARR_SUBSET, NULL, NULL)))
+                                compdof, &ioid, PIO_REARR_SUBSET, NULL, NULL, NULL)))
         ERR(ret);
 
     /* Write the decomposition file. */

diff --git a/examples/c/example1.c b/examples/c/example1.c
@@ -342,7 +342,7 @@ int check_file(int ntasks, char *filename) {
         if (verbose)
             printf("rank: %d Creating decomposition...\n", my_rank);
         if ((ret = PIOc_InitDecomp(iosysid, PIO_INT, NDIM, dim_len, (PIO_Offset)elements_per_pe,
-                                   compdof, &ioid, NULL, NULL, NULL)))
+                                   compdof, &ioid, NULL, NULL, NULL, NULL)))
             ERR(ret);
         free(compdof);
 

diff --git a/examples/c/examplePio.c b/examples/c/examplePio.c
@@ -319,7 +319,7 @@ struct examplePioClass* epc_createDecomp( struct examplePioClass* this )
     if (this->verbose)
 	printf("rank: %d Creating decomposition...\n", this->myRank);
     PIOc_InitDecomp(this->pioIoSystem, PIO_INT, 1, this->dimLen, (PIO_Offset)(this->arrIdxPerPe),
-                    this->compdof, &this->iodescNCells, NULL, NULL, NULL);
+                    this->compdof, &this->iodescNCells, NULL, NULL, NULL, NULL);
     return this;
 }
 

diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt
@@ -143,7 +143,7 @@ endif ()
 target_include_directories (pioc
   PUBLIC ${PIO_C_EXTRA_INCLUDE_DIRS})
 target_link_libraries (pioc
-  PUBLIC ${PIO_C_EXTRA_LIBRARIES})
+  PUBLIC ${PIO_C_EXTRA_LIBRARIES} ${CMAKE_DL_LIBS})
 target_compile_options (pioc
   PRIVATE ${PIO_C_EXTRA_COMPILE_OPTIONS})
 target_compile_definitions (pioc

diff --git a/src/clib/pio.h b/src/clib/pio.h
@@ -802,23 +802,44 @@ extern "C" {
     int PIOc_set_log_level(int level);
     int PIOc_set_global_log_level(int iosysid, int level);
 
+    /**
+     * @brief Function type for custom partitioning strategies (subset partitioning)
+     *
+     * @param ios      Pointer to the iosystem description
+     * @param iodesc   Pointer to the IO description structure
+     * @param color    Pointer to store the computed color value
+     * @param key      Pointer to store the computed key value
+     * @return PIO_NOERR on success, error code otherwise
+     */
+    typedef int (*pio_partition_fn)(int comprank, int iorank, int comptasks, int iotasks,
+				int *color, int *key);
+
+
     /* Decomposition. */
 
     /* Init decomposition with 1-based compmap array. */
     int PIOc_InitDecomp_ReadOnly(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen,
                         const PIO_Offset *compmap, int *ioidp, const int *rearr,
-                        const PIO_Offset *iostart, const PIO_Offset *iocount);
+				 const PIO_Offset *iostart, const PIO_Offset *iocount, pio_partition_fn partition_fn);
     int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen,
                         const PIO_Offset *compmap, int *ioidp, const int *rearr,
-                        const PIO_Offset *iostart, const PIO_Offset *iocount);
+                        const PIO_Offset *iostart, const PIO_Offset *iocount, pio_partition_fn partition_fn);
     int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *gdimlen,
                            const long int *start, const long int *count, int *ioidp);
 
     /* Init decomposition with 0-based compmap array. */
     int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen,
                          const PIO_Offset *compmap, int *ioidp, int rearranger,
-                         const PIO_Offset *iostart, const PIO_Offset *iocount);
-
+                         const PIO_Offset *iostart, const PIO_Offset *iocount, pio_partition_fn partition_fn);
+
+    int PIOc_InitDecomp_DynamicPartitioner(int iosysid, int basetype, int ndims, 
+					   const int *dims, int maplen,
+					   const PIO_Offset *compmap, int *ioidp, 
+					   const int *rearranger,
+					   const PIO_Offset *iostart, 
+					   const PIO_Offset *iocount,
+					   const char *lib_path, 
+					   const char *func_name) ;
     /* Free resources associated with a decomposition. */
     int PIOc_freedecomp(int iosysid, int ioid);
 

diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h
@@ -17,6 +17,8 @@
 #include <limits.h>
 #include <math.h>
 #include <netcdf.h>
+#include <dlfcn.h> // used in PIOc_InitDecomp_DynamicPartitioner
+
 #ifdef NC_HAS_PAR_FILTERS
 #include <netcdf_filter.h>
 #include <netcdf_meta.h>
@@ -251,7 +253,7 @@ extern "C" {
                        const PIO_Offset *dest_ioindex);
 
     /* Create the MPI communicators needed by the subset rearranger. */
-    int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc);
+    int default_subset_partition(int comprank, int iorank, int comptasks, int iotasks, int *color, int *key);
 
     /* Like MPI_Alltoallw(), but with flow control. */
     int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendtypes,
@@ -295,9 +297,10 @@ extern "C" {
     /* Free a region list. */
     void free_region_list(io_region *top);
 
+
     /* Create a subset rearranger. */
     int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, const int *gsize,
-                                int ndim, io_desc_t *iodesc);
+                                int ndim, io_desc_t *iodesc, pio_partition_fn partition_fn);
 
     /* Create a box rearranger. */
     int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *compmap, const int *gsize,

diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c
@@ -2513,9 +2513,10 @@ int initdecomp_dof_handler(iosystem_desc_t *ios)
     if (iocount_present)
         iocountp = iocount;
 
+    /* TODO: support for custom partition functions */
     /* Call the function. */
     PIOc_InitDecomp(iosysid, pio_type, ndims, dims, maplen, compmap, &ioid, rearrangerp,
-                    iostartp, iocountp);
+                    iostartp, iocountp, NULL);
 
     PLOG((1, "PIOc_InitDecomp returned"));
 
@@ -2940,7 +2941,7 @@ int set_loglevel_handler(iosystem_desc_t *ios)
 {
 #if PIO_ENABLE_LOGGING
     int iosysid;
-    int level;
+    int level=0;
     int mpierr;
 #endif