diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 330c865c6..b73739241 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -14,3 +14,4 @@ | harry-shepherd | Harry Shepherd | Met Office | 2026-01-08 | | EdHone | Ed Hone | Met Office | 2026-01-09 | | tom-j-h | Tom Hill | Met Office | 2026-01-19 | +| thomasmelvin | Thomas Melvin | Met Office | 2026-01-15 | diff --git a/components/driver/source/mesh/runtime_partition_mod.f90 b/components/driver/source/mesh/runtime_partition_mod.f90 index 34cd14101..38b662634 100644 --- a/components/driver/source/mesh/runtime_partition_mod.f90 +++ b/components/driver/source/mesh/runtime_partition_mod.f90 @@ -78,15 +78,15 @@ subroutine get_partition_strategy( mesh_selection, total_ranks, partitioner_ptr call log_event( "Using serial cubed sphere partitioner", & log_level_debug ) - else if (mod(total_ranks,6) == 0) then + else if (mod(total_ranks,3) == 0 .or. mod(total_ranks,2) == 0) then ! Paralled run job partitioner_ptr => partitioner_cubedsphere call log_event( "Using parallel cubed sphere partitioner", & log_level_debug ) else - call log_event( "Total number of processors must be 1 (serial) "// & - "or a multiple of 6 for a cubed-sphere domain.", & + call log_event( "Total number of processors must be 1 (serial) "// & + "or a multiple of 2 or 3 for a cubed-sphere domain.", & log_level_error ) end if diff --git a/documentation/source/how_to_use_it/technical_articles/lfric_distmem_impl.rst b/documentation/source/how_to_use_it/technical_articles/lfric_distmem_impl.rst index 14e77aa21..1a4ad0da8 100644 --- a/documentation/source/how_to_use_it/technical_articles/lfric_distmem_impl.rst +++ b/documentation/source/how_to_use_it/technical_articles/lfric_distmem_impl.rst @@ -177,8 +177,12 @@ supplied to the partitioner through the use of a function passed in via a function pointer. For efficiency reasons, the algorithm for partitioning a cubed-sphere -mesh is a specific algorithm that will only work for a cubed-sphere mesh -(each panel of the cubed-sphere is split into rectangular partitions). +mesh is a specific algorithm that will only work for a cubed-sphere mesh. +By default each panel of the cubed-sphere is split into rectangular partitions, +alternatively if the 'custom' decomposition type is used then sets of 2 or 3 +cubed-sphere panels can be grouped together before these sets are decomposed into +a number of rectangular panels. These restrictions mean that the total number of +partitions for cubed-sphere meshes needs to be a multiple of 2 or 3. This specific partitioner could be replaced with a more general partitioner for use on fully unstructured meshes. diff --git a/infrastructure/source/mesh/global_mesh_mod.F90 b/infrastructure/source/mesh/global_mesh_mod.F90 index 178f6d818..1dbcb74f0 100644 --- a/infrastructure/source/mesh/global_mesh_mod.F90 +++ b/infrastructure/source/mesh/global_mesh_mod.F90 @@ -1019,16 +1019,19 @@ end function get_equatorial_latitude !> !> @note For a cubed -sphere mesh, this will only return correct cell IDs if !> the offset cell remains on same the cubed-sphere "face" as the - !> start cell. + !> start cell unless check_orientation is specified. !> !> @param[in] cell_number ID of the anchor element. !> @param[in] x_cells Offset in the E/W direction. !> @param[in] y_cells Offset in the N/S direction. + !> @param[in] check_orientation Switch to check for orientation changes + !! such as when crossing panel boundaries !> !> @return cell_id ID of the cell at the given offset to the start cell. !> function get_cell_id( self, cell_number, & - x_cells, y_cells ) result ( cell_id ) + x_cells, y_cells, & + check_orientation ) result ( cell_id ) use reference_element_mod, only : W, S, E, N @@ -1036,62 +1039,84 @@ function get_cell_id( self, cell_number, & class(global_mesh_type), intent(in) :: self - integer(i_def), intent(in) :: cell_number - integer(i_def), intent(in) :: x_cells, y_cells + integer(i_def), intent(in) :: cell_number + integer(i_def), intent(in) :: x_cells, y_cells + logical(l_def), optional, intent(in) :: check_orientation - integer(i_def) :: cell_id + integer(i_def) :: cell_id, old_cell_id - integer(i_def) :: index_x, dist_x - integer(i_def) :: index_y, dist_y - integer(i_def) :: i + integer(i_def) :: x_index, y_index, x_dist, y_dist, i, j + integer(i_def) :: opposite(4), rotate(4) - cell_id = cell_number + logical(l_def) :: check - ! Determine march along local x-axis - if (x_cells > 0) then - index_x = E - dist_x = x_cells - else if (x_cells < 0) then - index_x = W - dist_x = abs(x_cells) + opposite = (/ E, N, W, S /) + rotate = (/ S, E, N, W /) + if ( present( check_orientation ) ) then + check = check_orientation else - index_x = W - dist_x = 0 + check = .false. end if - ! Determine march along local y-axis - if (y_cells > 0) then - index_y = N - dist_y = y_cells - else if (y_cells < 0) then - index_y = S - dist_y = abs(y_cells) + cell_id=cell_number + + if (x_cells >= 0 )then + x_index = E + x_dist = x_cells + else + x_index = W + x_dist = abs(x_cells) + end if + if (y_cells >= 0 )then + y_index = N + y_dist = y_cells else - index_y = S - dist_y = 0 + y_index = S + y_dist = abs(y_cells) end if - !======================================== - ! March from anchor along local x/y axes. - !======================================== - do i=1, dist_x + ! x_dist cells in the x-direction + do i = 1, x_dist + old_cell_id = cell_id + cell_id = self%cell_next_2d(x_index,cell_id) if (cell_id == self%void_cell) then ! The current cell is not on the domain write(log_scratch_space,'(A)') & 'No adjacent cell, the current cell is not on mesh domain' call log_event(log_scratch_space, LOG_LEVEL_ERROR) end if - cell_id = self%cell_next_2d(index_x, cell_id) + ! Check if we've changed direction + if ( check .and. self%cell_next_2d(opposite(x_index),cell_id) /= old_cell_id ) then + ! We have changed direction so we need to find the correct + ! index and reset + do j = 1,4 + if ( self%cell_next_2d(opposite(j),cell_id) == old_cell_id ) x_index = j + end do + end if end do - - do i=1, dist_y + ! y_dist cells in the y-direction + ! Since the direction may have changed we need to recompute + y_index = rotate(x_index) + if ( y_cells < 0 ) y_index = opposite(y_index) + + ! y_index and y_dist + do i = 1,y_dist + old_cell_id = cell_id + cell_id = self%cell_next_2d(y_index,cell_id) if (cell_id == self%void_cell) then ! The current cell is not on the domain write(log_scratch_space,'(A)') & 'No adjacent cell, the current cell is not on mesh domain' call log_event(log_scratch_space, LOG_LEVEL_ERROR) end if - cell_id = self%cell_next_2d(index_y, cell_id) + ! Check if we've changed direction + if ( check .and. self%cell_next_2d(opposite(y_index),cell_id) /= old_cell_id ) then + ! We have changed direction so we need to find the correct + ! index and reset + do j = 1,4 + if ( self%cell_next_2d(opposite(j),cell_id) == old_cell_id ) y_index = j + end do + end if end do end function get_cell_id diff --git a/infrastructure/source/mesh/panel_decomposition_mod.f90 b/infrastructure/source/mesh/panel_decomposition_mod.f90 index 385651c0a..0c0545722 100644 --- a/infrastructure/source/mesh/panel_decomposition_mod.f90 +++ b/infrastructure/source/mesh/panel_decomposition_mod.f90 @@ -20,6 +20,7 @@ module panel_decomposition_mod type, public, abstract :: panel_decomposition_type contains procedure(get_partition_interface), deferred :: get_partition + procedure(get_nprocs_interface), deferred :: get_nprocs end type panel_decomposition_type !> @brief Decomposition that accepts user specified number of xprocs and yprocs @@ -27,6 +28,7 @@ module panel_decomposition_mod integer(i_def) :: num_xprocs, num_yprocs contains procedure, public :: get_partition => get_custom_partition + procedure, public :: get_nprocs => get_custom_nprocs end type custom_decomposition_type ! Constructor interface custom_decomposition_type @@ -37,24 +39,28 @@ module panel_decomposition_mod type, extends(panel_decomposition_type), public :: auto_decomposition_type contains procedure, public :: get_partition => get_auto_partition + procedure, public :: get_nprocs => get_auto_nprocs end type auto_decomposition_type !> @brief Decomposition only in x direction type, extends(panel_decomposition_type), public :: row_decomposition_type contains procedure, public :: get_partition => get_row_partition + procedure, public :: get_nprocs => get_row_nprocs end type row_decomposition_type !> @brief Decomposition only in y direction type, extends(panel_decomposition_type), public :: column_decomposition_type contains procedure, public :: get_partition => get_column_partition + procedure, public :: get_nprocs => get_column_nprocs end type column_decomposition_type !> @brief Decomposition that automatically generates a nonuniform decomposition type, extends(panel_decomposition_type), public :: auto_nonuniform_decomposition_type contains procedure, public :: get_partition => get_auto_nonuniform_partition + procedure, public :: get_nprocs => get_auto_nonuniform_nprocs end type auto_nonuniform_decomposition_type ! @brief Decomposition that accepts user specified number of xprocs to @@ -63,6 +69,7 @@ module panel_decomposition_mod integer(i_def) :: num_xprocs contains procedure, public :: get_partition => get_guided_nonuniform_partition + procedure, public :: get_nprocs => get_guided_nonuniform_nprocs end type guided_nonuniform_decomposition_type ! Constructor interface guided_nonuniform_decomposition_type @@ -105,6 +112,19 @@ end subroutine get_partition_interface end interface + abstract interface + + function get_nprocs_interface(self) result(nprocs) + use constants_mod, only: i_def + import :: panel_decomposition_type + + class(panel_decomposition_type), intent(in) :: self + + integer(i_def) :: nprocs(2) + end function get_nprocs_interface + + end interface + contains !> @brief Partition the panel into a given number of x and y processes @@ -184,7 +204,6 @@ subroutine get_custom_partition( self, & end subroutine get_custom_partition - !> @brief Constructor for custom_decomposition_type !> @param[in] xprocs The requested number of partitions in the x direction !> @param[in] yprocs The requested number of partitions in the y direction @@ -199,6 +218,19 @@ function custom_decomposition_constructor(xprocs, yprocs) result(self) end function custom_decomposition_constructor + !> @brief Get the number of processors in the x- and y-direction + !> @result nprocs Number of processors (x-dir, y-dir) + function get_custom_nprocs(self) result(nprocs) + use constants_mod, only: i_def + + class(custom_decomposition_type), intent(in) :: self + + integer(i_def) :: nprocs(2) + + nprocs(:) = (/ self%num_xprocs, self%num_yprocs /) + + end function get_custom_nprocs + !> @brief Partition the panel into an automatically determined number of x and ! y processes @@ -239,6 +271,8 @@ subroutine get_auto_partition( self, & partition_x_pos, & partition_y_pos + integer(i_def), parameter :: max_factor_iters = 10000 + integer(i_def) :: num_xprocs, num_yprocs integer(i_def) :: mp_num_cells_x, mp_num_cells_y integer(i_def) :: start_xprocs, start_width, i @@ -328,6 +362,24 @@ subroutine get_auto_partition( self, & end subroutine get_auto_partition + !> @brief Get the number of processors in the x- and y-direction. + !! For this class the function is not needed and so only + !! returns default values + !> @result nprocs Number of processors (x-dir, y-dir) + function get_auto_nprocs(self) result(nprocs) + use constants_mod, only: i_def + + class(auto_decomposition_type), intent(in) :: self + + integer(i_def) :: nprocs(2) + + ! These values aren't needed for auto decomposition + ! (and aren't available until get_auto_partition has been called) + ! so just return something that won't break the code + nprocs(:) = (/ 1, 1 /) + + end function get_auto_nprocs + !> @brief Partition the panel only in the x direction !> @param[in] relative_rank The number of this rank in the order of all @@ -400,6 +452,24 @@ subroutine get_row_partition( self, & end subroutine get_row_partition + !> @brief Get the number of processors in the x- and y-direction. + !! For this class the function is not needed and so only + !! returns default values + !> @result nprocs Number of processors (x-dir, y-dir) + function get_row_nprocs(self) result(nprocs) + use constants_mod, only: i_def + + class(row_decomposition_type), intent(in) :: self + + integer(i_def) :: nprocs(2) + + ! These values aren't needed for row decomposition + ! (and arenn't available until get_row_partition has been called) + ! so just return something that won't break the code + nprocs(:) = (/ 1, 1 /) + + end function get_row_nprocs + !> @brief Partition the panel only in the y direction !> @param[in] relative_rank The number of this rank in the order of all @@ -472,6 +542,24 @@ subroutine get_column_partition( self, & end subroutine get_column_partition + !> @brief Get the number of processors in the x- and y-direction. + !! For this class the function is not needed and so only + !! returns default values + !> @result nprocs Number of processors (x-dir, y-dir) + function get_column_nprocs(self) result(nprocs) + use constants_mod, only: i_def + + class(column_decomposition_type), intent(in) :: self + + integer(i_def) :: nprocs(2) + + ! These values aren't needed for column decomposition + ! (and aren't available until get_column_partition has been called) + ! so just return something that won't break the code + nprocs(:) = (/ 1, 1 /) + + end function get_column_nprocs + !> @brief Partition the panel into an automatically determined number of ! columns of partitions of variable size. !> @param[in] relative_rank The number of this rank in the order of all @@ -570,6 +658,24 @@ subroutine get_auto_nonuniform_partition( self, & end subroutine get_auto_nonuniform_partition + !> @brief Get the number of processors in the x- and y-direction. + !! For this class the function is not needed and so only + !! returns default values + !> @result nprocs Number of processors (x-dir, y-dir) + function get_auto_nonuniform_nprocs(self) result(nprocs) + use constants_mod, only: i_def + + class(auto_nonuniform_decomposition_type), intent(in) :: self + + integer(i_def) :: nprocs(2) + + ! These values aren't needed for auto_nonuniform decomposition + ! (and aren't available until get_auto_nonuniform_partition has been called) + ! so just return something that won't break the code + nprocs(:) = (/ 1, 1 /) + + end function get_auto_nonuniform_nprocs + !> @brief Partition the panel into a given number of columns of partitions of ! variable size. !> @param[in] relative_rank The number of this rank in the order of all @@ -661,6 +767,24 @@ function guided_nonuniform_decomposition_constructor( xprocs ) result(self) end function guided_nonuniform_decomposition_constructor + !> @brief Get the number of processors in the x- and y-direction. + !! For this class the function is not needed and so only + !! returns default values + !> @result nprocs Number of processors (x-dir, y-dir) + function get_guided_nonuniform_nprocs(self) result(nprocs) + use constants_mod, only: i_def + + class(guided_nonuniform_decomposition_type), intent(in) :: self + + integer(i_def) :: nprocs(2) + + ! These values aren't needed for guided_nonuniform decomposition + ! (and aren't available until get_guided_nonuniform_partition has been called) + ! so just return something that won't break the code + nprocs(:) = (/ 1, 1 /) + + end function get_guided_nonuniform_nprocs + !> @brief Helper function for generating identical partitions arranged in a ! rectangular grid diff --git a/infrastructure/source/mesh/partition_mod.F90 b/infrastructure/source/mesh/partition_mod.F90 index 3348427a6..a9ff9097e 100644 --- a/infrastructure/source/mesh/partition_mod.F90 +++ b/infrastructure/source/mesh/partition_mod.F90 @@ -25,6 +25,7 @@ module partition_mod use sort_mod, only : bubble_sort use log_mod, only : log_event, & log_scratch_space, & + LOG_LEVEL_INFO, & LOG_LEVEL_ERROR, & LOG_LEVEL_DEBUG use constants_mod, only: i_def, r_def, l_def @@ -524,11 +525,6 @@ subroutine partitioner_cubedsphere( global_mesh, & ! A cubed sphere has 6 panels num_panels = 6 - !check that we have a number of ranks that is compatible with this partitioner - if( modulo(total_ranks,num_panels) /= 0 ) call log_event( & - 'The cubed-sphere partitioner requires a multiple of six processors.', & - LOG_LEVEL_ERROR ) - call partitioner_rectangular_panels( global_mesh, & num_panels, & decomposition, & @@ -676,18 +672,20 @@ subroutine partitioner_rectangular_panels( global_mesh, & ! but not in the partitioned domain logical(l_def), intent(in) :: generate_inner_halos ! Flag to control the generation of inner halos - integer(i_def) :: face ! which face of the cube is implied by local_rank (0->5) - integer(i_def) :: start_cell ! lowest cell id of the face implaced by local_rank - integer(i_def) :: start_rank ! The number of the first rank on the face implied by local_rank - integer(i_def) :: panel_ranks! The number of ranks per panel on the mesh - integer(i_def) :: relative_rank ! The position of the current rank relative to the first rank in its panel - integer(i_def) :: start_x ! global cell id of start of the domain on this partition in x-dirn - integer(i_def) :: num_x ! number of cells in the domain on this partition in x-dirn - integer(i_def) :: start_y ! global cell id of start of the domain on this partition in y-dirn - integer(i_def) :: num_y ! number of cells in the domain on this partition in y-dirn - integer(i_def) :: ix, iy ! loop counters over cells on this partition in x- and y-dirns - integer(i_def) :: void_cell ! Cell id that marks the cell as a cell outside of the partition. - logical :: any_maps ! Whether there exist maps between meshes, meaning their partitions must align. + integer(i_def) :: face ! which face of the cube is implied by local_rank (0->5) + integer(i_def) :: start_cell ! lowest cell id of the face implaced by local_rank + integer(i_def) :: start_rank ! The number of the first rank on the face implied by local_rank + integer(i_def) :: panel_ranks ! The number of ranks per panel on the mesh + integer(i_def) :: relative_rank ! The position of the current rank relative to the first rank in its panel + integer(i_def) :: start_x ! global cell id of start of the domain on this partition in x-dirn + integer(i_def) :: num_x ! number of cells in the domain on this partition in x-dirn + integer(i_def) :: start_y ! global cell id of start of the domain on this partition in y-dirn + integer(i_def) :: num_y ! number of cells in the domain on this partition in y-dirn + integer(i_def) :: ix, iy ! loop counters over cells on this partition in x- and y-dirns + integer(i_def) :: void_cell ! Cell id that marks the cell as a cell outside of the partition. + logical :: any_maps ! Whether there exist maps between meshes, meaning their partitions must align. + integer(i_def) :: start1, end1, inc1 ! Loop indices for inserting cells into the partition + integer(i_def) :: start2, end2, inc2 ! Loop indices for inserting cells into the partition ! Create linked lists @@ -699,26 +697,38 @@ subroutine partitioner_rectangular_panels( global_mesh, & type(linked_list_item_type), pointer :: insert_point ! where to insert in a list type(linked_list_item_type), pointer :: loop ! temp ptr to loop through list - integer :: i, j ! loop counters - integer :: cells(4) ! The cells around the vertex being queried - integer :: oth1, oth2 ! When querying a cell around a vertex, these are - ! the indices of the other two cells + integer :: i, j ! loop counters + integer :: cells(4) ! The cells around the vertex being queried + integer :: oth1, oth2 ! When querying a cell around a vertex, these are + ! the indices of the other two cells integer, allocatable :: sw_corner_cells(:) - ! List of cells at the SW corner of the panels - integer :: panel ! panel number - integer :: cell ! starting point for num_cells_x calculation - integer :: cell_next(4) ! The cells around the cell being queried - integer :: cell_next_e ! The cell to the east of the cell being queried - integer :: num_cells_x ! number of cells across a panel in x-direction - integer :: num_cells_y ! number of cells across a panel in y-direction + ! List of cells at the SW corner of the panels + integer :: panel ! panel number + integer :: cell ! starting point for num_cells_x calculation + integer :: cell_next(4) ! The cells around the cell being queried + integer :: cell_next_e ! The cell to the east of the cell being queried + integer :: num_cells_x ! number of cells across a panel in x-direction + integer :: num_cells_y ! number of cells across a panel in y-direction integer :: start_sort, end_sort ! range over which to sort cells - integer :: depth ! counter over the halo depths - integer :: orig_num_in_list ! number of cells in list before halos are added + integer :: depth ! counter over the halo depths + integer :: orig_num_in_list ! number of cells in list before halos are added + integer :: ncell ! Number of cells + integer :: cell_id ! Id of cell in the partition + logical :: cross_panels ! Flag for partitioning across multiple cubed sphere panels + logical :: check_orientation ! Check for orientation changes when getting cell id's + integer :: n_cross_panels ! number of panels to decompose across + integer, allocatable :: face_of_combined_panels(:) + ! The first face in sections of combined cubed sphere panels + integer :: nprocs(2) ! number of processors in the x- & y-direction + integer :: xproc ! number of processsors in x-direction + integer :: yproc ! number of processsors in y-direction integer(i_def) :: num_apply logical(l_def) :: periodic_xy(2) ! Periodic in the x/y-axes periodic_xy = global_mesh%get_mesh_periodicity() void_cell = global_mesh%get_void_cell() + cross_panels = .false. + n_cross_panels = 1 any_maps = global_mesh%get_nmaps() > 0 nullify( last ) @@ -726,6 +736,10 @@ subroutine partitioner_rectangular_panels( global_mesh, & nullify( insert_point ) nullify( loop ) + nprocs = decomposition%get_nprocs() + xproc = nprocs(1) + yproc = nprocs(2) + if (num_panels==1) then ! A single panelled mesh might be rectangluar - so find the dimensions ! First determine the southwest corner cell depending on periodicity. If @@ -780,12 +794,55 @@ subroutine partitioner_rectangular_panels( global_mesh, & num_cells_y=global_mesh%get_ncells()/num_cells_x else - ! For multi-panel meshes, the panels must be square - num_cells_x = nint(sqrt( real(global_mesh%get_ncells(), kind=r_def)/ & - real(num_panels, kind=r_def) )) - num_cells_y = num_cells_x + if( num_panels == 6 .and. & + xproc*yproc*2 == total_ranks .and. & + modulo(total_ranks,2) == 0 ) then + ncell = nint(sqrt( real(global_mesh%get_ncells(), kind=r_def)/ & + real(num_panels, kind=r_def) )) + ! We will partition across 3 panels giving 2 'super' panels of 3 cubed-sphere panels each + cross_panels = .true. + n_cross_panels = 2 + num_cells_x = ncell * 3 + num_cells_y = ncell + allocate( face_of_combined_panels(n_cross_panels) ) + ! Super panels consist of faces (1, 2, 3), (6, 4, 5) + ! In terms of the panel numbers used here is then + ! (3, 2, 6), (5, 4, 1) + face_of_combined_panels(:) = (/ 3, 5 /) + elseif( num_panels == 6 .and. & + xproc*yproc*3 == total_ranks .and. & + modulo(total_ranks,3) == 0 ) then + ncell = nint(sqrt( real(global_mesh%get_ncells(), kind=r_def)/ & + real(num_panels, kind=r_def) )) + ! We will partition across 2 panels giving 3 'super' panels of 2 cubed-sphere panels each + cross_panels = .true. + n_cross_panels = 3 + num_cells_x = ncell * 2 + num_cells_y = ncell + allocate( face_of_combined_panels(n_cross_panels) ) + ! Super panels consist of faces (1, 2), (4,5) & (6,3) + ! In terms of the panel numbers used here is then + ! (3, 2), (4, 1), (5, 6) + face_of_combined_panels(:) = (/ 3, 4, 5 /) + elseif( modulo(total_ranks,num_panels) == 0 ) then + ! For multi-panel meshes, the panels must be square + num_cells_x = nint(sqrt( real(global_mesh%get_ncells(), kind=r_def)/ & + real(num_panels, kind=r_def) )) + num_cells_y = num_cells_x + else + write(log_scratch_space,*) 'Unable to find a partition strategy. Total ranks (',& + total_ranks,') needs to either have a factor of ', num_panels, & + ' or if num_panels = 6 and custom decomposition is used then total ranks needs to have a factor of 2 or 3' + call log_event(log_scratch_space, LOG_LEVEL_ERROR) + end if ! Calculate the South West corner cells of all the panels in the global mesh + ! Note the 'panel' numbers used here do not correspond to the panel numbers + ! used in the rest of the model as this uses the vertex index to find + ! the cell in the corner of each panel and it is not true that the first corner + ! cell found in this manner will correspond to panel 1. + ! In fact for the cubed sphere meshes used the relationship is: + ! SW corner cells are on panels: (5, 2, 1, 4, 6, 3) allocate(sw_corner_cells(num_panels)) panel=1 do i=1,global_mesh%get_nverts() @@ -812,11 +869,18 @@ subroutine partitioner_rectangular_panels( global_mesh, & endif - face = ((num_panels * local_rank) / total_ranks) + 1 - panel_ranks = total_ranks / num_panels + if ( cross_panels ) then + face = ((n_cross_panels * local_rank) / total_ranks) + 1 + panel_ranks = total_ranks / n_cross_panels + else + face = ((num_panels * local_rank) / total_ranks) + 1 + panel_ranks = total_ranks / num_panels + end if start_rank = panel_ranks * (face - 1) relative_rank = local_rank - start_rank + 1 + ! Work out the start index and number of cells (in x- and y-dirn) for + ! the local partition call decomposition%get_partition( relative_rank, & panel_ranks, & mapping_factor, & @@ -829,7 +893,41 @@ subroutine partitioner_rectangular_panels( global_mesh, & start_y ) - start_cell = sw_corner_cells(face) + + ! Set up limits for finding cells + start1 = start_x + end1 = start_x + num_x - 1 + inc1 = 1 + start2 = start_y + end2 = start_y + num_y - 1 + inc2 = 1 + + if ( cross_panels ) then + ! Generally we want the start cell to be the SW corner cell + ! but there is an exception when we are spanning 3 panels and looking + ! at cubed sphere panels (6,4,5) when we want to start in the NW corner. + ! Cubed sphere panels (6,4,5) correspond to the indices (5, 4, 1) of the + ! sw_corner_cells array since the sw_corner_cells array indices (1,..,6) do not + ! correspond to cubed sphere panels (1,..,6) and are in fact + ! ordered (5, 2, 1, 4, 6, 3) + start_cell = sw_corner_cells(face_of_combined_panels(face)) + if (n_cross_panels == 2 .and. start_cell == sw_corner_cells(5) ) then + ! Start in the NW corner and go in (-y, x) direction + start_cell = global_mesh%get_cell_id(start_cell, 0, num_cells_y-1) + + num_y = num_cells_x / xproc + num_x = num_cells_y / yproc + start1 = start_y + end1 = start1 + num_x - 1 + inc1 = 1 + start2 = -start_x + 2 + end2 = start2 - num_y + 1 + inc2 = -1 + end if + deallocate( face_of_combined_panels ) + else + start_cell = sw_corner_cells(face) + end if deallocate(sw_corner_cells) write(log_scratch_space,"(a,i0,a,i0,a,i0,a,i0)") "start_x ", start_x, & @@ -837,53 +935,38 @@ subroutine partitioner_rectangular_panels( global_mesh, & " num_x ", num_x, & " num_y ", num_y call log_event( log_scratch_space, LOG_LEVEL_DEBUG ) + write(log_scratch_space,"(a,i0,a,i0)") "Number of cells in partition ", num_x, " X ", num_y + call log_event( log_scratch_space, lOG_LEVEL_INFO ) - ! Create a linked list of all cells that are part of this partition (not halos) - + ! Create a linked list of all cells in the partition and at the same time + ! create a linked-list of all edge cells known to the partition, excluding halos. partition = linked_list_type() + known_cells = linked_list_type() + + ! We only need to check for orientation changes in the partition + ! covers multiple panels + check_orientation = cross_panels + + do iy = start2, end2, inc2 + do ix = start1, end1, inc1 + cell_id = global_mesh%get_cell_id(start_cell, ix-1, iy-1, & + check_orientation) + call partition%insert_item( linked_list_int_type( cell_id ) ) + + ! If this is an edge cell then add it to the list of known edge cells + ! (if it is not already there) + if ( ix == start1 .or. ix == end1 .or. iy == start2 .or. iy == end2 ) then + if ( .not. known_cells%item_exists(cell_id) ) then + call known_cells%insert_item( linked_list_int_type( cell_id ) ) + end if + end if - do iy = start_y, start_y+num_y - 1 - do ix = start_x, start_x+num_x - 1 - call partition%insert_item( linked_list_int_type( & - global_mesh%get_cell_id(start_cell, ix-1, iy-1))) end do end do ! Create a linked-list of all cells known to the partition, including halos. ! This will be ordered as: ! inner n, inner n-1 ... inner 1, edge, halo 1 ... halo n-1, halo n - ! - ! Start with the edge cells - those cells owned by the partition - but are on - ! the edge of the partitioned domain, so may have dofs shared with halo cells - - known_cells = linked_list_type() - - ! Those cells along the top/bottom - - do ix = start_x, start_x+num_x-1 - ! start inserting the edge cells - call known_cells%insert_item(linked_list_int_type( & - global_mesh%get_cell_id(start_cell, ix-1, start_y-1))) - ! insert but check for duplicates between start and end of known_cells list - if(.not. (known_cells%item_exists(global_mesh%get_cell_id( & - start_cell, ix-1, start_y+num_y-2)) ) ) then - call known_cells%insert_item( linked_list_int_type( & - global_mesh%get_cell_id(start_cell, ix-1, start_y+num_y-2))) - end if - end do - ! Those along the left/right - do iy = start_y+1, start_y+num_y-2 - if(.not. (known_cells%item_exists(global_mesh%get_cell_id( & - start_cell, start_x-1, iy-1)) )) then - call known_cells%insert_item( linked_list_int_type( & - global_mesh%get_cell_id(start_cell, start_x-1, iy-1))) - end if - if(.not. (known_cells%item_exists(global_mesh%get_cell_id( & - start_cell, start_x+num_x-2, iy-1)) )) then - call known_cells%insert_item( linked_list_int_type( & - global_mesh%get_cell_id(start_cell, start_x+num_x-2, iy-1))) - end if - end do ! get the number of edge cells currently stored in the known_cells list num_edge = known_cells%get_length() diff --git a/infrastructure/unit-test/mesh/panel_decomposition_mod_test.pf b/infrastructure/unit-test/mesh/panel_decomposition_mod_test.pf new file mode 100644 index 000000000..7f60ae364 --- /dev/null +++ b/infrastructure/unit-test/mesh/panel_decomposition_mod_test.pf @@ -0,0 +1,87 @@ +!----------------------------------------------------------------------------- +! Copyright (c) 2026, Met Office, on behalf of HMSO and Queen's Printer +! For further details please refer to the file LICENCE which you +! should have received as part of this distribution. +!----------------------------------------------------------------------------- + +!> Test the panel decomposition module +module panel_decomposition_mod_test + + use pfunit + use constants_mod, only: i_def + use panel_decomposition_mod, only: custom_decomposition_type, & + auto_decomposition_type, & + row_decomposition_type, & + column_decomposition_type, & + auto_nonuniform_decomposition_type, & + guided_nonuniform_decomposition_type + + implicit none + + private + public :: test_all + +contains + + @test + subroutine test_all + + implicit none + + type(custom_decomposition_type) :: custom_decomposition + type(auto_decomposition_type) :: auto_decomposition + type(row_decomposition_type) :: row_decomposition + type(column_decomposition_type) :: column_decomposition + type(auto_nonuniform_decomposition_type) :: auto_nonuniform_decomposition + type(guided_nonuniform_decomposition_type) :: guided_nonuniform_decomposition + + integer(i_def), parameter :: xprocs = 3 + integer(i_def), parameter :: yprocs = 4 + integer(i_def) :: nprocs(2) + + ! Test the custom decomposition type + custom_decomposition = custom_decomposition_type(xprocs, yprocs) + + nprocs = custom_decomposition%get_nprocs() + @assertEqual( xprocs, nprocs(1) ) + @assertEqual( yprocs, nprocs(2) ) + + ! Test the auto decomposition type + auto_decomposition = auto_decomposition_type() + + nprocs = auto_decomposition%get_nprocs() + @assertEqual( 1, nprocs(1) ) + @assertEqual( 1, nprocs(2) ) + + ! Test the row decomposition type + row_decomposition = row_decomposition_type() + + nprocs = row_decomposition%get_nprocs() + @assertEqual( 1, nprocs(1) ) + @assertEqual( 1, nprocs(2) ) + + ! Test the column decomposition type + column_decomposition = column_decomposition_type() + + nprocs = column_decomposition%get_nprocs() + @assertEqual( 1, nprocs(1) ) + @assertEqual( 1, nprocs(2) ) + + ! Test the auto_nonuniform decomposition type + auto_nonuniform_decomposition = auto_nonuniform_decomposition_type() + + nprocs = auto_nonuniform_decomposition%get_nprocs() + @assertEqual( 1, nprocs(1) ) + @assertEqual( 1, nprocs(2) ) + + ! Test the guided_nonuniform decomposition type + guided_nonuniform_decomposition = guided_nonuniform_decomposition_type(xprocs) + + nprocs = guided_nonuniform_decomposition%get_nprocs() + @assertEqual( 1, nprocs(1) ) + @assertEqual( 1, nprocs(2) ) + + end subroutine test_all + +end module panel_decomposition_mod_test + diff --git a/mesh_tools/source/support/gencube_ps_mod.F90 b/mesh_tools/source/support/gencube_ps_mod.F90 index b1bb088f1..3620b38ca 100644 --- a/mesh_tools/source/support/gencube_ps_mod.F90 +++ b/mesh_tools/source/support/gencube_ps_mod.F90 @@ -2050,9 +2050,28 @@ subroutine set_partition_parameters( decomposition, partitioner_ptr ) call log_event( "Using parallel cubed sphere partitioner", & LOG_LEVEL_INFO ) + else if( NPANELS == 6 .and. & + (mod(n_partitions, 3) == 0) .or. & + (mod(n_partitions, 2) == 0 ) ) then + ! Use the parallel cubed-sphere partitioner + partitioner_ptr => partitioner_cubedsphere + + select case(panel_decomposition) + case( panel_decomposition_custom ) + decomposition = custom_decomposition_type( panel_xproc, panel_yproc ) + case default + call log_event( "Decomposing across 2 or 3 panels requires "// & + "'custom' decomposition.", LOG_LEVEL_ERROR ) + end select + + call log_event( "Using parallel cubed sphere partitioner", & + LOG_LEVEL_INFO ) + else - call log_event( "Number of partitions must be 1 "// & - "or a multiple of the number of panels", & + call log_event( "Number of partitions must be 1 "// & + "or a multiple of the number of panels "// & + "or a multiple of 2 or 3 for 6 panels "// & + "and using 'custom' decomposition", & LOG_LEVEL_ERROR ) end if diff --git a/rose-stem/app/mesh_tools/opt/rose-app-cubedsphere-op-2panels.conf b/rose-stem/app/mesh_tools/opt/rose-app-cubedsphere-op-2panels.conf new file mode 100644 index 000000000..b9fa6b5b1 --- /dev/null +++ b/rose-stem/app/mesh_tools/opt/rose-app-cubedsphere-op-2panels.conf @@ -0,0 +1,22 @@ +[env] +mesh_generator=Cubed-Sphere + +[namelist:cubedsphere_mesh] +edge_cells=6,3 + +[namelist:mesh] +coord_sys='ll' +geometry='spherical' +mesh_maps='C6:C3' +mesh_names='C6','C3' +partition_mesh=.true. +rotate_mesh=.false. + +[namelist:partitions] +n_partitions=2 +panel_decomposition='custom' +panel_xproc=1 +panel_yproc=1 +partition_range=0,1 + +[!!namelist:planar_mesh] diff --git a/rose-stem/app/mesh_tools/opt/rose-app-cubedsphere-op-3panels.conf b/rose-stem/app/mesh_tools/opt/rose-app-cubedsphere-op-3panels.conf new file mode 100644 index 000000000..e7ddad949 --- /dev/null +++ b/rose-stem/app/mesh_tools/opt/rose-app-cubedsphere-op-3panels.conf @@ -0,0 +1,22 @@ +[env] +mesh_generator=Cubed-Sphere + +[namelist:cubedsphere_mesh] +edge_cells=6,3 + +[namelist:mesh] +coord_sys='ll' +geometry='spherical' +mesh_maps='C6:C3' +mesh_names='C6','C3' +partition_mesh=.true. +rotate_mesh=.false. + +[namelist:partitions] +n_partitions=3 +panel_decomposition='custom' +panel_xproc=1 +panel_yproc=1 +partition_range=0,2 + +[!!namelist:planar_mesh] diff --git a/rose-stem/site/common/mesh_tools/tasks_mesh_tools.cylc b/rose-stem/site/common/mesh_tools/tasks_mesh_tools.cylc index a2a18705e..8e4e6d850 100644 --- a/rose-stem/site/common/mesh_tools/tasks_mesh_tools.cylc +++ b/rose-stem/site/common/mesh_tools/tasks_mesh_tools.cylc @@ -20,7 +20,9 @@ "opt_confs": task_ns.conf_name }) %} -{% elif task_ns.conf_name == "cubedsphere-op" %} +{% elif task_ns.conf_name == "cubedsphere-op" or + task_ns.conf_name == "cubedsphere-op-2panels" or + task_ns.conf_name == "cubedsphere-op-3panels" %} {% do task_dict.update({ "resolution": "", diff --git a/rose-stem/site/meto/groups/group_mesh_tools.cylc b/rose-stem/site/meto/groups/group_mesh_tools.cylc index d3bee961e..64e88682f 100644 --- a/rose-stem/site/meto/groups/group_mesh_tools.cylc +++ b/rose-stem/site/meto/groups/group_mesh_tools.cylc @@ -49,6 +49,8 @@ "mesh_tools_cubedsphere-c2_azspice_gnu_fast-debug-64bit", "mesh_tools_cubedsphere-c3_azspice_gnu_fast-debug-64bit", "mesh_tools_cubedsphere-op_azspice_gnu_fast-debug-64bit", + "mesh_tools_cubedsphere-op-2panels_azspice_gnu_fast-debug-64bit", + "mesh_tools_cubedsphere-op-3panels_azspice_gnu_fast-debug-64bit", "mesh_tools_cubedsphere-op-nonuniform_azspice_gnu_fast-debug-64bit", "mesh_tools_cubedsphere-rotated_azspice_gnu_fast-debug-64bit", "mesh_tools_cubedsphere-maps_azspice_gnu_fast-debug-64bit", @@ -59,6 +61,8 @@ "mesh_tools_cubedsphere-c2_azspice_gnu_full-debug-64bit", "mesh_tools_cubedsphere-c3_azspice_gnu_full-debug-64bit", "mesh_tools_cubedsphere-op_azspice_gnu_full-debug-64bit", + "mesh_tools_cubedsphere-op-2panels_azspice_gnu_full-debug-64bit", + "mesh_tools_cubedsphere-op-3panels_azspice_gnu_full-debug-64bit", "mesh_tools_cubedsphere-op-nonuniform_azspice_gnu_full-debug-64bit", "mesh_tools_cubedsphere-rotated_azspice_gnu_full-debug-64bit", "mesh_tools_cubedsphere-maps_azspice_gnu_full-debug-64bit", @@ -127,6 +131,8 @@ "mesh_tools_cubedsphere-c2_ex1a_cce_full-debug-64bit", "mesh_tools_cubedsphere-c3_ex1a_gnu_fast-debug-64bit", "mesh_tools_cubedsphere-op_ex1a_gnu_fast-debug-64bit", + "mesh_tools_cubedsphere-op-2panels_ex1a_gnu_fast-debug-64bit", + "mesh_tools_cubedsphere-op-3panels_ex1a_gnu_fast-debug-64bit", "mesh_tools_cubedsphere-op-nonuniform_ex1a_gnu_fast-debug-64bit", "mesh_tools_cubedsphere-maps_ex1a_gnu_fast-debug-64bit", "mesh_tools_cubedsphere_ex1a_cce_fast-debug-64bit", diff --git a/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-2panels_0-2.gnu.kgo.nc b/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-2panels_0-2.gnu.kgo.nc new file mode 100644 index 000000000..3b5b015a1 Binary files /dev/null and b/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-2panels_0-2.gnu.kgo.nc differ diff --git a/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-2panels_1-2.gnu.kgo.nc b/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-2panels_1-2.gnu.kgo.nc new file mode 100644 index 000000000..2bc19f124 Binary files /dev/null and b/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-2panels_1-2.gnu.kgo.nc differ diff --git a/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-3panels_0-3.gnu.kgo.nc b/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-3panels_0-3.gnu.kgo.nc new file mode 100644 index 000000000..93d6526e9 Binary files /dev/null and b/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-3panels_0-3.gnu.kgo.nc differ diff --git a/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-3panels_1-3.gnu.kgo.nc b/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-3panels_1-3.gnu.kgo.nc new file mode 100644 index 000000000..f69b9ec66 Binary files /dev/null and b/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-3panels_1-3.gnu.kgo.nc differ diff --git a/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-3panels_2-3.gnu.kgo.nc b/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-3panels_2-3.gnu.kgo.nc new file mode 100644 index 000000000..ddefc38ce Binary files /dev/null and b/rose-stem/site/meto/kgos/mesh_tools/azspice/mesh_cubedsphere-op-3panels_2-3.gnu.kgo.nc differ diff --git a/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-2panels_0-2.gnu.kgo.nc b/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-2panels_0-2.gnu.kgo.nc new file mode 100644 index 000000000..b2dae2f3f Binary files /dev/null and b/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-2panels_0-2.gnu.kgo.nc differ diff --git a/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-2panels_1-2.gnu.kgo.nc b/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-2panels_1-2.gnu.kgo.nc new file mode 100644 index 000000000..8fc6536ff Binary files /dev/null and b/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-2panels_1-2.gnu.kgo.nc differ diff --git a/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-3panels_0-3.gnu.kgo.nc b/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-3panels_0-3.gnu.kgo.nc new file mode 100644 index 000000000..1668204d3 Binary files /dev/null and b/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-3panels_0-3.gnu.kgo.nc differ diff --git a/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-3panels_1-3.gnu.kgo.nc b/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-3panels_1-3.gnu.kgo.nc new file mode 100644 index 000000000..a8c3b6cd4 Binary files /dev/null and b/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-3panels_1-3.gnu.kgo.nc differ diff --git a/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-3panels_2-3.gnu.kgo.nc b/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-3panels_2-3.gnu.kgo.nc new file mode 100644 index 000000000..7de8f74a5 Binary files /dev/null and b/rose-stem/site/meto/kgos/mesh_tools/ex1a/mesh_cubedsphere-op-3panels_2-3.gnu.kgo.nc differ