Revert obcount runtime config on branch-25.10 (#3034)

Author: marcinz

.github/workflows/gh-gasnet.yml

@@ -16,7 +16,7 @@ jobs:
     name: Build Gasnet wrapper
     strategy:
       fail-fast: false
-    runs-on: linux-amd64-gpu-l4-latest-1
+    runs-on: linux-amd64-cpu4
     timeout-minutes: 15
     container:
       options: -u root

.github/workflows/gh-mpi.yml

@@ -16,7 +16,7 @@ jobs:
     name: Build MPI wrapper
     strategy:
       fail-fast: false
-    runs-on: linux-amd64-gpu-l4-latest-1
+    runs-on: linux-amd64-cpu4
     timeout-minutes: 15
     container:
       options: -u root

conda/gasnet_wrapper/build-gex-wrapper.sh

@@ -8,6 +8,10 @@ set -eo pipefail
 readonly DEFAULT_CONDUIT="ofi"
 readonly DEFAULT_SYSTEM_CONFIG="slingshot11"
 readonly DEFAULT_CUDA="ON"
+# Threading suffix used by GASNet archives (libgasnet-<conduit>-<thread>.a)
+readonly DEFAULT_THREADING="par"
+# Pinned GASNet commit (matches what we validated on Perlmutter)
+readonly GASNET_GITREF_SHA="e0af36ac9d3d632824be1851bcb3bc23bf05e489"
 
 # Determine script directory dynamically
 readonly SCRIPT_DIR="${CONDA_PREFIX}/gex-wrapper"
@@ -16,22 +20,25 @@ readonly SCRIPT_DIR="${CONDA_PREFIX}/gex-wrapper"
 conduit="${DEFAULT_CONDUIT}"
 system_config="${DEFAULT_SYSTEM_CONFIG}"
 cuda="${DEFAULT_CUDA}"
+threading="${DEFAULT_THREADING}"
+extra_linker_flags=""
 
 # Help function to display usage
 gex_wrapper_help() {
-   echo "Usage: build-gex-wrapper [-h | --help] [-c conduit | --conduit conduit] [-s system_config | --system_config system_config] [-u ON/OFF | --use-cuda ON/OFF]"
+   echo "Usage: build-gex-wrapper [-h | --help] [-c conduit | --conduit conduit] [-s system_config | --system_config system_config] [-u ON/OFF | --use-cuda ON/OFF] [-f flags | --linker-flags \"<flags>\"]"
    echo "Build the Realm GASNet-EX wrapper in your conda environment."
    echo
    echo "Options:"
    echo "  -h, --help               Display this help and exit"
-   echo "  -c, --conduit CONDUIT     Specify the GASNet conduit to use (default '${DEFAULT_CONDUIT}')"
-   echo "  -s, --system_config SYS   Specify the system-specific configuration (default '${DEFAULT_SYSTEM_CONFIG}')"
+   echo "  -c, --conduit CONDUIT     GASNet conduit to use (default '${DEFAULT_CONDUIT}')"
+   echo "  -s, --system_config SYS   System-specific configuration (default '${DEFAULT_SYSTEM_CONFIG}')"
    echo "  -u, --use-cuda ON/OFF     Enable (ON) or disable (OFF) CUDA (default '${DEFAULT_CUDA}')"
+   echo "  -f, --linker-flags STR    Extra linker flags to append (default '-lhugetlbfs' when conduit='ofi' and system='slingshot11')"
    echo
 }
 
 # Parse command-line options (supporting both single-dash and double-dash)
-ARGS=$(getopt -o hc:s:u: -l help,conduit:,system_config:,use-cuda: -- "$@") || {
+ARGS=$(getopt -o hc:s:u:f: -l help,conduit:,system_config:,use-cuda:,linker-flags: -- "$@") || {
   gex_wrapper_help
   exit 1
 }
@@ -59,6 +66,10 @@ while true; do
       fi
       shift 2
       ;;
+    -f | --linker-flags)
+      extra_linker_flags="$2"
+      shift 2
+      ;;
     --)
       shift
       break
@@ -71,11 +82,16 @@ while true; do
   esac
 done
 
+# Default linker flags for Perlmutter OFI/slingshot11, unless overridden
+if [[ -z "${extra_linker_flags}" && "${conduit}" == "ofi" && "${system_config}" == "slingshot11" ]]; then
+  extra_linker_flags="-lhugetlbfs"
+fi
+
 # Ensure CONDA_PREFIX is set
 if [[ -z "${CONDA_PREFIX}" ]]; then
   echo "Error: Please activate a conda environment before running this script."
   echo "Run:"
-  echo "  \$ conda activate <your-env-name>"
+  echo "  $ conda activate <your-env-name>"
   echo "Then re-run this script."
   exit 1
 fi
@@ -84,7 +100,7 @@ fi
 if ! command -v cmake &>/dev/null; then
   echo "Error: cmake is not installed or not in PATH."
   echo "Please install it via your package manager or conda:"
-  echo "  \$ conda install -c conda-forge cmake"
+  echo "  $ conda install -c conda-forge cmake"
   exit 1
 fi
 
@@ -110,19 +126,35 @@ CMAKE_ARGS=(
   -DGASNet_CONDUIT="${conduit}"
   -DGASNet_SYSTEM="${system_config}"
   -DGEX_WRAPPER_BUILD_SHARED=ON
+  -DGASNet_GITREF="${GASNET_GITREF_SHA}"
 )
 
 if [[ "${cuda}" == "ON" ]]; then
   CMAKE_ARGS+=(-DGASNet_CONFIGURE_ARGS="--enable-kind-cuda-uva")
 fi
 
+# Whole-archive embed of the conduit archive into the wrapper DSO.
+# Note: libgasnet-<conduit>-par.a already contains gasnet_tools, so do NOT also
+# link libgasnet_tools-par.a to avoid duplicate symbols.
+GASNET_LIBDIR_EMBED="${SCRIPT_DIR}/src/build/embed-gasnet/install/lib"
+MAIN_A="${GASNET_LIBDIR_EMBED}/libgasnet-${conduit}-${threading}.a"
+LINK_FLAGS=("-Wl,--whole-archive,${MAIN_A},-no-whole-archive")
+
+if [[ -n "${extra_linker_flags}" ]]; then
+  read -r -a extra_linker_flags_array <<< "${extra_linker_flags}"
+  LINK_FLAGS+=("${extra_linker_flags_array[@]}")
+fi
+
+CMAKE_ARGS+=(-DCMAKE_SHARED_LINKER_FLAGS="${LINK_FLAGS[*]}")
+
 cmake "${CMAKE_ARGS[@]}" ..
 cmake --build .
 cmake --install .
 
 echo
 echo "Reactivate the conda environment to set necessary environment variables:"
 echo
-echo "  \$ conda deactivate"
 # shellcheck disable=SC2154
-echo "  \$ conda activate ${CONDA_DEFAULT_ENV}"
+echo "  $ conda deactivate"
+# shellcheck disable=SC2154
+echo "  $ conda activate ${CONDA_DEFAULT_ENV}"

conda/gasnet_wrapper/meta.yaml

@@ -19,7 +19,7 @@ source:
 
 build:
   include_recipe: false
-  number: 18
+  number: 22
   skip: true  # [not linux]
   noarch: generic
   script_env:

conda/mpi_wrapper/meta.yaml

@@ -15,7 +15,7 @@ source:
 
 build:
   include_recipe: false
-  number: 19
+  number: 20
   skip: true  # [not linux]
   noarch: generic
   script_env:

share/legate/mpi_wrapper/install.bash

@@ -38,9 +38,7 @@ cmake_install_args=(--install "${script_dir}/build")
 
 if [[ "${prefix}" != "" ]]; then
   cmake_configure_args+=("-DCMAKE_INSTALL_PREFIX=${prefix}")
-  # Export the same value as all 3
   export CMAKE_INSTALL_PREFIX="${prefix}"
-  export DESTDIR="${prefix}"
   export PREFIX="${prefix}"
 fi
 

src/cpp/legate/runtime/detail/argument_parsing/util.cc

@@ -10,7 +10,6 @@
 
 #include <legate/utilities/assert.h>
 #include <legate/utilities/detail/env.h>
-#include <legate/utilities/detail/string_utils.h>
 #include <legate/utilities/macros.h>
 
 #include <kvikio/shim/cufile.hpp>
@@ -26,32 +25,17 @@
 
 namespace legate::detail {
 
-unsigned int num_ranks()
+bool multi_node_job()
 {
   constexpr EnvironmentVariable<std::uint32_t> OMPI_COMM_WORLD_SIZE{"OMPI_COMM_WORLD_SIZE"};
   constexpr EnvironmentVariable<std::uint32_t> MV2_COMM_WORLD_SIZE{"MV2_COMM_WORLD_SIZE"};
   constexpr EnvironmentVariable<std::uint32_t> SLURM_NTASKS{"SLURM_NTASKS"};
 
-  const auto ompi_comm_world_size = OMPI_COMM_WORLD_SIZE.get().value_or(1);
-  if (ompi_comm_world_size > 1) {
-    return ompi_comm_world_size;
-  }
-
-  const auto mv2_comm_world_size = MV2_COMM_WORLD_SIZE.get().value_or(1);
-  if (mv2_comm_world_size > 1) {
-    return mv2_comm_world_size;
-  }
-
-  const auto slurm_ntasks = SLURM_NTASKS.get().value_or(1);
-  if (slurm_ntasks > 1) {
-    return slurm_ntasks;
-  }
-
-  return 1;
+  return OMPI_COMM_WORLD_SIZE.get().value_or(1) > 1 ||  //
+         MV2_COMM_WORLD_SIZE.get().value_or(1) > 1 ||   //
+         SLURM_NTASKS.get().value_or(1) > 1;
 }
 
-bool multi_node_job() { return num_ranks() > 1; }
-
 std::vector<std::string> deduplicate_command_line_flags(Span<const std::string> args)
 {
   // A dummy name that is used only in case the first arguments are positional. Currently

src/cpp/legate/runtime/detail/argument_parsing/util.h

@@ -13,14 +13,6 @@
 
 namespace legate::detail {
 
-/**
- * @brief Determine the number of ranks in use based on environment variables
- * OMPI_COMM_WORLD_SIZE, MV2_COMM_WORLD_SIZE, and SLURM_NTASKS.
- *
- * @return the number of ranks in use
- */
-[[nodiscard]] std::uint32_t num_ranks();
-
 /**
  * @return `true` when Legate is being invoked as a multi-node job, `false` otherwise.
  */

src/cpp/legate/runtime/detail/runtime.cc

@@ -61,7 +61,6 @@
 #include <legate/utilities/detail/env_defaults.h>
 #include <legate/utilities/detail/formatters.h>
 #include <legate/utilities/detail/linearize.h>
-#include <legate/utilities/detail/string_utils.h>
 #include <legate/utilities/detail/traced_exception.h>
 #include <legate/utilities/detail/tuple.h>
 #include <legate/utilities/hash.h>
@@ -1731,7 +1730,6 @@ void handle_realm_default_args(bool need_network_init)
         "Legate was run on multiple nodes but was not built with networking support. Please "
         "install Legate again with networking support (e.g. configured \"--with-ucx\")"};
     }
-
     // We have to pass an explicit `-ll:networks` flag, otherwise Realm will silently continue with
     // single-node execution if network initialization fails. Therefore, even though Realm's default
     // priority list for network modules is good enough for us, we have to duplicate it here.
@@ -1740,25 +1738,6 @@ void handle_realm_default_args(bool need_network_init)
 #endif
 #ifdef REALM_USE_GASNETEX
     ss << " -ll:networks gasnetex";
-
-    // Need to set -gex:obcount appropriately, see
-    // https://github.com/StanfordLegion/realm/issues/239
-    const auto num_gpus = Runtime::get_runtime().local_machine().total_gpu_count();
-    auto ranks          = num_ranks();
-
-    if (ranks == 1) {
-      constexpr EnvironmentVariable<std::string> WORKER_PEERS_INFO{"WORKER_PEERS_INFO"};
-      const auto realm_ucp_bootstrap_mode = REALM_UCP_BOOTSTRAP_MODE.get();
-
-      if (realm_ucp_bootstrap_mode == "p2p") {
-        if (const auto workers_peer_info = WORKER_PEERS_INFO.get(); workers_peer_info.has_value()) {
-          ranks = string_split(*workers_peer_info).size();
-        }
-      }
-    }
-    const auto obcount = 4 * ranks + 2 * num_gpus;
-
-    ss << " -gex:obcount " << obcount;
 #endif
 #ifdef REALM_USE_GASNET1
     ss << " -ll:networks gasnet1";