From 548adc2ad162b6c486e3a0e820e44b24b4dd1d1b Mon Sep 17 00:00:00 2001 From: Joshua Einstein-Curtis Date: Fri, 15 Jul 2022 09:12:08 -0400 Subject: [PATCH 1/7] Removing MPI initialization from module testing --- rsopt/util.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/rsopt/util.py b/rsopt/util.py index 0672e65..ad07b57 100644 --- a/rsopt/util.py +++ b/rsopt/util.py @@ -55,11 +55,14 @@ def return_nodelist(nodelist_string): def return_used_nodes(): """Returns all used processor names to rank 0 or an empty list if MPI not used. For ranks != 0 returns None.""" try: - from mpi4py import MPI + import mpi4py + mpi4py.rc.initialize = False except ModuleNotFoundError: # If MPI not being used to start rsopt then no nodes will have srun executed yet return [] + from mpi4py import MPI + MPI.Init() rank = MPI.COMM_WORLD.Get_rank() name = MPI.Get_processor_name() all_names = MPI.COMM_WORLD.gather(name, root=0) @@ -93,11 +96,14 @@ def return_unused_node(): def broadcast(data, root_rank=0): """broadcast, or don't bother""" try: - from mpi4py import MPI + import mpi4py + mpi4py.rc.initialize = False except ModuleNotFoundError: # If MPI not available for import then assume it isn't needed return data + from mpi4py import MPI + MPI.Init() if MPI.COMM_WORLD.Get_size() == 1: return data @@ -121,4 +127,4 @@ def _libe_save(H, persis_info, mess, filename): np.save(filename, H) with open(filename + ".pickle", "wb") as f: - pickle.dump(persis_info, f) \ No newline at end of file + pickle.dump(persis_info, f) From 38ca43d8ef82d9f4dcaabcc6bee607af95b848c1 Mon Sep 17 00:00:00 2001 From: Joshua Einstein-Curtis Date: Mon, 18 Jul 2022 11:52:05 -0400 Subject: [PATCH 2/7] Adding same change to mpi.py per request --- rsopt/mpi.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rsopt/mpi.py b/rsopt/mpi.py index 456b612..7d90b4c 100644 --- a/rsopt/mpi.py +++ b/rsopt/mpi.py @@ -1,7 +1,8 @@ def get_mpi_environment(): try: - from mpi4py import MPI + import mpi4py + mpi4py.rc.initialize = False except ModuleNotFoundError: # mpi4py not installed so it can't be used return @@ -11,8 +12,10 @@ def get_mpi_environment(): # (if user did start MPI with size 1 this would be an illegal configuration since: main + 1 worker = 2 ranks) return + from mpi4py import MPI + MPI.Init() nworkers = MPI.COMM_WORLD.Get_size() - 1 is_manager = MPI.COMM_WORLD.Get_rank() == 0 mpi_environment = {'mpi_comm': MPI.COMM_WORLD, 'comms': 'mpi', 'nworkers': nworkers, 'is_manager': is_manager} - return mpi_environment \ No newline at end of file + return mpi_environment From 92a2c22ea9e06daba921cc1be26662e22105ab8f Mon Sep 17 00:00:00 2001 From: Joshua Einstein-Curtis Date: Mon, 18 Jul 2022 16:33:08 -0400 Subject: [PATCH 3/7] Fixing bug pointed out by @cchall --- rsopt/mpi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rsopt/mpi.py b/rsopt/mpi.py index 7d90b4c..c70e352 100644 --- a/rsopt/mpi.py +++ b/rsopt/mpi.py @@ -7,13 +7,14 @@ def get_mpi_environment(): # mpi4py not installed so it can't be used return + from mpi4py import MPI + MPI.Init() + if not MPI.COMM_WORLD.Get_size() - 1: # MPI not being used # (if user did start MPI with size 1 this would be an illegal configuration since: main + 1 worker = 2 ranks) return - from mpi4py import MPI - MPI.Init() nworkers = MPI.COMM_WORLD.Get_size() - 1 is_manager = MPI.COMM_WORLD.Get_rank() == 0 mpi_environment = {'mpi_comm': MPI.COMM_WORLD, 'comms': 'mpi', 'nworkers': nworkers, 'is_manager': is_manager} From d698629d8da41da9f1ef6670c3dd629722f15b9a Mon Sep 17 00:00:00 2001 From: Joshua Einstein-Curtis Date: Tue, 19 Jul 2022 12:13:55 -0400 Subject: [PATCH 4/7] Updating with subprocess-based MPI checking --- rsopt/__main__.py | 2 ++ rsopt/mpi.py | 29 ++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 rsopt/__main__.py diff --git a/rsopt/__main__.py b/rsopt/__main__.py new file mode 100644 index 0000000..8d0ac04 --- /dev/null +++ b/rsopt/__main__.py @@ -0,0 +1,2 @@ +from mpi4py import MPI +MPI.Init() diff --git a/rsopt/mpi.py b/rsopt/mpi.py index c70e352..47f7136 100644 --- a/rsopt/mpi.py +++ b/rsopt/mpi.py @@ -1,19 +1,38 @@ - def get_mpi_environment(): + # Test for mpi4py install try: import mpi4py mpi4py.rc.initialize = False + from mpi4py import MPI except ModuleNotFoundError: # mpi4py not installed so it can't be used - return + return None + + from inspect import currentframe, getframeinfo + frameinfo = getframeinfo(currentframe()) + print(f"Initializing MPI from {frameinfo.filename}:L{frameinfo.lineno}", flush=True) + + #import faulthandler + #import sys + #faulthandler.enable(file=sys.stderr, all_threads=True) + + # Test MPI intialization in another thread + import subprocess + import os + import rsopt + fname = os.path.dirname(rsopt.__file__) + "/__main__.py" + pp = subprocess.run(["python", fname]) + + if pp.returncode != 0: + return None - from mpi4py import MPI - MPI.Init() + # Should already be initialized + # MPI.Init() if not MPI.COMM_WORLD.Get_size() - 1: # MPI not being used # (if user did start MPI with size 1 this would be an illegal configuration since: main + 1 worker = 2 ranks) - return + return None nworkers = MPI.COMM_WORLD.Get_size() - 1 is_manager = MPI.COMM_WORLD.Get_rank() == 0 From a04deafb1988128fb41707d05e59975d87ace4a0 Mon Sep 17 00:00:00 2001 From: Joshua Einstein-Curtis Date: Tue, 19 Jul 2022 13:39:32 -0400 Subject: [PATCH 5/7] Updating MPI handling with a global --- rsopt/mpi.py | 11 +++++++++++ rsopt/util.py | 24 +++++++----------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/rsopt/mpi.py b/rsopt/mpi.py index 47f7136..6851d4e 100644 --- a/rsopt/mpi.py +++ b/rsopt/mpi.py @@ -1,4 +1,8 @@ +active_env = None + def get_mpi_environment(): + global active_env + # Test for mpi4py install try: import mpi4py @@ -8,6 +12,10 @@ def get_mpi_environment(): # mpi4py not installed so it can't be used return None + # If we already ran this process, return the active environment + if active_env: + return active_env + from inspect import currentframe, getframeinfo frameinfo = getframeinfo(currentframe()) print(f"Initializing MPI from {frameinfo.filename}:L{frameinfo.lineno}", flush=True) @@ -38,4 +46,7 @@ def get_mpi_environment(): is_manager = MPI.COMM_WORLD.Get_rank() == 0 mpi_environment = {'mpi_comm': MPI.COMM_WORLD, 'comms': 'mpi', 'nworkers': nworkers, 'is_manager': is_manager} + # Save global environment + active_env = mpi_environment + return mpi_environment diff --git a/rsopt/util.py b/rsopt/util.py index ad07b57..89934e6 100644 --- a/rsopt/util.py +++ b/rsopt/util.py @@ -4,6 +4,8 @@ import numpy as np import pickle from libensemble.tools import save_libE_output +from .mpi import active_env as MPI_ENV +from .mpi import get_mpi_environment SLURM_PREFIX = 'nid' @@ -54,15 +56,9 @@ def return_nodelist(nodelist_string): def return_used_nodes(): """Returns all used processor names to rank 0 or an empty list if MPI not used. For ranks != 0 returns None.""" - try: - import mpi4py - mpi4py.rc.initialize = False - except ModuleNotFoundError: - # If MPI not being used to start rsopt then no nodes will have srun executed yet - return [] - - from mpi4py import MPI - MPI.Init() + if not MPI_ENV: + get_mpi_environment() + rank = MPI.COMM_WORLD.Get_rank() name = MPI.Get_processor_name() all_names = MPI.COMM_WORLD.gather(name, root=0) @@ -95,15 +91,9 @@ def return_unused_node(): def broadcast(data, root_rank=0): """broadcast, or don't bother""" - try: - import mpi4py - mpi4py.rc.initialize = False - except ModuleNotFoundError: - # If MPI not available for import then assume it isn't needed - return data + if not MPI_ENV: + get_mpi_environment() - from mpi4py import MPI - MPI.Init() if MPI.COMM_WORLD.Get_size() == 1: return data From 4a77a37d3466adad55cd0c49979c9d514e807975 Mon Sep 17 00:00:00 2001 From: Joshua Einstein-Curtis Date: Tue, 19 Jul 2022 14:25:18 -0400 Subject: [PATCH 6/7] fixing up environment checking --- rsopt/mpi.py | 27 ++++++++++++++++++--------- rsopt/util.py | 15 +++++++++------ 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/rsopt/mpi.py b/rsopt/mpi.py index 6851d4e..8c3aef6 100644 --- a/rsopt/mpi.py +++ b/rsopt/mpi.py @@ -1,7 +1,14 @@ -active_env = None +__active_env = None def get_mpi_environment(): - global active_env + """Checks MPI environment and whether or not MPI is initialized + + Params: + None + + Returns: + None if mpi is unavailable; else a dict representing the active MPI environment""" + global __active_env # Test for mpi4py install try: @@ -10,11 +17,14 @@ def get_mpi_environment(): from mpi4py import MPI except ModuleNotFoundError: # mpi4py not installed so it can't be used + __active_env = "no_mpi" + + if __active_env == "no_mpi": return None - # If we already ran this process, return the active environment - if active_env: - return active_env + # If we already ran this process and have an environment, return the active environment + if __active_env: + return __active_env from inspect import currentframe, getframeinfo frameinfo = getframeinfo(currentframe()) @@ -32,14 +42,13 @@ def get_mpi_environment(): pp = subprocess.run(["python", fname]) if pp.returncode != 0: + __active_env = "no_mpi" return None - # Should already be initialized - # MPI.Init() - if not MPI.COMM_WORLD.Get_size() - 1: # MPI not being used # (if user did start MPI with size 1 this would be an illegal configuration since: main + 1 worker = 2 ranks) + __active_env = "no_mpi" return None nworkers = MPI.COMM_WORLD.Get_size() - 1 @@ -47,6 +56,6 @@ def get_mpi_environment(): mpi_environment = {'mpi_comm': MPI.COMM_WORLD, 'comms': 'mpi', 'nworkers': nworkers, 'is_manager': is_manager} # Save global environment - active_env = mpi_environment + __active_env = mpi_environment return mpi_environment diff --git a/rsopt/util.py b/rsopt/util.py index 89934e6..9857fb9 100644 --- a/rsopt/util.py +++ b/rsopt/util.py @@ -4,7 +4,6 @@ import numpy as np import pickle from libensemble.tools import save_libE_output -from .mpi import active_env as MPI_ENV from .mpi import get_mpi_environment SLURM_PREFIX = 'nid' @@ -56,9 +55,11 @@ def return_nodelist(nodelist_string): def return_used_nodes(): """Returns all used processor names to rank 0 or an empty list if MPI not used. For ranks != 0 returns None.""" - if not MPI_ENV: - get_mpi_environment() - + if not get_mpi_environment(): + return [] + + from mpi4py import MPI + rank = MPI.COMM_WORLD.Get_rank() name = MPI.Get_processor_name() all_names = MPI.COMM_WORLD.gather(name, root=0) @@ -91,8 +92,10 @@ def return_unused_node(): def broadcast(data, root_rank=0): """broadcast, or don't bother""" - if not MPI_ENV: - get_mpi_environment() + if not get_mpi_environment(): + return data + + from mpi4py import MPI if MPI.COMM_WORLD.Get_size() == 1: return data From aaaa4ec7c68691975c4d1901844bf8f5d2924034 Mon Sep 17 00:00:00 2001 From: Joshua Einstein-Curtis Date: Tue, 19 Jul 2022 15:21:25 -0400 Subject: [PATCH 7/7] Moving imports to beginning of file for mpi.py --- rsopt/mpi.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/rsopt/mpi.py b/rsopt/mpi.py index 8c3aef6..f761ac4 100644 --- a/rsopt/mpi.py +++ b/rsopt/mpi.py @@ -1,3 +1,8 @@ +import os +import subprocess +from inspect import currentframe, getframeinfo +import rsopt + __active_env = None def get_mpi_environment(): @@ -25,8 +30,7 @@ def get_mpi_environment(): # If we already ran this process and have an environment, return the active environment if __active_env: return __active_env - - from inspect import currentframe, getframeinfo + frameinfo = getframeinfo(currentframe()) print(f"Initializing MPI from {frameinfo.filename}:L{frameinfo.lineno}", flush=True) @@ -35,9 +39,6 @@ def get_mpi_environment(): #faulthandler.enable(file=sys.stderr, all_threads=True) # Test MPI intialization in another thread - import subprocess - import os - import rsopt fname = os.path.dirname(rsopt.__file__) + "/__main__.py" pp = subprocess.run(["python", fname])