debug and file log handler support for hyperion-blueapi

conftest.py

@@ -4,6 +4,9 @@
 
 import pytest
 
+# Ensure that the blueapi entry point is not invoked by doctest as this will fail
+collect_ignore = ["src/mx_bluesky/hyperion/blueapi_plans/blueapi.py"]
+
 environ["HYPERION_TEST_MODE"] = "true"
 
 

docs/user/hyperion/logging.rst

@@ -38,6 +38,6 @@ Debug Log
 The standard logs files do not record all messages, only those at INFO level or higher, in order to keep storage 
 requirements to a minimum. 
 In the event of an error occurring, then trace-level logging for the most recent events (by default 5000) is flushed 
-to a separate set of log files. Due to the size of these files, they are stored separately from the main log files
-. These logs are located in ``/dls/tmp/<beamline>/logs/bluesky`` by default, or 
-otherwise as specified by the ``DEBUG_LOG_DIR`` environment variable. 
+to a separate set of log files. Due to the size of these files, they are stored separately from the main
+log files. These logs are typically located in ``/dls/tmp/<beamline>/logs/bluesky``,
+as specified by the ``DEBUG_LOG_DIR`` environment variable.

run_hyperion.sh

@@ -141,6 +141,16 @@ if [[ $START == 1 ]]; then
     echo "$(date) Logging to $LOG_DIR"
     export LOG_DIR
     mkdir -p "$LOG_DIR"
+    if [ -z "$DEBUG_LOG_DIR" ]; then
+        if [ $IN_DEV = true ]; then
+            DEBUG_LOG_DIR=$LOG_DIR
+        else
+            DEBUG_LOG_DIR=/dls/tmp/$BEAMLINE/logs/bluesky
+        fi
+    fi
+    echo "Debug log file set to $DEBUG_LOG_DIR"
+    export DEBUG_LOG_DIR
+    mkdir -p "$DEBUG_LOG_DIR"
     if [ $MODE = "supervisor" ]; then
       start_log_path=$LOG_DIR/supervisor_start_log.log
     else

src/mx_bluesky/common/utils/log.py

@@ -10,6 +10,8 @@
     DodalLogHandlers,
     integrate_bluesky_and_ophyd_logging,
     set_up_all_logging_handlers,
+    set_up_debug_memory_handler,
+    set_up_info_file_handler,
 )
 from dodal.log import LOGGER as DODAL_LOGGER
 
@@ -66,6 +68,23 @@ def set_uid_tag(uid):
     tag_filter.run_uid = uid
 
 
+def setup_hyperion_blueapi_logging(log_file_name: str):
+    """Configure debug logging for hyperion-blueapi.
+    Args:
+        log_file_name: Base name of the log file.
+    """
+    dodal_logger = DODAL_LOGGER
+    logging_path, debug_logging_path = _get_logging_dirs(False)
+    set_up_debug_memory_handler(
+        dodal_logger, debug_logging_path, log_file_name, ERROR_LOG_BUFFER_LINES
+    )
+    set_up_info_file_handler(
+        dodal_logger,
+        logging_path,
+        log_file_name,
+    )
+
+
 def do_default_logging_setup(
     file_name: str,
     graylog_port: int,
@@ -115,26 +134,31 @@ def flush_debug_handler() -> str:
 def _get_logging_dirs(dev_mode: bool) -> tuple[Path, Path]:
     """Get the paths to write the mx_bluesky log files to.
 
-    Log location can be specified in the LOG_DIR environment variable, otherwise MX bluesky logs are written to 'dls_sw/ixx/logs/bluesky'.
-    This directory will be created if it is not found
+    Log location must be specified in the LOG_DIR environment variable,
+    and the debug log location specified in the DEBUG_LOG_DIR environment variable.
+    This directory will be created if it is not found.
 
     Logs are written to ./tmp/logs/bluesky if BEAMLINE environment variable is not found
 
+    Args:
+        dev_mode (bool): If True, the logs will be written to /tmp/logs/bluesky if the environment variables are not set
     Returns:
         tuple[Path, Path]: Paths to the standard log file and to the debug log file, for the file handlers to write to
+    Raises:
+        ValueError: If LOG_DIR or DEBUG_LOG_DIR environment variable is not set and dev_mode is False
     """
-
-    beamline = environ.get("BEAMLINE")
-
-    if beamline and not dev_mode:
-        default_logging_str = f"/dls_sw/{beamline}/logs/bluesky/"
-        default_debug_logging_str = f"/dls/tmp/{beamline}/logs/bluesky/"
+    if dev_mode:
+        logging_path = Path(environ.get("LOG_DIR", "/tmp/logs/bluesky"))
+        debug_logging_path = Path(environ.get("DEBUG_LOG_DIR", "/tmp/logs/bluesky"))
     else:
-        default_logging_str = "/tmp/logs/bluesky"
-        default_debug_logging_str = default_logging_str
-
-    logging_path = Path(environ.get("LOG_DIR", default_logging_str))
-    debug_logging_path = Path(environ.get("DEBUG_LOG_DIR", default_debug_logging_str))
+        logging_dir = environ.get("LOG_DIR")
+        if not logging_dir:
+            raise ValueError("LOG_DIR environment variable is not set")
+        debug_logging_dir = environ.get("DEBUG_LOG_DIR")
+        if not debug_logging_dir:
+            raise ValueError("DEBUG_LOG_DIR environment variable is not set")
+        logging_path = Path(logging_dir)
+        debug_logging_path = Path(debug_logging_dir)
 
     Path.mkdir(logging_path, exist_ok=True, parents=True)
     Path.mkdir(debug_logging_path, exist_ok=True, parents=True)

src/mx_bluesky/hyperion/blueapi_config.yaml

@@ -3,7 +3,7 @@ env:
     - kind: deviceManager
       module: dodal.beamlines.i03
     - kind: planFunctions
-      module: mx_bluesky.hyperion.blueapi_plans
+      module: mx_bluesky.hyperion.blueapi_plans.blueapi
   events:
     broadcast_status_events: false
 api:

src/mx_bluesky/hyperion/blueapi_dev_config.yaml

@@ -4,7 +4,7 @@ env:
       module: dodal.beamlines.i03
       mock: true
     - kind: planFunctions
-      module: mx_bluesky.hyperion.blueapi_plans
+      module: mx_bluesky.hyperion.blueapi_plans.blueapi
   events:
     broadcast_status_events: false
 api:

src/mx_bluesky/hyperion/blueapi_plans/__init__.py

@@ -1,96 +0,0 @@
-"""
-This module contains the bluesky plan entry points for use with hyperion-blueapi.
-The json schema and documentation therein generated by the blueapi /plans endpoint
-from this file constitutes the hyperion-blueapi interface to the hyperion supervisor
-process.
-"""
-
-from bluesky import plan_stubs as bps
-from bluesky.utils import MsgGenerator
-from dodal.common import inject
-from dodal.devices.aperturescatterguard import ApertureScatterguard
-from dodal.devices.detector.detector_motion import DetectorMotion, ShutterState
-from dodal.devices.motors import XYZStage
-from dodal.devices.robot import BartRobot
-from dodal.devices.smargon import Smargon
-
-from mx_bluesky.common.device_setup_plans.robot_load_unload import (
-    robot_unload as _robot_unload,
-)
-from mx_bluesky.hyperion.experiment_plans.load_centre_collect_full_plan import (
-    LoadCentreCollectComposite,
-)
-from mx_bluesky.hyperion.experiment_plans.load_centre_collect_full_plan import (
-    load_centre_collect_full as _load_centre_collect_full,
-)
-from mx_bluesky.hyperion.experiment_plans.udc_default_state import (
-    UDCDefaultDevices,
-)
-from mx_bluesky.hyperion.experiment_plans.udc_default_state import (
-    move_to_udc_default_state as _move_to_udc_default_state,
-)
-from mx_bluesky.hyperion.parameters.load_centre_collect import LoadCentreCollect
-
-__all__ = [
-    "LoadCentreCollectComposite",
-    "LoadCentreCollect",
-    "UDCDefaultDevices",
-    "clean_up_udc",
-    "load_centre_collect",
-    "move_to_udc_default_state",
-    "robot_unload",
-]
-
-
-def load_centre_collect(
-    parameters: LoadCentreCollect, composite: LoadCentreCollectComposite = inject()
-) -> MsgGenerator:
-    """
-    Attempt a complete data collection experiment, consisting of the following:
-        * Load the sample if necessary
-        * Move to the specified goniometer start angles
-        * Perform optical centring, then X-ray centring
-        * If X-ray centring finds one or more diffracting centres then for each centre
-    that satisfies the chosen selection function,
-    move to that centre and do a collection with the specified parameters.
-    """
-    yield from _load_centre_collect_full(composite, parameters)
-
-
-def robot_unload(
-    visit: str,
-    robot: BartRobot = inject("robot"),
-    smargon: Smargon = inject("gonio"),
-    aperture_scatterguard: ApertureScatterguard = inject("aperture_scatterguard"),
-    lower_gonio: XYZStage = inject("lower_gonio"),
-) -> MsgGenerator:
-    """
-    Unload the currently mounted pin into the location that it was loaded from.
-    This is to be invoked as the final step upon successful completion of the UDC queue.
-    """
-    yield from _robot_unload(robot, smargon, aperture_scatterguard, lower_gonio, visit)
-
-
-def clean_up_udc(
-    visit: str,
-    cleanup_group: str = "cleanup",
-    robot: BartRobot = inject("robot"),
-    smargon: Smargon = inject("gonio"),
-    aperture_scatterguard: ApertureScatterguard = inject("aperture_scatterguard"),
-    lower_gonio: XYZStage = inject("lower_gonio"),
-    detector_motion: DetectorMotion = inject("detector_motion"),
-) -> MsgGenerator:
-    yield from bps.abs_set(
-        detector_motion.shutter, ShutterState.CLOSED, group=cleanup_group
-    )
-    yield from _robot_unload(robot, smargon, aperture_scatterguard, lower_gonio, visit)
-    yield from bps.wait(cleanup_group)
-
-
-def move_to_udc_default_state(
-    composite: UDCDefaultDevices = inject(),
-) -> MsgGenerator:
-    """
-    Move beamline hardware to known positions prior to UDC start.
-    """
-    yield from _move_to_udc_default_state(composite)