Support ispyb deposition for jungfrau rotation scans

pyproject.toml

@@ -55,7 +55,7 @@ dependencies = [
     "ophyd >= 1.10.5",
     "ophyd-async >= 0.14.0",
     "bluesky >= 1.14.6",
-    "dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@main",
+    "dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@2ce4bb42ef4c388fa23e944acb3cc28f2c59144b",
 ]
 
 
@@ -94,7 +94,7 @@ dev = [
     "tox-uv",
     "types-mock",
     "types-requests",
-    "sphinxcontrib.mermaid", # To build dodal docs
+    "sphinxcontrib.mermaid",     # To build dodal docs
 ]
 
 [project.scripts]

src/mx_bluesky/beamlines/i24/jungfrau_commissioning/callbacks/metadata_writer.py

@@ -3,7 +3,8 @@
 
 from bluesky.callbacks import CallbackBase
 
-from mx_bluesky.beamlines.i24.parameters.constants import PlanNameConstants
+from mx_bluesky.common.external_interaction.ispyb.ispyb_store import IspybIds
+from mx_bluesky.common.parameters.constants import PlanNameConstants
 from mx_bluesky.common.parameters.rotation import SingleRotationScan
 from mx_bluesky.common.utils.log import LOGGER
 
@@ -13,11 +14,25 @@
 class JsonMetadataWriter(CallbackBase):
     """Callback class to handle the creation of metadata json files for commissioning.
 
-    To use, subscribe the Bluesky RunEngine to an instance of this class.
-    E.g.:
-        metadata_writer_callback = JsonMetadataWriter(parameters)
-        RE.subscribe(metadata_writer_callback)
-    Or decorate a plan using bluesky.preprocessors.subs_decorator.
+    Currently, nexus files aren't being written by nexgen, so writer needs to include
+    the dcid produced by an ispyb callback. To get this working, the ispyb callback need
+    to be triggered in the right way before the start function here is run.
+
+    To use:
+        1. subscribe the Bluesky RunEngine to an instance of this class.
+        E.g.:
+            metadata_writer_callback = JsonMetadataWriter(parameters)
+            RE.subscribe(metadata_writer_callback)
+        Or decorate a plan using bluesky.preprocessors.subs_decorator.
+        2. Subscribe the RE to the rotation callbacks:
+            ispyb_callback = RotationISPyBCallback()
+            @bpp.subs_decorator(ispyb_callback)
+            ...
+        3. Open a run with key decorator PlanNameConstants.ROTATION_OUTER, with metadata:
+        "activate_callbacks": ["RotationISPyBCallback"]
+        4. Open a run with key decorator PlanNameConstants.ROTATION_MAIN, with metadata:
+        "dcid": ispyb_callback.ispyb_ids
+
 
     See: https://blueskyproject.io/bluesky/callbacks.html#ways-to-invoke-callbacks
 
@@ -43,6 +58,12 @@ def start(self, doc: dict):  # type: ignore
             )
             self.parameters = SingleRotationScan(**json.loads(json_params))
             self.run_start_uid = doc.get("uid")
+            dcid = doc.get("dcid")
+            assert isinstance(dcid, IspybIds), (
+                "Rotation start document should include dcid of type IspybIds to use JF metadata writer. This should come from RotationISPyBCallback activated by a "
+                "PlanNameConstants.ROTATION_OUTER run"
+            )
+            self.dcid = dcid
 
     def descriptor(self, doc: dict):  # type: ignore
         self.descriptors[doc["uid"]] = doc
@@ -81,6 +102,7 @@ def stop(self, doc: dict):  # type: ignore
                             "energy_kev": self.energy_in_kev,
                             "angular_increment_deg": self.parameters.rotation_increment_deg,
                             "detector_distance_mm": self.detector_distance_mm,
+                            "dcid": self.dcid.data_collection_ids[0],
                         }
                     )
                 )

src/mx_bluesky/beamlines/i24/jungfrau_commissioning/experiment_plans/rotation_scan_plan.py

@@ -27,9 +27,6 @@
     JF_COMPLETE_GROUP,
     fly_jungfrau,
 )
-from mx_bluesky.beamlines.i24.parameters.constants import (
-    PlanNameConstants,
-)
 from mx_bluesky.common.device_setup_plans.setup_zebra_and_shutter import (
     setup_zebra_for_rotation,
     tidy_up_zebra_after_rotation_scan,
@@ -41,10 +38,14 @@
     RotationMotionProfile,
     calculate_motion_profile,
 )
+from mx_bluesky.common.external_interaction.callbacks.rotation.ispyb_callback import (
+    RotationISPyBCallback,
+)
 from mx_bluesky.common.parameters.components import get_param_version
 from mx_bluesky.common.parameters.constants import (
     USE_NUMTRACKER,
     PlanGroupCheckpointConstants,
+    PlanNameConstants,
 )
 from mx_bluesky.common.parameters.rotation import (
     SingleRotationScan,
@@ -152,8 +153,17 @@ def single_rotation_plan(
     about a fixed axis - for now this axis is limited to omega.
     Needs additional setup of the sample environment and a wrapper to clean up."""
 
-    @bpp.set_run_key_decorator(PlanNameConstants.SINGLE_ROTATION_SCAN)
-    @run_decorator()
+    ispyb_callback = RotationISPyBCallback()
+
+    @bpp.subs_decorator(ispyb_callback)
+    @bpp.set_run_key_decorator(PlanNameConstants.ROTATION_OUTER)
+    @run_decorator(
+        md={
+            "subplan_name": PlanNameConstants.ROTATION_OUTER,
+            "mx_bluesky_parameters": params.model_dump_json(),
+            "activate_callbacks": ["RotationISPyBCallback"],
+        }
+    )
     def _plan_in_run_decorator():
         if not params.detector_distance_mm:
             LOGGER.info(
@@ -185,6 +195,7 @@ def _plan_in_run_decorator():
                 "scan_points": [params.scan_points],
                 "rotation_scan_params": params.model_dump_json(),
                 "detector_file_template": params.file_name,
+                "dcid": ispyb_callback.ispyb_ids,
             }
         )
         def _rotation_scan_plan(
@@ -232,13 +243,18 @@ def _rotation_scan_plan(
 
             # Read hardware after preparing jungfrau so that device metadata output from callback is correct
             # Whilst metadata is being written in bluesky we need to access the private writer here
+            # We also need to access the private writer so that we get path information to send to ispyb.
+            # This will be needed until numtracker / StartDocumentPathProvider can fully support the MX
+            # usecase, see #issue
             read_hardware_partial = partial(
                 read_hardware_plan,
                 [
                     composite.dcm.energy_in_keV,
                     composite.dcm.wavelength_in_a,
                     composite.det_stage.z,
                     composite.jungfrau._writer.file_path,  # noqa: SLF001 N
+                    composite.jungfrau._writer.file_name,  # noqa: SLF001 N
+                    composite.jungfrau.ispyb_detector_id,
                 ],
                 PlanNameConstants.ROTATION_DEVICE_READ,
             )

src/mx_bluesky/common/external_interaction/callbacks/common/ispyb_mapping.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from event_model import RunStart
+
 from mx_bluesky.common.external_interaction.ispyb.data_model import (
     DataCollectionGroupInfo,
     DataCollectionInfo,
@@ -8,6 +10,7 @@
     get_current_time_string,
 )
 from mx_bluesky.common.parameters.components import DiffractionExperimentWithSample
+from mx_bluesky.common.parameters.constants import USE_NUMTRACKER
 
 EIGER_FILE_SUFFIX = "h5"
 
@@ -26,16 +29,25 @@ def populate_remaining_data_collection_info(
     data_collection_group_id,
     data_collection_info: DataCollectionInfo,
     params: DiffractionExperimentWithSample,
+    doc: RunStart | None = None,
 ):
     data_collection_info.sample_id = params.sample_id
-    data_collection_info.visit_string = params.visit
     data_collection_info.parent_id = data_collection_group_id
     data_collection_info.comments = comment
     data_collection_info.detector_distance = params.detector_params.detector_distance
     data_collection_info.exp_time = params.detector_params.exposure_time_s
-    data_collection_info.imgdir = params.detector_params.directory
-    data_collection_info.imgprefix = params.detector_params.prefix
-    data_collection_info.imgsuffix = EIGER_FILE_SUFFIX
+    if params.visit == USE_NUMTRACKER:
+        # Plans using StandardDetector, numtracker and BlueAPI get its information via metadata.
+        # This metadata gets set after a run is open, and is provided by numtracker
+        assert doc, (
+            "Expected RunStart doc to be provided to populate_remaining_data_collection_info when using numtracker for visit"
+        )
+        data_collection_info.visit_string = doc.get("instrument_session")
+    else:
+        data_collection_info.visit_string = params.visit
+        data_collection_info.imgdir = params.detector_params.directory
+        data_collection_info.imgprefix = params.detector_params.prefix
+        data_collection_info.imgsuffix = EIGER_FILE_SUFFIX
     # Both overlap and n_passes included for backwards compatibility,
     # planned to be removed later
     data_collection_info.n_passes = 1
@@ -47,7 +59,10 @@ def populate_remaining_data_collection_info(
     data_collection_info.xbeam = beam_position[0]
     data_collection_info.ybeam = beam_position[1]
     data_collection_info.start_time = get_current_time_string()
-    if data_collection_info.data_collection_number is not None:
+    if (
+        data_collection_info.data_collection_number is not None
+        and params.visit != USE_NUMTRACKER
+    ):
         # Do not write the file template if we don't have sufficient information - for gridscans we  may not
         # know the data collection number until later
         data_collection_info.file_template = f"{params.detector_params.prefix}_{data_collection_info.data_collection_number}_master.h5"

src/mx_bluesky/hyperion/external_interaction/callbacks/rotation/ispyb_callback.py → src/mx_bluesky/common/external_interaction/callbacks/rotation/ispyb_callback.py

@@ -15,6 +15,9 @@
 from mx_bluesky.common.external_interaction.callbacks.common.zocalo_callback import (
     ZocaloInfoGenerator,
 )
+from mx_bluesky.common.external_interaction.callbacks.rotation.ispyb_mapping import (
+    populate_data_collection_info_for_rotation,
+)
 from mx_bluesky.common.external_interaction.ispyb.data_model import (
     DataCollectionInfo,
     DataCollectionPositionInfo,
@@ -25,14 +28,12 @@
     StoreInIspyb,
 )
 from mx_bluesky.common.parameters.components import IspybExperimentType
+from mx_bluesky.common.parameters.constants import USE_NUMTRACKER, PlanNameConstants
 from mx_bluesky.common.parameters.rotation import (
     SingleRotationScan,
 )
 from mx_bluesky.common.utils.log import ISPYB_ZOCALO_CALLBACK_LOGGER, set_dcgid_tag
 from mx_bluesky.common.utils.utils import number_of_frames_from_scan_spec
-from mx_bluesky.hyperion.external_interaction.callbacks.rotation.ispyb_mapping import (
-    populate_data_collection_info_for_rotation,
-)
 from mx_bluesky.hyperion.parameters.constants import CONST
 
 if TYPE_CHECKING:
@@ -69,9 +70,21 @@ def activity_gated_start(self, doc: RunStart):
             ISPYB_ZOCALO_CALLBACK_LOGGER.info(
                 "ISPyB callback received start document with experiment parameters."
             )
-            hyperion_params = doc.get("mx_bluesky_parameters")
-            assert isinstance(hyperion_params, str)
-            self.params = SingleRotationScan.model_validate_json(hyperion_params)
+            params = doc.get("mx_bluesky_parameters")
+            assert isinstance(params, str)
+            self.params = SingleRotationScan.model_validate_json(params)
+
+            # Todo this chunk needs to go at the start of any numtracker+ispyb-using plan
+            if self.params.visit == USE_NUMTRACKER:
+                try:
+                    visit = doc.get("instrument_session")
+                    assert isinstance(visit, str)
+                    self.params.visit = visit
+                except Exception as e:
+                    raise ValueError(
+                        f"Error trying to retrieve instrument session from document {doc}"
+                    ) from e
+
             dcgid = (
                 self.ispyb_ids.data_collection_group_id
                 if (self.params.sample_id == self.last_sample_id)
@@ -97,10 +110,7 @@ def activity_gated_start(self, doc: RunStart):
                 self.params
             )
             data_collection_info = populate_remaining_data_collection_info(
-                self.params.comment,
-                dcgid,
-                data_collection_info,
-                self.params,
+                self.params.comment, dcgid, data_collection_info, self.params, doc
             )
             data_collection_info.parent_id = dcgid
             scan_data_info = ScanDataInfo(
@@ -150,6 +160,7 @@ def _handle_ispyb_hardware_read(self, doc: Event):
 
     def activity_gated_event(self, doc: Event):
         doc = super().activity_gated_event(doc)
+        assert self.params, "IspyB callback event triggered before parameters were set"
         set_dcgid_tag(self.ispyb_ids.data_collection_group_id)
 
         descriptor_name = self.descriptors[doc["descriptor"]].get("name")
@@ -158,6 +169,20 @@ def activity_gated_event(self, doc: Event):
             self.ispyb_ids = self.ispyb.update_deposition(
                 self.ispyb_ids, scan_data_infos
             )
+        elif descriptor_name == PlanNameConstants.ROTATION_DEVICE_READ:
+            data = doc["data"]
+            filepath = data.get("commissioning_jungfrau-_writer-file_path")
+            filename = data.get("commissioning_jungfrau-_writer-file_name")
+            updated_dc = DataCollectionInfo(
+                file_template=f"{filename}.nxs",
+                imgdir=f"{filepath}/",
+                imgprefix=f"{filename}",
+                imgsuffix="nxs",
+            )
+            scan_data_info = self.populate_info_for_update(
+                updated_dc, None, self.params
+            )
+            self.ispyb.update_deposition(self.ispyb_ids, scan_data_info)
 
         return doc
 

src/mx_bluesky/common/parameters/components.py

@@ -12,6 +12,7 @@
     DetectorParams,
     TriggerMode,
 )
+from dodal.utils import BeamlinePrefix, get_beamline_name
 from pydantic import (
     BaseModel,
     ConfigDict,
@@ -24,14 +25,17 @@
 from semver import Version
 
 from mx_bluesky.common.parameters.constants import (
-    TEST_MODE,
     USE_NUMTRACKER,
     DetectorParamConstants,
     GridscanParamConstants,
 )
 
+TEST_MODE = os.environ.get("HYPERION_TEST_MODE")
+
 PARAMETER_VERSION = Version.parse("5.3.0")
 
+BL = get_beamline_name("i03")
+
 
 def get_param_version() -> SemanticVersion:
     return SemanticVersion.validate_from_str(str(PARAMETER_VERSION))
@@ -158,7 +162,9 @@ class WithVisit(BaseModel):
         default=DetectorParamConstants.BEAM_XY_LUT_PATH
     )
     detector_distance_mm: float | None = Field(default=None, gt=0)
-    insertion_prefix: str = "SR03S" if TEST_MODE else "SR03I"
+    insertion_prefix: str = (
+        f"{BeamlinePrefix(BL).insertion_prefix}" if TEST_MODE else "SR03I"
+    )
 
 
 class DiffractionExperiment(
@@ -191,7 +197,7 @@ def validate_directories(cls, values):
                 Path(values["storage_directory"], "snapshots").as_posix(),
             )
         else:
-            values["snapshot_directory"] = Path("/tmp")
+            values["snapshot_directory"] = Path("/tmp").as_posix()
         return values
 
     @property
@@ -232,7 +238,9 @@ def scan_indices(self) -> Sequence[SupportsInt]:
 
 
 class WithSample(BaseModel):
-    sample_id: int
+    sample_id: int | None = (
+        None  # None is invalid for dc groups, but valid for regular data collections
+    )
     sample_puck: int | None = None
     sample_pin: int | None = None
 

src/mx_bluesky/common/parameters/constants.py

@@ -76,6 +76,7 @@ class PlanNameConstants:
     ROTATION_MULTI_OUTER = "multi_rotation_outer"
     ROTATION_OUTER = "rotation_scan_with_cleanup"
     ROTATION_MAIN = "rotation_scan_main"
+    ROTATION_DEVICE_READ = "ROTATION DEVICE READ"
 
     SET_ENERGY = "set_energy"