Updates to floco recovery and more

daq_lib.py

@@ -237,14 +237,14 @@ def flocoUnlock():
   unlockGUI()
   govMonOn()
 
-def flocoStopOperations():
+def floco_stop_operations():
   try:
     daq_macros.run_recovery_procedure(stop=True)
     lockGUI()
   except Exception as e:
-    logger.exception("Error encountered while running flocoStopOperations. Stopping...")
+    logger.exception("Error encountered while running floco_stop_operations. Stopping...")
 
-def flocoContinueOperations():
+def floco_continue_operations():
   try:
     daq_macros.run_recovery_procedure(stop=False)
     flocoUnlock()

daq_macros.py

@@ -48,6 +48,7 @@
 from fmx_annealer import govStatusGet, govStateSet, fmxAnnealer, amxAnnealer # for using annealer specific to FMX and AMX
 from config_params import ON_MOUNT_OPTION, OnMountAvailOptions, BEAMSIZE_OPTIONS
 from mxbluesky.plans import detect_loop, topview_optimized
+import json
 
 if daq_utils.beamline == 'fmx':
   from setenergy_lsdc import setELsdc
@@ -299,12 +300,24 @@ def recoverCS8():
 
 def run_recovery_procedure(stop=True):
   """
-  Manual recovery procedure used in daq_lib.flocoStopOperations and daq_lib.flocoContinueOperations
+  Manual recovery procedure used in daq_lib.floco_stop_operations and daq_lib.floco_continue_operations
   """
+  RESET  = "\033[0m"
+  BOLD = "\033[1m"
+  RED    = "\033[31m"
+  GREEN  = "\033[32m"
+  YELLOW = "\033[33m"
   logger.info(f"Running recovery procedure: {'stop operations' if stop else 'continue operations'}")
+
   def check_robot():
-    if not getBlConfig("robot_online") or not getBlConfig("mountEnabled"):
-      raise ValueError("Robot is offline or mount is disabled, sample found in gripper. Stopping recovery...")
+    if not getBlConfig("robot_online"):
+      logger.info(f"{YELLOW}Robot was off, running robotOn(){RESET}")
+      robotOn()
+
+    if not getBlConfig("mountEnabled"):
+      logger.info(f"{YELLOW}Mounting was disabled. Running enableMount(){RESET}")
+      enableMount()
+      # raise ValueError(f"{RED}{BOLD}Robot is offline or mount is disabled. Stopping recovery...{RESET}")
 
   def run_home_pin():
     from start_bs import home_pins
@@ -315,31 +328,61 @@ def gov_state_to_se():
 
   def check_robot_speed():
     if not robot_arm.is_full_speed():
-      logger.error("Robot arm speed is NOT 100%")
+      logger.error(f"{RED}{BOLD}Robot arm speed is NOT 100%{RESET}")
 
   def check_beam():
     if not (getPvDesc("beamAvailable") or getBlConfig(BEAM_CHECK) == 0):
-      logger.error("Beam not available, please open shutter")
+      logger.error(f"{RED}{BOLD}Beam not available, please open shutter{RESET}")
+
+  def check_gonio():
+    print(f"BoostStatus: {getPvDesc('boostStatus') == 0}" )
+    print(f'Mount status {daq_lib.get_field("mounted_pin") != ""}')
+    print(f'Sample detected: {getPvDesc("sampleDetected") == 0}')
+    if (getPvDesc("boostStatus") != 0 
+       or daq_lib.get_field("mounted_pin") != ''
+       or getPvDesc("sampleDetected") == 0):
+      logger.error("Pin is mounted on gonio")
+      raise Exception(f"{RED}{BOLD}Gonio has a sample, stopping recovery procedure{RESET}")
+
+  def set_fts():
+    # Value is set to 9 because 0.1 seconds is the 10th option in the css dropdown!
+    force_torque_sensor.put(9)
 
-  steps_to_run = {"Recover robot": robot_lib.recoverRobot, 
-                  "Check robot status": check_robot, 
-                  "Dry gripper": robot_lib.dryGripper, 
-                  "Home pin": run_home_pin, }
   if stop:
-    steps_to_run.update({"Disable mount": disableMount, "Robot off": robotOff})
+    steps_to_run = {
+      "Logging": logMe,
+      "Recover robot": robot_lib.recoverRobot,
+      "Setting FTS to 0.1 sec polling": set_fts, 
+      "Disable mount": disableMount, 
+      "Robot off": robotOff}
   else:
-    steps_to_run.update({"Enable mount": enableMount, "Robot on": robotOn})
-
-  steps_to_run.update({"Move Governor to SE": gov_state_to_se, })
-  if not stop:
-    steps_to_run.update({"Checking robot arm speed": check_robot_speed,
-                       "Checking beam": check_beam})
-
-  for i, (step_message, func) in enumerate(steps_to_run.items()):
-    logger.info(f"Step {i+1} of {len(steps_to_run)}: {step_message}")
-    func()
-    logger.info(f"Completed step: {step_message}")
-
+    steps_to_run = {
+      "Logging": logMe,
+      "Checking sample in gonio": check_gonio,
+      "Recover robot": robot_lib.recoverRobotFloco,
+      "Setting FTS to 0.1 sec polling": set_fts, 
+      "Check robot status": check_robot, 
+      "Dry gripper": robot_lib.dryGripperFloco, 
+      "Home pin": run_home_pin, 
+      "Enable mount": enableMount, 
+      "Robot on": robotOn, 
+      "Move Governor to SE": gov_state_to_se,
+      "Checking robot arm speed": check_robot_speed,
+      "Checking beam": check_beam}
+
+  try:
+    daq_lib.set_field("program_state","Running floco recovery")
+    for i, (step_message, func) in enumerate(steps_to_run.items()):
+      logger.info(f"{YELLOW}Executing step {i+1} of {len(steps_to_run)}:{RESET} {GREEN}{step_message}{RESET}")
+      func()
+      # logger.info(f"{GREEN}Completed step: {step_message}{RESET}")
+  except Exception as e:
+    logger.info(f"{RED}Exception while running floco recovery{RESET} : {e}")
+    if "unrecognized location" in str(e).lower():
+      message = "Unrecognized location, Manual recovery required"
+      logger.info(f"{RED}Recovery aborted: {message}{RESET}")
+  finally:
+    daq_lib.set_field("program_state","Program Ready")
 
 def run_top_view_optimized():
     RE(topview_optimized())
@@ -4349,11 +4392,13 @@ def setAttenRI():
 def robotOn():
   """robotOn() : use the robot to mount samples"""
   setBlConfig("robot_online",1)
+  daq_lib.gui_message(json.dumps({"robot_online": True}))
 
 
 def robotOff():
   """robotOff() : fake mounting samples"""  
   setBlConfig("robot_online",0)
+  daq_lib.gui_message(json.dumps({"robot_online": False}))
 
 
 def zebraVecDaqSetup(angle_start,imgWidth,exposurePeriodPerImage,numImages,filePrefix,data_directory_name,file_number_start,scanEncoder=3): #scan encoder 0=x, 1=y,2=z,3=omega
@@ -4467,6 +4512,14 @@ def queueCollectOff():
   """queueCollectOff() : do not allow creating requests for samples that are not mounted"""  
   setBlConfig("queueCollect",0)
 
+def unmountColdOn():
+  setBlConfig(UNMOUNT_COLD_CHECK, 1)
+  daq_lib.gui_message(json.dumps({"unmount_cold": True}))
+
+def unmountColdOff():
+  setBlConfig(UNMOUNT_COLD_CHECK, 0)
+  daq_lib.gui_message(json.dumps({"unmount_cold": False}))
+
 def guiLocal(): #monitor omega RBV
   """guiLocal() : show the readback of the Omega motor as it's moving. Can lead to lags when operating remotely with reduced bandwidth."""
   setBlConfig("omegaMonitorPV","RBV")
@@ -4527,10 +4580,12 @@ def backoffDetector():
 def disableMount():
   """disableMount() : turn off robot mounting. Usually done in an error situation where we want staff intervention before resuming."""
   setBlConfig("mountEnabled",0)
+  daq_lib.gui_message(json.dumps({"enable_mount": False}))
 
 def enableMount():
   """enableMount() : allow robot mounting"""
   setBlConfig("mountEnabled",1)
+  daq_lib.gui_message(json.dumps({"enable_mount": True}))
 
 def set_beamsize(sizeV, sizeH):
   if (sizeV == 'V0'):

daq_main_common.py

@@ -83,8 +83,17 @@ def setGovState(state):
     unlatchGov,
     backoffDetector,
     enableMount,
+    disableMount,
     robotOn,
-    set_energy
+    robotOff,
+    set_energy,
+    anneal,
+    beamCheckOn,
+    beamCheckOff,
+    queueCollectOn,
+    queueCollectOff,
+    unmountColdOn,
+    unmountColdOff
     ]
 
 if daq_utils.beamline != "nyx":

embl_robot.py

@@ -66,23 +66,33 @@ def warmupGripperRecoverThread(self, savedThreshold,junk):
       setPvDesc("warmupThreshold",savedThreshold)
 
 
-    def recoverRobot(self):
+    def _recoverRobot(self, raise_on_error=False):
       try:
         self.rebootEMBL()
         time.sleep(8.0)
         _,bLoaded,_ = RobotControlLib.recover()
         if bLoaded:
           daq_macros.robotOff()
           daq_macros.disableMount()
-          daq_lib.gui_message("Found a sample in the gripper - CALL STAFF! disableMount() executed.")
+          message = "FATAL ROBOT ERROR Found a sample in the gripper - CALL STAFF! disableMount() and robotOff() executed."
+          daq_lib.gui_message(message)
+          if raise_on_error:
+            raise Exception(message)
         else:
           RobotControlLib.runCmd("goHome")
       except Exception as e:
         e_s = str(e)
         daq_lib.gui_message("ROBOT Recover failed! " + e_s)
+        if raise_on_error:
+          raise
 
+    def recoverRobot(self):
+      self._recoverRobot()
 
-    def dryGripper(self):
+    def recoverRobotFloco(self):
+      self._recoverRobot(raise_on_error=True)
+
+    def _dryGripper(self, raise_on_error=False):
       try:
         saveThreshold = getPvDesc("warmupThresholdRBV")
         setPvDesc("warmupThreshold",50)
@@ -92,6 +102,14 @@ def dryGripper(self):
         e_s = str(e)
         daq_lib.gui_message("Dry gripper failed! " + e_s)
         setPvDesc("warmupThreshold",saveThreshold)
+        if raise_on_error:
+          raise
+
+    def dryGripper(self):
+      self._dryGripper()
+
+    def dryGripperFloco(self):
+      self._dryGripper(raise_on_error=True)
 
     def DewarAutoFillOn(self):
       RobotControlLib.runCmd("turnOnAutoFill")

gui/control_main.py

@@ -160,6 +160,8 @@ class ControlMain(QtWidgets.QMainWindow):
     lowMagCursorChangeSignal = QtCore.Signal(int, str)
     cryostreamTempSignal = QtCore.Signal(object)
     sampleZoomChangeSignal = QtCore.Signal(object)
+    gov_state_change_signal = QtCore.Signal(str)
+    dewar_plate_change_signal = QtCore.Signal(int)
 
     def __init__(self):
         super(ControlMain, self).__init__()
@@ -183,6 +185,8 @@ def __init__(self):
         if daq_utils.beamline != "nyx":
           self.albulaInterface = AlbulaInterface(ip=os.environ["EIGER_DCU_IP"], 
                                                  gov_message_pv_name=daq_utils.pvLookupDict["governorMessage"],)
+
+        self.dewar_plate_pos_pv = PV(daq_utils.pvLookupDict["dewarPlatePos"])
         self.initUI()
         self.initOphyd()
         self.govStateMessagePV = PV(daq_utils.pvLookupDict["governorMessage"])
@@ -1572,15 +1576,15 @@ def createSampleTab(self):
 
 
         self.hutchCornerCamThread = VideoThread(
-            parent=self, delay=HUTCH_TIMER_DELAY, url=getBlConfig("hutchCornerCamURL")
+            parent=self, delay=HUTCH_TIMER_DELAY, url=getBlConfig("hutchCornerCamURL") + "?resolution=320x180"
         )
         self.hutchCornerCamThread.frame_ready.connect(
             lambda frame: self.updateCam(self.pixmap_item_HutchCorner, frame)
         )
         self.hutchCornerCamThread.start()
 
         self.hutchTopCamThread = VideoThread(
-            parent=self, delay=HUTCH_TIMER_DELAY, url=getBlConfig("hutchTopCamURL")
+            parent=self, delay=HUTCH_TIMER_DELAY, url=getBlConfig("hutchTopCamURL") + "?resolution=320x180"
         )
         self.hutchTopCamThread.frame_ready.connect(
             lambda frame: self.updateCam(self.pixmap_item_HutchTop, frame)
@@ -1609,6 +1613,27 @@ def annealButtonCB(self):
         except:
             pass
 
+    def manage_gov_state_change(self, state: str):
+        # This function reacts to changes in governor state
+        # Currently changes what camera angle is shown in the center
+        if state in ("state SE", "transition SA to SE"):
+            logger.info("Govstate: %s", state)
+            self.sampleCameraThread.updateCam("http://xf17id1b-webcam1.nsls2.bnl.local/axis-cgi/mjpg/video.cgi?resolution=640x360")
+        elif state in ("transition SE to TA"):
+            logger.info("Govstate: %s", state)
+            self.sampleCameraThread.updateCam("http://xf17id1b-webcam4.nsls2.bnl.local/axis-cgi/mjpg/video.cgi?resolution=640x360")
+        elif state in ("state TA"):
+            logger.info("Govstate: %s", state)
+            self.sampleCameraThread.updateCam(self.capture)
+        elif state in ("state SA"):
+            logger.info("Govstate: %s", state)
+            self.sampleCameraThread.updateCam(self.capture)
+
+
+    def update_dewar_plate_position(self, state: int):
+        position = int(state) if state else "Rotating"
+        self.dewar_plate_position_status_widget.setText(f"Plate Position: {position}")
+
     def hideRastersCB(self, state):
         if state == QtCore.Qt.Checked:
             self.eraseRastersCB()
@@ -5131,6 +5156,12 @@ def pauseButtonStateCB(self, value=None, char_value=None, **kw):
         pauseButtonStateVar = value
         self.pauseButtonStateSignal.emit(pauseButtonStateVar)
 
+    def manage_gov_state_change_cb(self, value=None, char_value=None, **kw):
+        self.gov_state_change_signal.emit(char_value)
+
+    def dewar_plate_position_cb(self, value=None, char_value=None, **kw):
+        self.dewar_plate_change_signal.emit(value)
+
     def initOphyd(self):
         if daq_utils.beamline == "nyx":
             # initialize devices
@@ -5200,8 +5231,18 @@ def initUI(self):
         exitAction.setStatusTip("Exit application")
         exitAction.triggered.connect(self.closeAll)
         self.statusBar()
-        self.queue_collect_status_widget = QtWidgets.QLabel("Queue Collect: ON")
+        self.statusBar().setStyleSheet("QStatusBar { font-size: 14pt; }")
+        queue_collect_status = "ON" if getBlConfig("queueCollect") else "OFF"
+        self.queue_collect_status_widget = QtWidgets.QLabel(f"Queue Collect: {queue_collect_status}")
+        self.dewar_plate_position_status_widget = QtWidgets.QLabel(f"Plate Position: {int(self.dewar_plate_pos_pv.get())}")
+        sep = QtWidgets.QFrame()
+        sep.setFrameShape(QtWidgets.QFrame.VLine)
+        sep.setFrameShadow(QtWidgets.QFrame.Sunken)
+        sep.setLineWidth(1)
         self.statusBar().addPermanentWidget(self.queue_collect_status_widget)
+        self.statusBar().addPermanentWidget(sep)
+        self.statusBar().addPermanentWidget(self.dewar_plate_position_status_widget)
+
 
         menubar = self.menuBar()
         fileMenu = menubar.addMenu("&File")
@@ -5446,6 +5487,12 @@ def initCallbacks(self):
         self.lowMagCursorChangeSignal.connect(self.processLowMagCursorChange)
         self.lowMagCursorX_pv.add_callback(self.processLowMagCursorChangeCB, ID="x")
         self.lowMagCursorY_pv.add_callback(self.processLowMagCursorChangeCB, ID="y")
+
+        self.gov_state_change_signal.connect(self.manage_gov_state_change)
+        self.govStateMessagePV.add_callback(self.manage_gov_state_change_cb)
+
+        self.dewar_plate_change_signal.connect(self.update_dewar_plate_position)
+        self.dewar_plate_pos_pv.add_callback(self.dewar_plate_position_cb)
 
     def popupServerMessage(self, message_s):
         if self.popUpMessageInit:
@@ -5455,7 +5502,38 @@ def popupServerMessage(self, message_s):
         if message_s == "killMessage":
             return
         else:
-            self.popupMessage.showMessage(message_s)
+            try:
+                broadcast_command = json.loads(message_s)
+                self.processServerCommand(broadcast_command)
+            except json.JSONDecodeError:
+                self.popupMessage.showMessage(message_s)
+            except Exception as e:
+                logger.error(f"Could not process command: {e}")
+
+    def processServerCommand(self, command: "dict[str, Any]"):
+        """
+        Process a command broadcasted from the server
+        """
+        if "highlight_cells" in command:
+            if self.rasterList:
+                raster_item: RasterGroup = self.rasterList[-1]["graphicsItem"]
+                raster_item.set_highlighted_cells(command["highlight_cells"])
+        if "robot_online" in command:
+            self.staffScreenDialog.robotOnCheckBox.setChecked(command["robot_online"])
+        if "enable_mount" in command:
+            self.staffScreenDialog.enableMountCheckBox.setChecked(command["enable_mount"])
+        if "beam_check" in command:
+            self.staffScreenDialog.beamCheckOnCheckBox.setChecked(command["beam_check"])
+        if "queue_collect" in command:
+            self.staffScreenDialog.queueCollectOnCheckBox.setChecked(command["queue_collect"])
+            self.userScreenDialog.queueCollectOnCheckBox.setChecked(command["queue_collect"])
+            # Update status widget and unmount-cold control based on queue_collect state
+            self.queue_collect_status_widget.setText(
+                f"Queue Collect: {'ON' if command['queue_collect'] else 'OFF'}"
+            )
+            self.staffScreenDialog.gripperUnmountColdCheckBox.setEnabled(command["queue_collect"])
+        if "unmount_cold" in command:
+            self.staffScreenDialog.gripperUnmountColdCheckBox.setChecked(command["unmount_cold"])
 
     def printServerMessage(self, message_s):
         if self.textWindowMessageInit: