diff --git a/daq_main_common.py b/daq_main_common.py index d39d40d9..4e1e0c05 100755 --- a/daq_main_common.py +++ b/daq_main_common.py @@ -1,3 +1,4 @@ +import random import string import sys import os @@ -42,7 +43,8 @@ def pybass_init(): daq_lib.init_var_channels() if getBlConfig(config_params.DETECTOR_OBJECT_TYPE) != config_params.DETECTOR_OBJECT_TYPE_NO_INIT: det_lib.init_detector() - daq_lib.message_string_pv = beamline_support.pvCreate(daq_utils.beamlineComm + "message_string") + daq_lib.message_string_pv = beamline_support.pvCreate(daq_utils.beamlineComm + "message_string") + daq_lib.heartbeat = beamline_support.pvCreate('XF:19ID2-ES:NYX{Comm}server_heartbeat') daq_lib.gui_popup_message_string_pv = beamline_support.pvCreate(daq_utils.beamlineComm + "gui_popup_message_string") beamline_lib.read_db() logger.info("init mots") @@ -93,11 +95,17 @@ def process_command_file(command_file_name): def process_immediate_commands(frequency): + memory_beat = beamline_support.pvGet(daq_lib.heartbeat) while (1): if (len(immediate_command_list) > 0): command = immediate_command_list.pop(0) logger.info('immediate command: %s' % command) process_input(command) + if memory_beat == beamline_support.pvGet(daq_lib.heartbeat): + memory_beat = random.randint(0, 1000) + beamline_support.pvPut(daq_lib.heartbeat, str(memory_beat)) + else: + print('Heartbeat mismatch, possibility of multiple servers running.') time.sleep(frequency) def process_commands(frequency): diff --git a/gui/control_main.py b/gui/control_main.py index 272b42f0..fa7843c4 100644 --- a/gui/control_main.py +++ b/gui/control_main.py @@ -200,6 +200,8 @@ def __init__(self): self.beamSize_pv = PV(daq_utils.beamlineComm + "size_mode") self.energy_pv = PV(daq_utils.motor_dict["energy"] + ".RBV") + self.heartbeat_pv = PV(daq_utils.beamlineComm + "server_heartbeat") + self.last_heartbeat = time.time() self.rasterStepDefs = {"Coarse": 30.0, "Fine": 20.0, "VFine": 10.0} self.createSampleTab() @@ -1553,6 +1555,8 @@ def createSampleTab(self): serverCheckThread.visit_dir_changed.connect(QApplication.instance().quit) serverCheckThread.start() + self.heartbeat_thread = _thread.start_new_thread(self.monitorHeartbeat, ()) + def updateCam(self, pixmapItem: "QGraphicsPixmapItem", frame): pixmapItem.setPixmap(frame) @@ -1576,6 +1580,22 @@ def annealButtonCB(self): except: pass + def heartbeatCB(self, **kwargs): + self.last_heartbeat = time.time() + + def monitorHeartbeat(self): + ''' + This monitors the frequency of changes to the heartbeat PV. + After 10 seconds of no changes, it reports the issue to the + user, and waits 30 more seconds to keep from spamming. + ''' + while True: + time.sleep(1) + if time.time() - self.last_heartbeat > 10: + self.popup_message_string_pv.put("Server not responding") + time.sleep(30) + #self.popupServerMessage("Server not responding") + def hideRastersCB(self, state): if state == QtCore.Qt.Checked: self.eraseRastersCB() @@ -5367,6 +5387,7 @@ def initCallbacks(self): self.beamSizeSignal.connect(self.processBeamSize) self.beamSize_pv.add_callback(self.beamSizeChangedCB) + self.heartbeat_pv.add_callback(self.heartbeatCB) self.treeChanged_pv = PV(daq_utils.beamlineComm + "live_q_change_flag") self.refreshTreeSignal.connect(self.dewarTree.refreshTree) self.treeChanged_pv.add_callback(self.treeChangedCB)