Server Heartbeat

daq_main_common.py

@@ -1,3 +1,4 @@
+import random
 import string
 import sys
 import os
@@ -42,7 +43,8 @@ def pybass_init():
   daq_lib.init_var_channels()
   if getBlConfig(config_params.DETECTOR_OBJECT_TYPE) != config_params.DETECTOR_OBJECT_TYPE_NO_INIT:
     det_lib.init_detector()  
-  daq_lib.message_string_pv = beamline_support.pvCreate(daq_utils.beamlineComm + "message_string")    
+  daq_lib.message_string_pv = beamline_support.pvCreate(daq_utils.beamlineComm + "message_string")
+  daq_lib.heartbeat = beamline_support.pvCreate('XF:19ID2-ES:NYX{Comm}server_heartbeat')    
   daq_lib.gui_popup_message_string_pv = beamline_support.pvCreate(daq_utils.beamlineComm + "gui_popup_message_string")    
   beamline_lib.read_db()
   logger.info("init mots")
@@ -93,11 +95,17 @@ def process_command_file(command_file_name):
 
 
 def process_immediate_commands(frequency):
+  memory_beat = beamline_support.pvGet(daq_lib.heartbeat)
   while (1):
     if (len(immediate_command_list) > 0):
       command = immediate_command_list.pop(0)
       logger.info('immediate command: %s' % command)
       process_input(command)
+    if memory_beat == beamline_support.pvGet(daq_lib.heartbeat):
+      memory_beat = random.randint(0, 1000)
+      beamline_support.pvPut(daq_lib.heartbeat, str(memory_beat))
+    else:
+      print('Heartbeat mismatch, possibility of multiple servers running.')
     time.sleep(frequency)      
 
 def process_commands(frequency):

gui/control_main.py

@@ -200,6 +200,8 @@ def __init__(self):
 
         self.beamSize_pv = PV(daq_utils.beamlineComm + "size_mode")
         self.energy_pv = PV(daq_utils.motor_dict["energy"] + ".RBV")
+        self.heartbeat_pv = PV(daq_utils.beamlineComm + "server_heartbeat")
+        self.last_heartbeat = time.time()
         self.rasterStepDefs = {"Coarse": 30.0, "Fine": 20.0, "VFine": 10.0}
         self.createSampleTab()
 
@@ -1553,6 +1555,8 @@ def createSampleTab(self):
         serverCheckThread.visit_dir_changed.connect(QApplication.instance().quit)
         serverCheckThread.start()
 
+        self.heartbeat_thread = _thread.start_new_thread(self.monitorHeartbeat, ())
+
     def updateCam(self, pixmapItem: "QGraphicsPixmapItem", frame):
         pixmapItem.setPixmap(frame)
 
@@ -1576,6 +1580,22 @@ def annealButtonCB(self):
         except:
             pass
 
+    def heartbeatCB(self, **kwargs):
+        self.last_heartbeat = time.time()
+
+    def monitorHeartbeat(self):
+        '''
+        This monitors the frequency of changes to the heartbeat PV.
+        After 10 seconds of no changes, it reports the issue to the
+        user, and waits 30 more seconds to keep from spamming.
+        '''
+        while True:
+            time.sleep(1)
+            if time.time() - self.last_heartbeat > 10:
+                self.popup_message_string_pv.put("Server not responding")
+                time.sleep(30)
+                #self.popupServerMessage("Server not responding")
+
     def hideRastersCB(self, state):
         if state == QtCore.Qt.Checked:
             self.eraseRastersCB()
@@ -5367,6 +5387,7 @@ def initCallbacks(self):
         self.beamSizeSignal.connect(self.processBeamSize)
         self.beamSize_pv.add_callback(self.beamSizeChangedCB)
 
+        self.heartbeat_pv.add_callback(self.heartbeatCB)
         self.treeChanged_pv = PV(daq_utils.beamlineComm + "live_q_change_flag")
         self.refreshTreeSignal.connect(self.dewarTree.refreshTree)
         self.treeChanged_pv.add_callback(self.treeChangedCB)