Merge "Add handling to metric collect error"

Author: Zuul

cloudkitty/orchestrator.py

@@ -20,7 +20,6 @@
 import hashlib
 import multiprocessing
 import random
-import sys
 import time
 
 import cotyledon
@@ -331,7 +330,7 @@ def _get_result(metric):
                 # collection will be retried during the next collect
                 # cycle. In the future, we should implement a retrying
                 # system in workers
-                sys.exit(1)
+                raise e
 
         return self._do_execute_collection(_get_result, metrics)
 
@@ -640,12 +639,19 @@ def internal_run(self):
                                                         lock_name=lock_name,
                                                         scope_id=tenant_id))
 
+                _success = True
                 try:
                     self.process_scope(tenant_id)
+                except Exception as e:
+                    _success = False
+                    LOG.error('Error processing scope %s, we will try it again'
+                              ' later.', tenant_id)
+                    LOG.exception(e)
                 finally:
                     lock.release()
 
-                LOG.debug("Finished processing scope [%s].", tenant_id)
+                if _success:
+                    LOG.debug("Finished processing scope [%s].", tenant_id)
             else:
                 LOG.debug("Could not acquire lock [%s] for processing "
                           "scope [%s] with worker [%s].", lock_name,

releasenotes/notes/fix-processor-killed-on-metric-collect-error-824897dc2252b504.yaml

@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    Changes the metric collector exception handling to raise an exception
+    instead of killing the processor when an error happens in the metric
+    collect, so the problematic scope will be skipped till next
+    collection and the other scopes will be processed normally.