daemons/lvmdbusd/cfg.py | 10 ++++++++++ daemons/lvmdbusd/fetch.py | 19 ++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/daemons/lvmdbusd/cfg.py b/daemons/lvmdbusd/cfg.py index 771909f..be497d0 100644 --- a/daemons/lvmdbusd/cfg.py +++ b/daemons/lvmdbusd/cfg.py @@ -87,3 +87,13 @@ blackbox = None # RequestEntry ctor create_request_entry = None + + +def exit_daemon(): + """ + Exit the daemon cleanly + :return: + """ + if run and loop: + run.value = 0 + loop.quit() diff --git a/daemons/lvmdbusd/fetch.py b/daemons/lvmdbusd/fetch.py index 69a4aae..e8f3521 100644 --- a/daemons/lvmdbusd/fetch.py +++ b/daemons/lvmdbusd/fetch.py @@ -14,6 +14,7 @@ from . import cfg from .utils import MThreadRunner, log_debug, log_error import threading import queue +import time import traceback @@ -82,6 +83,8 @@ class StateUpdate(object): @staticmethod def update_thread(obj): + exception_count = 0 + queued_requests = [] while cfg.run.value != 0: # noinspection PyBroadException @@ -136,12 +139,26 @@ class StateUpdate(object): # wake up if we get an exception queued_requests = [] + # We retrieved OK, clear exception count + exception_count = 0 + except queue.Empty: pass - except Exception: + except Exception as e: st = traceback.format_exc() log_error("update_thread exception: \n%s" % st) cfg.blackbox.dump() + exception_count += 1 + if exception_count >= 5: + for i in queued_requests: + i.set_result(e) + + log_error("Too many errors in update_thread, exiting daemon") + cfg.exit_daemon() + + else: + # Slow things down when encountering errors + time.sleep(1) def __init__(self): self.lock = threading.RLock()