daemons/lvmdbusd/cfg.py | 10 ++++++++++
daemons/lvmdbusd/fetch.py | 19 ++++++++++++++++++-
2 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/daemons/lvmdbusd/cfg.py b/daemons/lvmdbusd/cfg.py
index 771909f..be497d0 100644
--- a/daemons/lvmdbusd/cfg.py
+++ b/daemons/lvmdbusd/cfg.py
@@ -87,3 +87,13 @@ blackbox = None
# RequestEntry ctor
create_request_entry = None
+
+
+def exit_daemon():
+ """
+ Exit the daemon cleanly
+ :return:
+ """
+ if run and loop:
+ run.value = 0
+ loop.quit()
diff --git a/daemons/lvmdbusd/fetch.py b/daemons/lvmdbusd/fetch.py
index 69a4aae..e8f3521 100644
--- a/daemons/lvmdbusd/fetch.py
+++ b/daemons/lvmdbusd/fetch.py
@@ -14,6 +14,7 @@ from . import cfg
from .utils import MThreadRunner, log_debug, log_error
import threading
import queue
+import time
import traceback
@@ -82,6 +83,8 @@ class StateUpdate(object):
@staticmethod
def update_thread(obj):
+ exception_count = 0
+
queued_requests = []
while cfg.run.value != 0:
# noinspection PyBroadException
@@ -136,12 +139,26 @@ class StateUpdate(object):
# wake up if we get an exception
queued_requests = []
+ # We retrieved OK, clear exception count
+ exception_count = 0
+
except queue.Empty:
pass
- except Exception:
+ except Exception as e:
st = traceback.format_exc()
log_error("update_thread exception: \n%s" % st)
cfg.blackbox.dump()
+ exception_count += 1
+ if exception_count >= 5:
+ for i in queued_requests:
+ i.set_result(e)
+
+ log_error("Too many errors in update_thread, exiting daemon")
+ cfg.exit_daemon()
+
+ else:
+ # Slow things down when encountering errors
+ time.sleep(1)
def __init__(self):
self.lock = threading.RLock()