Blame SOURCES/pacemaker-async-systemd-reload.patch

1f016a
diff --git a/include/crm/common/logging.h b/include/crm/common/logging.h
1f016a
index 5b8e47f..642fa92 100644
1f016a
--- a/include/crm/common/logging.h
1f016a
+++ b/include/crm/common/logging.h
1f016a
@@ -157,7 +157,7 @@ unsigned int get_crm_log_level(void);
1f016a
         }                                                               \
1f016a
     } while(0)
1f016a
 
1f016a
-#  define do_crm_log_always(level, fmt, args...) qb_log(level, "%s: " fmt, __FUNCTION__ , ##args)
1f016a
+#  define do_crm_log_always(level, fmt, args...) qb_log(level, fmt , ##args)
1f016a
 
1f016a
 #  define crm_perror(level, fmt, args...) do {				\
1f016a
 	const char *err = strerror(errno);				\
1f016a
diff --git a/include/crm/services.h b/include/crm/services.h
1f016a
index 5310709..1a02656 100644
1f016a
--- a/include/crm/services.h
1f016a
+++ b/include/crm/services.h
1f016a
@@ -262,8 +262,9 @@ enum nagios_exitcode {
1f016a
  */
1f016a
     svc_action_t *services_action_create_generic(const char *exec, const char *args[]);
1f016a
 
1f016a
-    void
1f016a
-     services_action_free(svc_action_t * op);
1f016a
+    void services_action_cleanup(svc_action_t * op);
1f016a
+
1f016a
+    void services_action_free(svc_action_t * op);
1f016a
 
1f016a
     gboolean services_action_sync(svc_action_t * op);
1f016a
 
1f016a
diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
1f016a
index 15b354b..ea37c4b 100644
1f016a
--- a/lib/cib/cib_utils.c
1f016a
+++ b/lib/cib/cib_utils.c
1f016a
@@ -112,8 +112,13 @@ get_cib_copy(cib_t * cib)
1f016a
 {
1f016a
     xmlNode *xml_cib;
1f016a
     int options = cib_scope_local | cib_sync_call;
1f016a
-    int rc = cib->cmds->query(cib, NULL, &xml_cib, options);
1f016a
+    int rc = pcmk_ok;
1f016a
+
1f016a
+    if (cib->state == cib_disconnected) {
1f016a
+        return NULL;
1f016a
+    }
1f016a
 
1f016a
+    rc = cib->cmds->query(cib, NULL, &xml_cib, options);
1f016a
     if (rc == -EACCES) {
1f016a
         return NULL;
1f016a
 
1f016a
diff --git a/lib/services/dbus.c b/lib/services/dbus.c
1f016a
index c0153b5..f44b590 100644
1f016a
--- a/lib/services/dbus.c
1f016a
+++ b/lib/services/dbus.c
1f016a
@@ -145,7 +145,7 @@ DBusMessage *pcmk_dbus_send_recv(DBusMessage *msg, DBusConnection *connection, D
1f016a
     return reply;
1f016a
 }
1f016a
 
1f016a
-bool pcmk_dbus_send(DBusMessage *msg, DBusConnection *connection,
1f016a
+DBusPendingCall* pcmk_dbus_send(DBusMessage *msg, DBusConnection *connection,
1f016a
                     void(*done)(DBusPendingCall *pending, void *user_data), void *user_data)
1f016a
 {
1f016a
     DBusError error;
1f016a
@@ -161,27 +161,30 @@ bool pcmk_dbus_send(DBusMessage *msg, DBusConnection *connection,
1f016a
     // send message and get a handle for a reply
1f016a
     if (!dbus_connection_send_with_reply (connection, msg, &pending, -1/* aka. DBUS_TIMEOUT_USE_DEFAULT */)) { // -1 is default timeout
1f016a
         crm_err("Send with reply failed for %s", method);
1f016a
-        return FALSE;
1f016a
+        return NULL;
1f016a
 
1f016a
     } else if (pending == NULL) {
1f016a
         crm_err("No pending call found for %s", method);
1f016a
-        return FALSE;
1f016a
-
1f016a
+        return NULL;
1f016a
     }
1f016a
 
1f016a
+    crm_trace("DBus %s call sent", method);
1f016a
     if (dbus_pending_call_get_completed(pending)) {
1f016a
-        crm_info("DBus %s call completed too soon");
1f016a
-#if 1
1f016a
+        crm_info("DBus %s call completed too soon", method);
1f016a
+        if(done) {
1f016a
+#if 0
1f016a
         /* This sounds like a good idea, but allegedly it breaks things */
1f016a
         done(pending, user_data);
1f016a
+        pending = NULL;
1f016a
 #else
1f016a
         CRM_ASSERT(dbus_pending_call_set_notify(pending, done, user_data, NULL));
1f016a
 #endif
1f016a
+        }
1f016a
 
1f016a
-    } else {
1f016a
+    } else if(done) {
1f016a
         CRM_ASSERT(dbus_pending_call_set_notify(pending, done, user_data, NULL));
1f016a
     }
1f016a
-    return TRUE;
1f016a
+    return pending;
1f016a
 }
1f016a
 
1f016a
 bool pcmk_dbus_type_check(DBusMessage *msg, DBusMessageIter *field, int expected, const char *function, int line)
1f016a
@@ -286,6 +289,11 @@ pcmk_dbus_lookup_result(DBusMessage *reply, struct db_getall_data *data)
1f016a
         dbus_message_iter_next (&dict);
1f016a
     }
1f016a
 
1f016a
+    if(data->name && data->callback) {
1f016a
+        crm_trace("No value for property %s[%s]", data->object, data->name);
1f016a
+        data->callback(data->name, NULL, data->userdata);
1f016a
+    }
1f016a
+
1f016a
   cleanup:
1f016a
     free(data->target);
1f016a
     free(data->object);
1f016a
@@ -306,6 +314,9 @@ pcmk_dbus_lookup_cb(DBusPendingCall *pending, void *user_data)
1f016a
 
1f016a
     pcmk_dbus_lookup_result(reply, user_data);
1f016a
 
1f016a
+    if(pending) {
1f016a
+        dbus_pending_call_unref(pending);
1f016a
+    }
1f016a
     if(reply) {
1f016a
         dbus_message_unref(reply);
1f016a
     }
1f016a
@@ -375,20 +386,59 @@ static void pcmk_dbus_connection_dispatch(DBusConnection *connection, DBusDispat
1f016a
     crm_trace("status %d for %p", new_status, data);
1f016a
     if (new_status == DBUS_DISPATCH_DATA_REMAINS){
1f016a
         dbus_connection_dispatch(connection);
1f016a
+
1f016a
+        while (dbus_connection_get_dispatch_status(connection) == DBUS_DISPATCH_DATA_REMAINS) {
1f016a
+            dbus_connection_dispatch(connection);
1f016a
+        }
1f016a
     }
1f016a
 }
1f016a
 
1f016a
+/* Copied from dbus-watch.c */
1f016a
+
1f016a
+static const char*
1f016a
+dbus_watch_flags_to_string (int flags)
1f016a
+{
1f016a
+  const char *watch_type;
1f016a
+
1f016a
+  if ((flags & DBUS_WATCH_READABLE) &&
1f016a
+      (flags & DBUS_WATCH_WRITABLE))
1f016a
+    watch_type = "readwrite";
1f016a
+  else if (flags & DBUS_WATCH_READABLE)
1f016a
+    watch_type = "read";
1f016a
+  else if (flags & DBUS_WATCH_WRITABLE)
1f016a
+    watch_type = "write";
1f016a
+  else
1f016a
+    watch_type = "not read or write";
1f016a
+  return watch_type;
1f016a
+}
1f016a
+
1f016a
 static int
1f016a
 pcmk_dbus_watch_dispatch(gpointer userdata)
1f016a
 {
1f016a
+    bool oom = FALSE;
1f016a
     DBusWatch *watch = userdata;
1f016a
     int flags = dbus_watch_get_flags(watch);
1f016a
+    bool enabled = dbus_watch_get_enabled (watch);
1f016a
+    mainloop_io_t *client = dbus_watch_get_data(watch);
1f016a
 
1f016a
-    crm_trace("Dispatching %p with flags %d", watch, flags);
1f016a
-    if(flags & DBUS_WATCH_READABLE) {
1f016a
-        dbus_watch_handle(watch, DBUS_WATCH_READABLE);
1f016a
-    } else {
1f016a
-        dbus_watch_handle(watch, DBUS_WATCH_ERROR);
1f016a
+    crm_trace("Dispatching client %p: %s", client, dbus_watch_flags_to_string(flags));
1f016a
+    if (enabled && is_set(flags, DBUS_WATCH_READABLE)) {
1f016a
+        oom = !dbus_watch_handle(watch, flags);
1f016a
+
1f016a
+    } else if (enabled && is_set(flags, DBUS_WATCH_READABLE)) {
1f016a
+        oom = !dbus_watch_handle(watch, flags);
1f016a
+
1f016a
+    } else if(enabled) {
1f016a
+        oom = !dbus_watch_handle(watch, DBUS_WATCH_ERROR);
1f016a
+    }
1f016a
+
1f016a
+    if(flags != dbus_watch_get_flags(watch)) {
1f016a
+        flags = dbus_watch_get_flags(watch);
1f016a
+        crm_trace("Dispatched client %p: %s (%d)", client, dbus_watch_flags_to_string(flags), flags);
1f016a
+    }
1f016a
+
1f016a
+    if(oom) {
1f016a
+        crm_err("DBus encountered OOM while attempting to dispatch %p (%s)", client, dbus_watch_flags_to_string(flags));
1f016a
     }
1f016a
     return 0;
1f016a
 }
1f016a
@@ -396,7 +446,8 @@ pcmk_dbus_watch_dispatch(gpointer userdata)
1f016a
 static void
1f016a
 pcmk_dbus_watch_destroy(gpointer userdata)
1f016a
 {
1f016a
-    crm_trace("Destroyed %p", userdata);
1f016a
+    mainloop_io_t *client = dbus_watch_get_data(userdata);
1f016a
+    crm_trace("Destroyed %p", client);
1f016a
 }
1f016a
 
1f016a
 
1f016a
@@ -412,7 +463,7 @@ pcmk_dbus_watch_add(DBusWatch *watch, void *data){
1f016a
     mainloop_io_t *client = mainloop_add_fd(
1f016a
         "dbus", G_PRIORITY_DEFAULT, fd, watch, &pcmk_dbus_cb);
1f016a
 
1f016a
-    crm_trace("Added %p with fd=%d", watch, fd);
1f016a
+    crm_trace("Added watch %p with fd=%d to client %p", watch, fd, client);
1f016a
     dbus_watch_set_data(watch, client, NULL);
1f016a
     return TRUE;
1f016a
 }
1f016a
@@ -429,14 +480,14 @@ static void
1f016a
 pcmk_dbus_watch_remove(DBusWatch *watch, void *data){
1f016a
     mainloop_io_t *client = dbus_watch_get_data(watch);
1f016a
 
1f016a
-    crm_trace("Removed %p", watch);
1f016a
+    crm_trace("Removed client %p (%p)", client, data);
1f016a
     mainloop_del_fd(client);
1f016a
 }
1f016a
 
1f016a
 static gboolean
1f016a
 pcmk_dbus_timeout_dispatch(gpointer data)
1f016a
 {
1f016a
-    crm_trace("Timeout for %p");
1f016a
+    crm_info("Timeout %p expired", data);
1f016a
     dbus_timeout_handle(data);
1f016a
     return FALSE;
1f016a
 }
1f016a
@@ -445,6 +496,8 @@ static dbus_bool_t
1f016a
 pcmk_dbus_timeout_add(DBusTimeout *timeout, void *data){
1f016a
     guint id = g_timeout_add(dbus_timeout_get_interval(timeout), pcmk_dbus_timeout_dispatch, timeout);
1f016a
 
1f016a
+    crm_trace("Adding timeout %p (%ld)", timeout, dbus_timeout_get_interval(timeout));
1f016a
+
1f016a
     if(id) {
1f016a
         dbus_timeout_set_data(timeout, GUINT_TO_POINTER(id), NULL);
1f016a
     }
1f016a
@@ -456,6 +509,8 @@ pcmk_dbus_timeout_remove(DBusTimeout *timeout, void *data){
1f016a
     void *vid = dbus_timeout_get_data(timeout);
1f016a
     guint id = GPOINTER_TO_UINT(vid);
1f016a
 
1f016a
+    crm_trace("Removing timeout %p (%p)", timeout, data);
1f016a
+
1f016a
     if(id) {
1f016a
         g_source_remove(id);
1f016a
         dbus_timeout_set_data(timeout, 0, NULL);
1f016a
@@ -464,7 +519,11 @@ pcmk_dbus_timeout_remove(DBusTimeout *timeout, void *data){
1f016a
 
1f016a
 static void
1f016a
 pcmk_dbus_timeout_toggle(DBusTimeout *timeout, void *data){
1f016a
-    if(dbus_timeout_get_enabled(timeout)) {
1f016a
+    bool enabled = dbus_timeout_get_enabled(timeout);
1f016a
+
1f016a
+    crm_trace("Toggling timeout for %p to %s", timeout, enabled?"off":"on");
1f016a
+
1f016a
+    if(enabled) {
1f016a
         pcmk_dbus_timeout_add(timeout, data);
1f016a
     } else {
1f016a
         pcmk_dbus_timeout_remove(timeout, data);
1f016a
diff --git a/lib/services/pcmk-dbus.h b/lib/services/pcmk-dbus.h
1f016a
index ed80c5f..468020e 100644
1f016a
--- a/lib/services/pcmk-dbus.h
1f016a
+++ b/lib/services/pcmk-dbus.h
1f016a
@@ -2,7 +2,7 @@ DBusConnection *pcmk_dbus_connect(void);
1f016a
 void pcmk_dbus_connection_setup_with_select(DBusConnection *c);
1f016a
 void pcmk_dbus_disconnect(DBusConnection *connection);
1f016a
 
1f016a
-bool pcmk_dbus_send(DBusMessage *msg, DBusConnection *connection,
1f016a
+DBusPendingCall *pcmk_dbus_send(DBusMessage *msg, DBusConnection *connection,
1f016a
                     void(*done)(DBusPendingCall *pending, void *user_data), void *user_data);
1f016a
 DBusMessage *pcmk_dbus_send_recv(DBusMessage *msg, DBusConnection *connection, DBusError *error);
1f016a
 bool pcmk_dbus_type_check(DBusMessage *msg, DBusMessageIter *field, int expected, const char *function, int line);
1f016a
diff --git a/lib/services/services.c b/lib/services/services.c
1f016a
index 9936c72..582fbe1 100644
1f016a
--- a/lib/services/services.c
1f016a
+++ b/lib/services/services.c
1f016a
@@ -303,18 +303,24 @@ services_action_create_generic(const char *exec, const char *args[])
1f016a
 }
1f016a
 
1f016a
 void
1f016a
-services_action_free(svc_action_t * op)
1f016a
+services_action_cleanup(svc_action_t * op)
1f016a
 {
1f016a
-    unsigned int i;
1f016a
-
1f016a
-    if (op == NULL) {
1f016a
-        return;
1f016a
+    if(op->opaque->timerid != 0) {
1f016a
+        crm_trace("Removing timer for call %s to %s", op->action, op->rsc);
1f016a
+        g_source_remove(op->opaque->timerid);
1f016a
+        op->opaque->timerid = 0;
1f016a
     }
1f016a
 
1f016a
-    if (op->opaque->repeat_timer) {
1f016a
-        g_source_remove(op->opaque->repeat_timer);
1f016a
-        op->opaque->repeat_timer = 0;
1f016a
+    if(op->opaque->pending) {
1f016a
+        crm_trace("Cleaning up pending dbus call %p %s for %s", op->opaque->pending, op->action, op->rsc);
1f016a
+        if(dbus_pending_call_get_completed(op->opaque->pending)) {
1f016a
+            crm_warn("Pending dbus call %s for %s did not complete", op->action, op->rsc);
1f016a
+        }
1f016a
+        dbus_pending_call_cancel(op->opaque->pending);
1f016a
+        dbus_pending_call_unref(op->opaque->pending);
1f016a
+        op->opaque->pending = NULL;
1f016a
     }
1f016a
+
1f016a
     if (op->opaque->stderr_gsource) {
1f016a
         mainloop_del_fd(op->opaque->stderr_gsource);
1f016a
         op->opaque->stderr_gsource = NULL;
1f016a
@@ -324,6 +330,23 @@ services_action_free(svc_action_t * op)
1f016a
         mainloop_del_fd(op->opaque->stdout_gsource);
1f016a
         op->opaque->stdout_gsource = NULL;
1f016a
     }
1f016a
+}
1f016a
+
1f016a
+void
1f016a
+services_action_free(svc_action_t * op)
1f016a
+{
1f016a
+    unsigned int i;
1f016a
+
1f016a
+    if (op == NULL) {
1f016a
+        return;
1f016a
+    }
1f016a
+
1f016a
+    services_action_cleanup(op);
1f016a
+
1f016a
+    if (op->opaque->repeat_timer) {
1f016a
+        g_source_remove(op->opaque->repeat_timer);
1f016a
+        op->opaque->repeat_timer = 0;
1f016a
+    }
1f016a
 
1f016a
     free(op->id);
1f016a
     free(op->opaque->exec);
1f016a
diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c
1f016a
index 2279e4e..8d6f450 100644
1f016a
--- a/lib/services/services_linux.c
1f016a
+++ b/lib/services/services_linux.c
1f016a
@@ -264,6 +264,8 @@ operation_finalize(svc_action_t * op)
1f016a
         services_action_free(op);
1f016a
         return TRUE;
1f016a
     }
1f016a
+
1f016a
+    services_action_cleanup(op);
1f016a
     return FALSE;
1f016a
 }
1f016a
 
1f016a
diff --git a/lib/services/services_private.h b/lib/services/services_private.h
1f016a
index dd759e3..bcf882c 100644
1f016a
--- a/lib/services/services_private.h
1f016a
+++ b/lib/services/services_private.h
1f016a
@@ -19,6 +19,10 @@
1f016a
 #ifndef __MH_SERVICES_PRIVATE_H__
1f016a
 #  define __MH_SERVICES_PRIVATE_H__
1f016a
 
1f016a
+#if SUPPORT_DBUS
1f016a
+#  include <dbus/dbus.h>
1f016a
+#endif
1f016a
+
1f016a
 struct svc_action_private_s {
1f016a
     char *exec;
1f016a
     char *args[255];
1f016a
@@ -31,6 +35,10 @@ struct svc_action_private_s {
1f016a
 
1f016a
     int stdout_fd;
1f016a
     mainloop_io_t *stdout_gsource;
1f016a
+#if SUPPORT_DBUS
1f016a
+    DBusPendingCall* pending;
1f016a
+    unsigned timerid;
1f016a
+#endif
1f016a
 };
1f016a
 
1f016a
 GList *services_os_get_directory_list(const char *root, gboolean files, gboolean executable);
1f016a
diff --git a/lib/services/systemd.c b/lib/services/systemd.c
1f016a
index 9a7b078..51ade44 100644
1f016a
--- a/lib/services/systemd.c
1f016a
+++ b/lib/services/systemd.c
1f016a
@@ -110,20 +110,48 @@ systemd_service_name(const char *name)
1f016a
     return g_strdup_printf("%s.service", name);
1f016a
 }
1f016a
 
1f016a
-static bool
1f016a
-systemd_daemon_reload(void)
1f016a
+static void
1f016a
+systemd_daemon_reload_complete(DBusPendingCall *pending, void *user_data)
1f016a
 {
1f016a
-    /* TODO: Make this asynchronous */
1f016a
-    const char *method = "Reload";
1f016a
+    DBusError error;
1f016a
     DBusMessage *reply = NULL;
1f016a
-    DBusMessage *msg = systemd_new_method(BUS_NAME".Manager", method);
1f016a
+    unsigned int reload_count = GPOINTER_TO_UINT(user_data);
1f016a
 
1f016a
-    CRM_ASSERT(msg != NULL);
1f016a
-    reply = pcmk_dbus_send_recv(msg, systemd_proxy, NULL);
1f016a
-    dbus_message_unref(msg);
1f016a
+    dbus_error_init(&error);
1f016a
+    if(pending) {
1f016a
+        reply = dbus_pending_call_steal_reply(pending);
1f016a
+    }
1f016a
+
1f016a
+    if(pcmk_dbus_find_error("Reload", pending, reply, &error)) {
1f016a
+        crm_err("Could not issue systemd reload %d: %s", reload_count, error.message);
1f016a
+
1f016a
+    } else {
1f016a
+        crm_trace("Reload %d complete", reload_count);
1f016a
+    }
1f016a
+
1f016a
+    if(pending) {
1f016a
+        dbus_pending_call_unref(pending);
1f016a
+    }
1f016a
     if(reply) {
1f016a
         dbus_message_unref(reply);
1f016a
     }
1f016a
+}
1f016a
+
1f016a
+static bool
1f016a
+systemd_daemon_reload(void)
1f016a
+{
1f016a
+    static unsigned int reload_count = 0;
1f016a
+    const char *method = "Reload";
1f016a
+
1f016a
+
1f016a
+    reload_count++;
1f016a
+    if(reload_count % 10 == 0) {
1f016a
+        DBusMessage *msg = systemd_new_method(BUS_NAME".Manager", method);
1f016a
+
1f016a
+        CRM_ASSERT(msg != NULL);
1f016a
+        pcmk_dbus_send(msg, systemd_proxy, systemd_daemon_reload_complete, GUINT_TO_POINTER(reload_count));
1f016a
+        dbus_message_unref(msg);
1f016a
+    }
1f016a
     return TRUE;
1f016a
 }
1f016a
 
1f016a
@@ -155,13 +183,22 @@ static void
1f016a
 systemd_loadunit_cb(DBusPendingCall *pending, void *user_data)
1f016a
 {
1f016a
     DBusMessage *reply = NULL;
1f016a
+    svc_action_t * op = user_data;
1f016a
 
1f016a
     if(pending) {
1f016a
         reply = dbus_pending_call_steal_reply(pending);
1f016a
     }
1f016a
 
1f016a
+    if(op) {
1f016a
+        crm_trace("Got result: %p for %p for %s, %s", reply, pending, op->rsc, op->action);
1f016a
+    } else {
1f016a
+        crm_trace("Got result: %p for %p", reply, pending);
1f016a
+    }
1f016a
     systemd_loadunit_result(reply, user_data);
1f016a
 
1f016a
+    if(pending) {
1f016a
+        dbus_pending_call_unref(pending);
1f016a
+    }
1f016a
     if(reply) {
1f016a
         dbus_message_unref(reply);
1f016a
     }
1f016a
@@ -213,6 +250,7 @@ systemd_unit_by_name(const gchar * arg_name, svc_action_t *op)
1f016a
     }
1f016a
 
1f016a
     pcmk_dbus_send(msg, systemd_proxy, systemd_loadunit_cb, op);
1f016a
+    dbus_message_unref(msg);
1f016a
     return NULL;
1f016a
 }
1f016a
 
1f016a
@@ -421,6 +459,12 @@ systemd_async_dispatch(DBusPendingCall *pending, void *user_data)
1f016a
         reply = dbus_pending_call_steal_reply(pending);
1f016a
     }
1f016a
 
1f016a
+    if(op) {
1f016a
+        crm_trace("Got result: %p for %p for %s, %s", reply, pending, op->rsc, op->action);
1f016a
+    } else {
1f016a
+        crm_trace("Got result: %p for %p", reply, pending);
1f016a
+    }
1f016a
+    op->opaque->pending = NULL;
1f016a
     systemd_exec_result(reply, op);
1f016a
 
1f016a
     if(pending) {
1f016a
@@ -437,10 +481,13 @@ static void
1f016a
 systemd_unit_check(const char *name, const char *state, void *userdata)
1f016a
 {
1f016a
     svc_action_t * op = userdata;
1f016a
-    
1f016a
-    CRM_ASSERT(state != NULL);
1f016a
 
1f016a
-    if (g_strcmp0(state, "active") == 0) {
1f016a
+    crm_trace("Resource %s has %s='%s'", op->rsc, name, state);
1f016a
+
1f016a
+    if(state == NULL) {
1f016a
+        op->rc = PCMK_OCF_NOT_RUNNING;
1f016a
+
1f016a
+    } else if (g_strcmp0(state, "active") == 0) {
1f016a
         op->rc = PCMK_OCF_OK;
1f016a
     } else if (g_strcmp0(state, "activating") == 0) {
1f016a
         op->rc = PCMK_OCF_PENDING;
1f016a
@@ -449,6 +496,7 @@ systemd_unit_check(const char *name, const char *state, void *userdata)
1f016a
     }
1f016a
 
1f016a
     if (op->synchronous == FALSE) {
1f016a
+        op->opaque->pending = NULL;
1f016a
         operation_finalize(op);
1f016a
     }
1f016a
 }
1f016a
@@ -539,28 +587,29 @@ systemd_unit_exec_with_unit(svc_action_t * op, const char *unit)
1f016a
     }
1f016a
 
1f016a
     if (op->synchronous == FALSE) {
1f016a
-        return pcmk_dbus_send(msg, systemd_proxy, systemd_async_dispatch, op);
1f016a
+        DBusPendingCall* pending = pcmk_dbus_send(msg, systemd_proxy, systemd_async_dispatch, op);
1f016a
+
1f016a
+        dbus_message_unref(msg);
1f016a
+        if(pending) {
1f016a
+            dbus_pending_call_ref(pending);
1f016a
+            op->opaque->pending = pending;
1f016a
+            return TRUE;
1f016a
+        }
1f016a
+        return FALSE;
1f016a
 
1f016a
     } else {
1f016a
         DBusError error;
1f016a
 
1f016a
         reply = pcmk_dbus_send_recv(msg, systemd_proxy, &error);
1f016a
+        dbus_message_unref(msg);
1f016a
         systemd_exec_result(reply, op);
1f016a
 
1f016a
         if(reply) {
1f016a
             dbus_message_unref(reply);
1f016a
         }
1f016a
-        if(msg) {
1f016a
-            dbus_message_unref(msg);
1f016a
-        }
1f016a
-
1f016a
         return FALSE;
1f016a
     }
1f016a
 
1f016a
-    if(msg) {
1f016a
-        dbus_message_unref(msg);
1f016a
-    }
1f016a
-
1f016a
   cleanup:
1f016a
     if (op->synchronous == FALSE) {
1f016a
         operation_finalize(op);
1f016a
@@ -570,6 +619,18 @@ systemd_unit_exec_with_unit(svc_action_t * op, const char *unit)
1f016a
     return op->rc == PCMK_OCF_OK;
1f016a
 }
1f016a
 
1f016a
+static gboolean
1f016a
+systemd_timeout_callback(gpointer p)
1f016a
+{
1f016a
+    svc_action_t * op = p;
1f016a
+
1f016a
+    op->opaque->timerid = 0;
1f016a
+    crm_warn("%s operation on systemd unit %s named '%s' timed out", op->action, op->agent, op->rsc);
1f016a
+    operation_finalize(op);
1f016a
+
1f016a
+    return FALSE;
1f016a
+}
1f016a
+
1f016a
 gboolean
1f016a
 systemd_unit_exec(svc_action_t * op)
1f016a
 {
1f016a
@@ -596,6 +657,7 @@ systemd_unit_exec(svc_action_t * op)
1f016a
     free(unit);
1f016a
 
1f016a
     if (op->synchronous == FALSE) {
1f016a
+        op->opaque->timerid = g_timeout_add(op->timeout + 5000, systemd_timeout_callback, op);
1f016a
         return TRUE;
1f016a
     }
1f016a
 
1f016a
diff --git a/lib/services/upstart.c b/lib/services/upstart.c
1f016a
index 4c7211d..01ff817 100644
1f016a
--- a/lib/services/upstart.c
1f016a
+++ b/lib/services/upstart.c
1f016a
@@ -513,8 +513,15 @@ upstart_job_exec(svc_action_t * op, gboolean synchronous)
1f016a
     CRM_LOG_ASSERT(dbus_message_append_args(msg, DBUS_TYPE_BOOLEAN, &arg_wait, DBUS_TYPE_INVALID));
1f016a
 
1f016a
     if (op->synchronous == FALSE) {
1f016a
+        DBusPendingCall* pending = pcmk_dbus_send(msg, upstart_proxy, upstart_async_dispatch, op);
1f016a
         free(job);
1f016a
-        return pcmk_dbus_send(msg, upstart_proxy, upstart_async_dispatch, op);
1f016a
+
1f016a
+        if(pending) {
1f016a
+            dbus_pending_call_ref(pending);
1f016a
+            op->opaque->pending = pending;
1f016a
+            return TRUE;
1f016a
+        }
1f016a
+        return FALSE;
1f016a
     }
1f016a
 
1f016a
     dbus_error_init(&error);
1f016a
diff --git a/lrmd/regression.py.in b/lrmd/regression.py.in
1f016a
index a9a32ef..649c984 100755
1f016a
--- a/lrmd/regression.py.in
1f016a
+++ b/lrmd/regression.py.in
1f016a
@@ -27,12 +27,12 @@ build_dir="@abs_top_builddir@"
1f016a
 test_dir=sys.path[0]
1f016a
 
1f016a
 new_path=os.environ['PATH']
1f016a
-
1f016a
 if os.path.exists("%s/regression.py.in" % test_dir):
1f016a
     print "Running tests from the source tree: %s (%s)" % (build_dir, test_dir)
1f016a
     new_path = "%s/lrmd:%s" % (build_dir, new_path)    # For lrmd, lrmd_test and pacemaker_remoted
1f016a
     new_path = "%s/tools:%s" % (build_dir, new_path)   # For crm_resource
1f016a
     new_path = "%s/fencing:%s" % (build_dir, new_path) # For stonithd
1f016a
+
1f016a
 else:
1f016a
     print "Running tests from the install tree: @CRM_DAEMON_DIR@ (not %s)" % test_dir
1f016a
     new_path = "@CRM_DAEMON_DIR@:%s" % (new_path) # For stonithd, lrmd, lrmd_test and pacemaker_remoted
1f016a
@@ -434,13 +434,15 @@ if __name__ == "__main__":
1f016a
 			for ra in [ "Dummy", "Stateful", "ping" ]:
1f016a
 				os.system("cp %s/extra/resources/%s @OCF_RA_DIR@/pacemaker/%s" % (build_dir, ra, ra))
1f016a
 				os.system("chmod a+x @OCF_RA_DIR@/pacemaker/%s" % (ra))
1f016a
-			else:
1f016a
-				# Assume it's installed
1f016a
-				print "Using @datadir@/@PACKAGE@/tests/cts/LSBDummy"
1f016a
-				os.system("cp @datadir@/@PACKAGE@/tests/cts/LSBDummy /etc/init.d/LSBDummy")
1f016a
 
1f016a
-				os.system("chmod a+x /etc/init.d/LSBDummy")
1f016a
-				os.system("ls -al /etc/init.d/LSBDummy")
1f016a
+                else:
1f016a
+                    # Assume it's installed
1f016a
+                    print "Using @datadir@/@PACKAGE@/tests/cts/LSBDummy"
1f016a
+                    os.system("cp @datadir@/@PACKAGE@/tests/cts/LSBDummy /etc/init.d/LSBDummy")
1f016a
+
1f016a
+                os.system("chmod a+x /etc/init.d/LSBDummy")
1f016a
+                os.system("ls -al /etc/init.d/LSBDummy")
1f016a
+
1f016a
 		os.system("mkdir -p @CRM_CORE_DIR@/root")
1f016a
 
1f016a
 		if os.path.exists("/bin/systemctl"):
1f016a
@@ -747,6 +749,33 @@ if __name__ == "__main__":
1f016a
 		test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
1f016a
 			"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
1f016a
 
1f016a
+	### stress tests ###
1f016a
+	def build_stress_tests(self):
1f016a
+		timeout = "-t 20000"
1f016a
+		iterations = 25
1f016a
+
1f016a
+		test = self.new_test("ocf_stress", "Verify systemd dbus connection works under load")
1f016a
+		for i in range(iterations):
1f016a
+			test.add_cmd("-c register_rsc -r rsc_%s %s -C ocf -P heartbeat -T Dummy -l \"NEW_EVENT event_type:register rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
1f016a
+			test.add_cmd("-c exec -r rsc_%s -a start %s -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:start rc:ok op_status:complete\"" % (i, timeout, i))
1f016a
+			test.add_cmd("-c exec -r rsc_%s -a monitor %s -i 1000 -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:monitor rc:ok op_status:complete\"" % (i, timeout, i))
1f016a
+		for i in range(iterations):
1f016a
+			test.add_cmd("-c exec -r rsc_%s -a stop %s  -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:stop rc:ok op_status:complete\"" % (i, timeout, i))
1f016a
+			test.add_cmd("-c unregister_rsc -r rsc_%s %s -l \"NEW_EVENT event_type:unregister rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
1f016a
+
1f016a
+
1f016a
+		if "systemd" in self.rsc_classes:
1f016a
+			test = self.new_test("systemd_stress", "Verify systemd dbus connection works under load")
1f016a
+			for i in range(iterations):
1f016a
+				test.add_cmd("-c register_rsc -r rsc_%s %s -C systemd -T lrmd_dummy_daemon -l \"NEW_EVENT event_type:register rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
1f016a
+				test.add_cmd("-c exec -r rsc_%s -a start %s -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:start rc:ok op_status:complete\"" % (i, timeout, i))
1f016a
+				test.add_cmd("-c exec -r rsc_%s -a monitor %s -i 1000 -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:monitor rc:ok op_status:complete\"" % (i, timeout, i))
1f016a
+
1f016a
+			for i in range(iterations):
1f016a
+				test.add_cmd("-c exec -r rsc_%s -a stop %s -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:stop rc:ok op_status:complete\"" % (i, timeout, i))
1f016a
+				test.add_cmd("-c unregister_rsc -r rsc_%s %s -l \"NEW_EVENT event_type:unregister rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
1f016a
+
1f016a
+
1f016a
 	### These are tests that target specific cases ###
1f016a
 	def build_custom_tests(self):
1f016a
 
1f016a
@@ -1016,6 +1045,7 @@ def main(argv):
1f016a
 	tests.build_multi_rsc_tests()
1f016a
 	tests.build_negative_tests()
1f016a
 	tests.build_custom_tests()
1f016a
+	tests.build_stress_tests()
1f016a
 
1f016a
 	tests.setup_test_environment()
1f016a