From 07c645f1d568b51dc5ae4566dcb7456daba6934c Mon Sep 17 00:00:00 2001
From: "Gao,Yan" <ygao@suse.com>
Date: Wed, 14 Dec 2016 16:08:38 +0100
Subject: [PATCH] Fix: dbus: Prevent lrmd from hanging on dbus calls
With "service" class of resources, by chance, lrmd hangs on futex()
syscall:
root@node2:~ # cat /proc/2503/stack
[<ffffffff810fa0c0>] futex_wait_queue_me+0xc0/0x130
[<ffffffff810faf23>] futex_wait+0x163/0x250
[<ffffffff810fc870>] do_futex+0xe0/0x540
[<ffffffff810fcd3e>] SyS_futex+0x6e/0x140
[<ffffffff815e142e>] entry_SYSCALL_64_fastpath+0x12/0x6d
[<ffffffffffffffff>] 0xffffffffffffffff
Cluster no longer behaves and cannot recover from the situation.
According to the backtrace, it seems due to the reentrancy of
dbus_connection_dispatch():
(gdb) bt
#0 0x00007f07f7d2e0af in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f07f6c29925 in _dbus_connection_acquire_dispatch (connection=0x13411f0) at dbus-connection.c:4142
#2 0x00007f07f6c2b3bc in dbus_connection_dispatch (connection=connection@entry=0x13411f0) at dbus-connection.c:4577
#3 0x00007f07f8d88e50 in pcmk_dbus_connection_dispatch (connection=connection@entry=0x13411f0, new_status=new_status@entry=DBUS_DISPATCH_DATA_REMAINS, data=data@entry=0x0) at dbus.c:410
#4 0x00007f07f6c29b70 in _dbus_connection_update_dispatch_status_and_unlock (connection=0x13411f0, new_status=DBUS_DISPATCH_DATA_REMAINS) at dbus-connection.c:4346
#5 0x00007f07f6c29f79 in check_for_reply_and_update_dispatch_unlocked (connection=connection@entry=0x13411f0, pending=pending@entry=0x135a8b0) at dbus-connection.c:2355
#6 0x00007f07f6c2a08b in _dbus_connection_block_pending_call (pending=0x135a8b0) at dbus-connection.c:2461
#7 0x00007f07f6c396ba in dbus_pending_call_block (pending=<optimized out>) at dbus-pending-call.c:741
#8 0x00007f07f8d8929c in pcmk_dbus_send_recv (msg=msg@entry=0x1340940, connection=0x13411f0, error=error@entry=0x7ffc5d148fc0, timeout=-1) at dbus.c:141
#9 0x00007f07f8d8d2d7 in systemd_unit_by_name (arg_name=arg_name@entry=0x133dcb0 "service", op=op@entry=0x0) at systemd.c:296
#10 0x00007f07f8d8d45b in systemd_unit_exists (name=name@entry=0x133dcb0 "service") at systemd.c:416
#11 0x00007f07f8d83dc5 in resources_find_service_class (agent=0x133dcb0 "service") at services.c:88
#12 0x0000000000405b05 in action_complete (action=0x134e0b0) at lrmd.c:876
#13 0x00007f07f8d867e3 in operation_finalize (op=0x134e0b0) at services_linux.c:257
#14 0x00007f07f8d899d8 in pcmk_dbus_lookup_result (reply=reply@entry=0x135cc80, data=data@entry=0x1355e30) at dbus.c:289
#15 0x00007f07f8d89ba4 in pcmk_dbus_lookup_cb (pending=<optimized out>, user_data=0x1355e30) at dbus.c:334
#16 0x00007f07f6c28032 in complete_pending_call_and_unlock (connection=0x13411f0, pending=0x135a2c0, message=<optimized out>) at dbus-connection.c:2331
#17 0x00007f07f6c2b401 in dbus_connection_dispatch (connection=connection@entry=0x13411f0) at dbus-connection.c:4626
#18 0x00007f07f8d88e50 in pcmk_dbus_connection_dispatch (connection=connection@entry=0x13411f0, new_status=new_status@entry=DBUS_DISPATCH_DATA_REMAINS, data=data@entry=0x0) at dbus.c:410
#19 0x00007f07f6c29b70 in _dbus_connection_update_dispatch_status_and_unlock (connection=0x13411f0, new_status=DBUS_DISPATCH_DATA_REMAINS) at dbus-connection.c:4346
#20 0x00007f07f6c29ca6 in _dbus_connection_handle_watch (watch=<optimized out>, condition=1, data=0x13411f0) at dbus-connection.c:1520
#21 0x00007f07f6c40f2a in dbus_watch_handle (watch=watch@entry=0x133d6a0, flags=flags@entry=1) at dbus-watch.c:722
#22 0x00007f07f8d887da in pcmk_dbus_watch_dispatch (userdata=0x133d6a0) at dbus.c:448
#23 0x00007f07f8fcfef7 in mainloop_gio_callback (gio=<optimized out>, condition=G_IO_IN, data=0x133f210) at mainloop.c:673
#24 0x00007f07f82a0015 in g_main_context_dispatch () from /usr/lib64/libglib-2.0.so.0
#25 0x00007f07f82a0388 in ?? () from /usr/lib64/libglib-2.0.so.0
#26 0x00007f07f82a064a in g_main_loop_run () from /usr/lib64/libglib-2.0.so.0
#27 0x0000000000402c0e in main (argc=<optimized out>, argv=0x7ffc5d149818) at main.c:476
As described in:
https://dbus.freedesktop.org/doc/api/html/group__DBusConnection.html#ga55ff88cd22c0672441c7deffbfb68fbf
, dbus_connection_dispatch() MUST NOT BE CALLED from inside the
DBusDispatchStatusFunction. It seems that pcmk_dbus_watch_dispatch() is
an appropriate place to do it instead.
---
lib/services/dbus.c | 30 +++++++++++++++++++++++++-----
1 file changed, 25 insertions(+), 5 deletions(-)
diff --git a/lib/services/dbus.c b/lib/services/dbus.c
index 4ce7dfe..0748c86 100644
--- a/lib/services/dbus.c
+++ b/lib/services/dbus.c
@@ -13,6 +13,8 @@
#define BUS_PROPERTY_IFACE "org.freedesktop.DBus.Properties"
+static GList *conn_dispatches = NULL;
+
struct db_getall_data {
char *name;
char *target;
@@ -445,17 +447,31 @@ pcmk_dbus_get_property(DBusConnection *connection, const char *target,
}
static void
-pcmk_dbus_connection_dispatch(DBusConnection *connection,
+pcmk_dbus_connection_dispatch_status(DBusConnection *connection,
DBusDispatchStatus new_status, void *data)
{
- crm_trace("status %d for %p", new_status, data);
+ crm_trace("New status %d for connection %p", new_status, connection);
if (new_status == DBUS_DISPATCH_DATA_REMAINS){
- dbus_connection_dispatch(connection);
+ conn_dispatches = g_list_prepend(conn_dispatches, connection);
+ }
+}
+
+static void
+pcmk_dbus_connections_dispatch()
+{
+ GList *gIter = NULL;
+
+ for (gIter = conn_dispatches; gIter != NULL; gIter = gIter->next) {
+ DBusConnection *connection = gIter->data;
while (dbus_connection_get_dispatch_status(connection) == DBUS_DISPATCH_DATA_REMAINS) {
+ crm_trace("Dispatching for connection %p", connection);
dbus_connection_dispatch(connection);
}
}
+
+ g_list_free(conn_dispatches);
+ conn_dispatches = NULL;
}
/* Copied from dbus-watch.c */
@@ -506,7 +522,11 @@ pcmk_dbus_watch_dispatch(gpointer userdata)
if(oom) {
crm_err("DBus encountered OOM while attempting to dispatch %p (%s)",
client, dbus_watch_flags_to_string(flags));
+
+ } else {
+ pcmk_dbus_connections_dispatch();
}
+
return 0;
}
@@ -616,6 +636,6 @@ pcmk_dbus_connection_setup_with_select(DBusConnection *c)
dbus_connection_set_watch_functions(c, pcmk_dbus_watch_add,
pcmk_dbus_watch_remove,
pcmk_dbus_watch_toggle, NULL, NULL);
- dbus_connection_set_dispatch_status_function(c, pcmk_dbus_connection_dispatch, NULL, NULL);
- pcmk_dbus_connection_dispatch(c, dbus_connection_get_dispatch_status(c), NULL);
+ dbus_connection_set_dispatch_status_function(c, pcmk_dbus_connection_dispatch_status, NULL, NULL);
+ pcmk_dbus_connection_dispatch_status(c, dbus_connection_get_dispatch_status(c), NULL);
}
--
1.8.3.1