From 51b03e5e892cd2446c84dc78e17b0ad3bdbe76d2 Mon Sep 17 00:00:00 2001
From: Michele Baldessari <michele@acksyn.org>
Date: Tue, 28 Mar 2017 16:21:52 +0200
Subject: [PATCH] Allow the rabbitmq cluster to work on pacemaker remote nodes

This was first observed via
https://bugzilla.redhat.com/show_bug.cgi?id=1435982. Due to the way
the resource agent looks for attrd entries, it will filter out any
node which does not have the @crmd=online attribute. This is the
case for pacemaker-remote nodes. To fix this we chose the more
conservative approach and only do an additional query when the first
one returned no entries. Note that this issue exhibits itself
when 'pcs status' reports rabbitmq started on a bunch of nodes:
Clone Set: rabbitmq-clone [rabbitmq]
    Started: [ overcloud-rabbit-0 overcloud-rabbit-1 overcloud-rabbit-2

But the cluster_status command returns a single node:
[root@overcloud-rabbit-1 ~]# rabbitmqctl cluster_status
Cluster status of node 'rabbit@overcloud-rabbit-1' ...
[{nodes,[{disc,['rabbit@overcloud-rabbit-1']}]},
 {running_nodes,['rabbit@overcloud-rabbit-1']},
 {cluster_name,<<"rabbit@overcloud-rabbit-1.localdomain">>},
 {partitions,[]},
 {alarms,[{'rabbit@overcloud-rabbit-1',[]}]}]

Also add some text in the help explaining that currently a mixture of
pacemaker-remote and pacemaker nodes is not supported.

We tested this change on a pacemaker-remote only setup successfully:
Clone Set: rabbitmq-clone [rabbitmq]
    Started: [ overcloud-rabbit-0 overcloud-rabbit-1 overcloud-rabbit-2

[root@overcloud-rabbit-0 ~]# rabbitmqctl cluster_status
Cluster status of node 'rabbit@overcloud-rabbit-0' ...
[{nodes,[{disc,['rabbit@overcloud-rabbit-0','rabbit@overcloud-rabbit-1',
                'rabbit@overcloud-rabbit-2']}]},
 {running_nodes,['rabbit@overcloud-rabbit-2','rabbit@overcloud-rabbit-1',
                 'rabbit@overcloud-rabbit-0']},
 {cluster_name,<<"rabbit@overcloud-rabbit-0.localdomain">>},
 {partitions,[]},
 {alarms,[{'rabbit@overcloud-rabbit-2',[]},
          {'rabbit@overcloud-rabbit-1',[]},
          {'rabbit@overcloud-rabbit-0',[]}]}]

Signed-Off-By: Michele Baldessari <michele@acksyn.org>
Signed-Off-By: Damien Ciabrini <dciabrin@redhat.com>
---
 heartbeat/rabbitmq-cluster | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/heartbeat/rabbitmq-cluster b/heartbeat/rabbitmq-cluster
index 74378be..6a17590 100755
--- a/heartbeat/rabbitmq-cluster
+++ b/heartbeat/rabbitmq-cluster
@@ -56,7 +56,9 @@ meta_data() {
 <version>1.0</version>
 
 <longdesc lang="en">
-Starts cloned rabbitmq cluster instance
+Starts cloned rabbitmq cluster instance. NB: note that this RA
+cannot be spawned across a mix of pacemaker and pacemaker-remote nodes.
+Only on pacemaker *or* pacemaker-remote nodes exclusively.
 </longdesc>
 <shortdesc lang="en">rabbitmq clustered</shortdesc>
 
@@ -111,7 +113,25 @@ rmq_local_node()
 
 rmq_join_list()
 {
-    cibadmin -Q --xpath "//node_state[@crmd='online']//nvpair[@name='$RMQ_CRM_ATTR_COOKIE']" | grep "$RMQ_CRM_ATTR_COOKIE" | sed -n -e "s/^.*value=.\(.*\)\".*$/\1/p"
+	local join_list=$(cibadmin -Q --xpath "//node_state[@crmd='online']//nvpair[@name='$RMQ_CRM_ATTR_COOKIE']" | grep "$RMQ_CRM_ATTR_COOKIE" | sed -n -e "s/^.*value=.\(.*\)\".*$/\1/p")
+	# If join_list is empty we want to check if there are any remote nodes
+	# where rabbitmq is allowed to run (i.e. nodes without the crmd=online selector)
+	if [ -z "$join_list" ]; then
+		# Get all the nodes written in the ATTR_COOKIE no matter if
+		# they are online or not. This will be one line per node like
+		# rabbit@overcloud-rabbit-0
+		# rabbit@overcloud-rabbit-1
+		# ...
+		local remote_join_list=$(cibadmin -Q --xpath "//node_state//nvpair[@name='$RMQ_CRM_ATTR_COOKIE']" | grep "$RMQ_CRM_ATTR_COOKIE" | sed -n -e "s/^.*value=.\(.*\)\".*$/\1/p")
+		# The following expression prepares a filter like '-e overcloud-rabbit-0 -e overcloud-rabbit-1 -e ...'
+		local filter=$(crm_mon -r --as-xml | xmllint --format --xpath "//nodes//node[@online='true' and @standby='false']/@name" - | xargs -n1 echo | awk -F= '{print "-e "$2}')
+		# export the intersection which gives us only the nodes that
+		# a) wrote their namein the cib attrd
+		# b) run on nodes where pacemaker_remote is enabled
+		join_list="$(echo $remote_join_list | grep $filter)"
+	fi
+
+	echo $join_list
 }
 
 rmq_write_nodename()