From bc599f0f30c039a72540002d9a41a93c15626837 Mon Sep 17 00:00:00 2001

From: Ivan Devat <idevat@redhat.com>

Date: Wed, 14 Sep 2016 09:04:57 +0200

Subject: [PATCH] squash bz1158805 Add support for qdevice/qnetd pro

9c7f37ef37bb lib: do not merge external processes' stdout and stderr

db3ada5e27aa warn on stopping/destroying currently used qdevice

18df73397b54 handle SBD when removing qdevice from a cluster

766f86954b46 Allow to re-run "cluster node add" if failed due to qdevice

---

 pcs/cluster.py                               |  44 ++--

 pcs/common/report_codes.py                   |   6 +-

 pcs/common/tools.py                          |   3 +

 pcs/lib/booth/status.py                      |  27 +-

 pcs/lib/booth/test/test_status.py            |  26 +-

 pcs/lib/cib/tools.py                         |   7 +-

 pcs/lib/commands/booth.py                    |   5 +-

 pcs/lib/commands/qdevice.py                  |  35 ++-

 pcs/lib/commands/quorum.py                   |  12 +-

 pcs/lib/commands/test/test_booth.py          |   4 +-

 pcs/lib/corosync/live.py                     |  26 +-

 pcs/lib/corosync/qdevice_client.py           |   9 +-

 pcs/lib/corosync/qdevice_net.py              |  77 ++++--

 pcs/lib/external.py                          | 105 +++++---

 pcs/lib/pacemaker.py                         |  71 ++++--

 pcs/lib/reports.py                           |  97 ++++----

 pcs/lib/resource_agent.py                    |  31 ++-

 pcs/lib/sbd.py                               |   4 +-

 pcs/qdevice.py                               |   4 +-

 pcs/test/resources/corosync-qdevice.conf     |  34 +++

 pcs/test/test_common_tools.py                |  32 +++

 pcs/test/test_lib_cib_tools.py               |  10 +-

 pcs/test/test_lib_commands_qdevice.py        | 155 +++++++++++-

 pcs/test/test_lib_commands_quorum.py         | 105 +++++++-

 pcs/test/test_lib_corosync_live.py           |  30 ++-

 pcs/test/test_lib_corosync_qdevice_client.py |   8 +-

 pcs/test/test_lib_corosync_qdevice_net.py    | 110 +++++---

 pcs/test/test_lib_external.py                | 167 +++++++------

 pcs/test/test_lib_pacemaker.py               | 359 ++++++++++++++++++---------

 pcs/test/test_lib_resource_agent.py          |  39 ++-

 pcs/test/test_lib_sbd.py                     |  12 +-

 31 files changed, 1166 insertions(+), 488 deletions(-)

 create mode 100644 pcs/test/resources/corosync-qdevice.conf

diff --git a/pcs/cluster.py b/pcs/cluster.py

index 577e08e..e5ad1ec 100644

--- a/pcs/cluster.py

+++ b/pcs/cluster.py

@@ -1414,7 +1414,6 @@ def cluster_node(argv):

                 "cluster is not configured for RRP, "

                 "you must not specify ring 1 address for the node"

             )

-        corosync_conf = None

         (canAdd, error) =  utils.canAddNodeToCluster(node0)

         if not canAdd:

             utils.err("Unable to add '%s' to cluster: %s" % (node0, error))

@@ -1422,7 +1421,29 @@ def cluster_node(argv):

         report_processor = lib_env.report_processor

         node_communicator = lib_env.node_communicator()

         node_addr = NodeAddresses(node0, node1)

+

+        # First set up everything else than corosync. Once the new node is

+        # present in corosync.conf / cluster.conf, it's considered part of a

+        # cluster and the node add command cannot be run again. So we need to

+        # minimize the amout of actions (and therefore possible failures) after

+        # adding the node to corosync.

         try:

+            # qdevice setup

+            if not utils.is_rhel6():

+                conf_facade = corosync_conf_facade.from_string(

+                    utils.getCorosyncConf()

+                )

+                qdevice_model, qdevice_model_options, _ = conf_facade.get_quorum_device_settings()

+                if qdevice_model == "net":

+                    _add_device_model_net(

+                        lib_env,

+                        qdevice_model_options["host"],

+                        conf_facade.get_cluster_name(),

+                        [node_addr],

+                        skip_offline_nodes=False

+                    )

+

+            # sbd setup

             if lib_sbd.is_sbd_enabled(utils.cmd_runner()):

                 if "--watchdog" not in utils.pcs_options:

                     watchdog = settings.sbd_watchdog_default

@@ -1463,6 +1484,7 @@ def cluster_node(argv):

                     report_processor, node_communicator, node_addr

                 )

+            # booth setup

             booth_sync.send_all_config_to_node(

                 node_communicator,

                 report_processor,

@@ -1477,6 +1499,8 @@ def cluster_node(argv):

                 [node_communicator_exception_to_report_item(e)]

             )

+        # Now add the new node to corosync.conf / cluster.conf

+        corosync_conf = None

         for my_node in utils.getNodesFromCorosyncConf():

             retval, output = utils.addLocalNode(my_node, node0, node1)

             if retval != 0:

@@ -1512,24 +1536,6 @@ def cluster_node(argv):

                 except:

                     utils.err('Unable to communicate with pcsd')

-            # set qdevice-net certificates if needed

-            if not utils.is_rhel6():

-                try:

-                    conf_facade = corosync_conf_facade.from_string(

-                        corosync_conf

-                    )

-                    qdevice_model, qdevice_model_options, _ = conf_facade.get_quorum_device_settings()

-                    if qdevice_model == "net":

-                        _add_device_model_net(

-                            lib_env,

-                            qdevice_model_options["host"],

-                            conf_facade.get_cluster_name(),

-                            [node_addr],

-                            skip_offline_nodes=False

-                        )

-                except LibraryError as e:

-                    process_library_reports(e.args)

-

             print("Setting up corosync...")

             utils.setCorosyncConfig(node0, corosync_conf)

             if "--enable" in utils.pcs_options:

diff --git a/pcs/common/report_codes.py b/pcs/common/report_codes.py

index e6a86ec..23e931f 100644

--- a/pcs/common/report_codes.py

+++ b/pcs/common/report_codes.py

@@ -8,17 +8,18 @@ from __future__ import (

 # force cathegories

 FORCE_ACTIVE_RRP = "ACTIVE_RRP"

 FORCE_ALERT_RECIPIENT_VALUE_NOT_UNIQUE = "FORCE_ALERT_RECIPIENT_VALUE_NOT_UNIQUE"

-FORCE_BOOTH_REMOVE_FROM_CIB = "FORCE_BOOTH_REMOVE_FROM_CIB"

 FORCE_BOOTH_DESTROY = "FORCE_BOOTH_DESTROY"

+FORCE_BOOTH_REMOVE_FROM_CIB = "FORCE_BOOTH_REMOVE_FROM_CIB"

 FORCE_CONSTRAINT_DUPLICATE = "CONSTRAINT_DUPLICATE"

 FORCE_CONSTRAINT_MULTIINSTANCE_RESOURCE = "CONSTRAINT_MULTIINSTANCE_RESOURCE"

 FORCE_FILE_OVERWRITE = "FORCE_FILE_OVERWRITE"

 FORCE_LOAD_THRESHOLD = "LOAD_THRESHOLD"

+FORCE_METADATA_ISSUE = "METADATA_ISSUE"

 FORCE_OPTIONS = "OPTIONS"

 FORCE_QDEVICE_MODEL = "QDEVICE_MODEL"

+FORCE_QDEVICE_USED = "QDEVICE_USED"

 FORCE_UNKNOWN_AGENT = "UNKNOWN_AGENT"

 FORCE_UNSUPPORTED_AGENT = "UNSUPPORTED_AGENT"

-FORCE_METADATA_ISSUE = "METADATA_ISSUE"

 SKIP_OFFLINE_NODES = "SKIP_OFFLINE_NODES"

 SKIP_UNREADABLE_CONFIG = "SKIP_UNREADABLE_CONFIG"

@@ -135,6 +136,7 @@ QDEVICE_NOT_DEFINED = "QDEVICE_NOT_DEFINED"

 QDEVICE_NOT_INITIALIZED = "QDEVICE_NOT_INITIALIZED"

 QDEVICE_CLIENT_RELOAD_STARTED = "QDEVICE_CLIENT_RELOAD_STARTED"

 QDEVICE_REMOVE_OR_CLUSTER_STOP_NEEDED = "QDEVICE_REMOVE_OR_CLUSTER_STOP_NEEDED"

+QDEVICE_USED_BY_CLUSTERS = "QDEVICE_USED_BY_CLUSTERS"

 REQUIRED_OPTION_IS_MISSING = "REQUIRED_OPTION_IS_MISSING"

 RESOURCE_CLEANUP_ERROR = "RESOURCE_CLEANUP_ERROR"

 RESOURCE_CLEANUP_TOO_TIME_CONSUMING = 'RESOURCE_CLEANUP_TOO_TIME_CONSUMING'

diff --git a/pcs/common/tools.py b/pcs/common/tools.py

index 275f6b9..01194a5 100644

--- a/pcs/common/tools.py

+++ b/pcs/common/tools.py

@@ -38,3 +38,6 @@ def format_environment_error(e):

     if e.filename:

         return "{0}: '{1}'".format(e.strerror, e.filename)

     return e.strerror

+

+def join_multilines(strings):

+    return "\n".join([a.strip() for a in strings if a.strip()])

diff --git a/pcs/lib/booth/status.py b/pcs/lib/booth/status.py

index 4b93161..87cdc05 100644

--- a/pcs/lib/booth/status.py

+++ b/pcs/lib/booth/status.py

@@ -6,6 +6,7 @@ from __future__ import (

 )

 from pcs import settings

+from pcs.common.tools import join_multilines

 from pcs.lib.booth import reports

 from pcs.lib.errors import LibraryError

@@ -14,28 +15,36 @@ def get_daemon_status(runner, name=None):

     cmd = [settings.booth_binary, "status"]

     if name:

         cmd += ["-c", name]

-    output, return_value = runner.run(cmd)

+    stdout, stderr, return_value = runner.run(cmd)

     # 7 means that there is no booth instance running

     if return_value not in [0, 7]:

-        raise LibraryError(reports.booth_daemon_status_error(output))

-    return output

+        raise LibraryError(

+            reports.booth_daemon_status_error(join_multilines([stderr, stdout]))

+        )

+    return stdout

 def get_tickets_status(runner, name=None):

     cmd = [settings.booth_binary, "list"]

     if name:

         cmd += ["-c", name]

-    output, return_value = runner.run(cmd)

+    stdout, stderr, return_value = runner.run(cmd)

     if return_value != 0:

-        raise LibraryError(reports.booth_tickets_status_error(output))

-    return output

+        raise LibraryError(

+            reports.booth_tickets_status_error(

+                join_multilines([stderr, stdout])

+            )

+        )

+    return stdout

 def get_peers_status(runner, name=None):

     cmd = [settings.booth_binary, "peers"]

     if name:

         cmd += ["-c", name]

-    output, return_value = runner.run(cmd)

+    stdout, stderr, return_value = runner.run(cmd)

     if return_value != 0:

-        raise LibraryError(reports.booth_peers_status_error(output))

-    return output

+        raise LibraryError(

+            reports.booth_peers_status_error(join_multilines([stderr, stdout]))

+        )

+    return stdout

diff --git a/pcs/lib/booth/test/test_status.py b/pcs/lib/booth/test/test_status.py

index d47ffca..dfb7354 100644

--- a/pcs/lib/booth/test/test_status.py

+++ b/pcs/lib/booth/test/test_status.py

@@ -30,34 +30,34 @@ class GetDaemonStatusTest(TestCase):

         self.mock_run = mock.MagicMock(spec_set=CommandRunner)

     def test_no_name(self):

-        self.mock_run.run.return_value = ("output", 0)

+        self.mock_run.run.return_value = ("output", "", 0)

         self.assertEqual("output", lib.get_daemon_status(self.mock_run))

         self.mock_run.run.assert_called_once_with(

             [settings.booth_binary, "status"]

         )

     def test_with_name(self):

-        self.mock_run.run.return_value = ("output", 0)

+        self.mock_run.run.return_value = ("output", "", 0)

         self.assertEqual("output", lib.get_daemon_status(self.mock_run, "name"))

         self.mock_run.run.assert_called_once_with(

             [settings.booth_binary, "status", "-c", "name"]

         )

     def test_daemon_not_running(self):

-        self.mock_run.run.return_value = ("", 7)

+        self.mock_run.run.return_value = ("", "error", 7)

         self.assertEqual("", lib.get_daemon_status(self.mock_run))

         self.mock_run.run.assert_called_once_with(

             [settings.booth_binary, "status"]

         )

     def test_failure(self):

-        self.mock_run.run.return_value = ("out", 1)

+        self.mock_run.run.return_value = ("out", "error", 1)

         assert_raise_library_error(

             lambda: lib.get_daemon_status(self.mock_run),

             (

                 Severities.ERROR,

                 report_codes.BOOTH_DAEMON_STATUS_ERROR,

-                {"reason": "out"}

+                {"reason": "error\nout"}

             )

         )

         self.mock_run.run.assert_called_once_with(

@@ -70,14 +70,14 @@ class GetTicketsStatusTest(TestCase):

         self.mock_run = mock.MagicMock(spec_set=CommandRunner)

     def test_no_name(self):

-        self.mock_run.run.return_value = ("output", 0)

+        self.mock_run.run.return_value = ("output", "", 0)

         self.assertEqual("output", lib.get_tickets_status(self.mock_run))

         self.mock_run.run.assert_called_once_with(

             [settings.booth_binary, "list"]

         )

     def test_with_name(self):

-        self.mock_run.run.return_value = ("output", 0)

+        self.mock_run.run.return_value = ("output", "", 0)

         self.assertEqual(

             "output", lib.get_tickets_status(self.mock_run, "name")

         )

@@ -86,14 +86,14 @@ class GetTicketsStatusTest(TestCase):

         )

     def test_failure(self):

-        self.mock_run.run.return_value = ("out", 1)

+        self.mock_run.run.return_value = ("out", "error", 1)

         assert_raise_library_error(

             lambda: lib.get_tickets_status(self.mock_run),

             (

                 Severities.ERROR,

                 report_codes.BOOTH_TICKET_STATUS_ERROR,

                 {

-                    "reason": "out"

+                    "reason": "error\nout"

                 }

             )

         )

@@ -107,28 +107,28 @@ class GetPeersStatusTest(TestCase):

         self.mock_run = mock.MagicMock(spec_set=CommandRunner)

     def test_no_name(self):

-        self.mock_run.run.return_value = ("output", 0)

+        self.mock_run.run.return_value = ("output", "", 0)

         self.assertEqual("output", lib.get_peers_status(self.mock_run))

         self.mock_run.run.assert_called_once_with(

             [settings.booth_binary, "peers"]

         )

     def test_with_name(self):

-        self.mock_run.run.return_value = ("output", 0)

+        self.mock_run.run.return_value = ("output", "", 0)

         self.assertEqual("output", lib.get_peers_status(self.mock_run, "name"))

         self.mock_run.run.assert_called_once_with(

             [settings.booth_binary, "peers", "-c", "name"]

         )

     def test_failure(self):

-        self.mock_run.run.return_value = ("out", 1)

+        self.mock_run.run.return_value = ("out", "error", 1)

         assert_raise_library_error(

             lambda: lib.get_peers_status(self.mock_run),

             (

                 Severities.ERROR,

                 report_codes.BOOTH_PEERS_STATUS_ERROR,

                 {

-                    "reason": "out"

+                    "reason": "error\nout"

                 }

             )

         )

diff --git a/pcs/lib/cib/tools.py b/pcs/lib/cib/tools.py

index 8141360..6285931 100644

--- a/pcs/lib/cib/tools.py

+++ b/pcs/lib/cib/tools.py

@@ -11,6 +11,7 @@ import tempfile

 from lxml import etree

 from pcs import settings

+from pcs.common.tools import join_multilines

 from pcs.lib import reports

 from pcs.lib.errors import LibraryError

 from pcs.lib.pacemaker_values import validate_id

@@ -181,7 +182,7 @@ def upgrade_cib(cib, runner):

         temp_file = tempfile.NamedTemporaryFile("w+", suffix=".pcs")

         temp_file.write(etree.tostring(cib).decode())

         temp_file.flush()

-        output, retval = runner.run(

+        stdout, stderr, retval = runner.run(

             [

                 os.path.join(settings.pacemaker_binaries, "cibadmin"),

                 "--upgrade",

@@ -192,7 +193,9 @@ def upgrade_cib(cib, runner):

         if retval != 0:

             temp_file.close()

-            raise LibraryError(reports.cib_upgrade_failed(output))

+            raise LibraryError(

+                reports.cib_upgrade_failed(join_multilines([stderr, stdout]))

+            )

         temp_file.seek(0)

         return etree.fromstring(temp_file.read())

diff --git a/pcs/lib/commands/booth.py b/pcs/lib/commands/booth.py

index 7a3d348..bea966c 100644

--- a/pcs/lib/commands/booth.py

+++ b/pcs/lib/commands/booth.py

@@ -10,6 +10,7 @@ import os.path

 from functools import partial

 from pcs import settings

+from pcs.common.tools import join_multilines

 from pcs.lib import external, reports

 from pcs.lib.booth import (

     config_exchange,

@@ -185,7 +186,7 @@ def ticket_operation(operation, env, name, ticket, site_ip):

             )

         site_ip = site_ip_list[0]

-    command_output, return_code = env.cmd_runner().run([

+    stdout, stderr, return_code = env.cmd_runner().run([

         settings.booth_binary, operation,

         "-s", site_ip,

         ticket

@@ -195,7 +196,7 @@ def ticket_operation(operation, env, name, ticket, site_ip):

         raise LibraryError(

             booth_reports.booth_ticket_operation_failed(

                 operation,

-                command_output,

+                join_multilines([stderr, stdout]),

                 site_ip,

                 ticket

             )

diff --git a/pcs/lib/commands/qdevice.py b/pcs/lib/commands/qdevice.py

index 1d1d85f..ca0ae86 100644

--- a/pcs/lib/commands/qdevice.py

+++ b/pcs/lib/commands/qdevice.py

@@ -8,9 +8,10 @@ from __future__ import (

 import base64

 import binascii

+from pcs.common import report_codes

 from pcs.lib import external, reports

 from pcs.lib.corosync import qdevice_net

-from pcs.lib.errors import LibraryError

+from pcs.lib.errors import LibraryError, ReportItemSeverity

 def qdevice_setup(lib_env, model, enable, start):

@@ -31,13 +32,20 @@ def qdevice_setup(lib_env, model, enable, start):

     if start:

         _service_start(lib_env, qdevice_net.qdevice_start)

-def qdevice_destroy(lib_env, model):

+def qdevice_destroy(lib_env, model, proceed_if_used=False):

     """

     Stop and disable qdevice on local host and remove its configuration

     string model qdevice model to destroy

+    bool procced_if_used destroy qdevice even if it is used by clusters

     """

     _ensure_not_cman(lib_env)

     _check_model(model)

+    _check_qdevice_not_used(

+        lib_env.report_processor,

+        lib_env.cmd_runner(),

+        model,

+        proceed_if_used

+    )

     _service_stop(lib_env, qdevice_net.qdevice_stop)

     _service_disable(lib_env, qdevice_net.qdevice_disable)

     qdevice_net.qdevice_destroy()

@@ -83,12 +91,20 @@ def qdevice_start(lib_env, model):

     _check_model(model)

     _service_start(lib_env, qdevice_net.qdevice_start)

-def qdevice_stop(lib_env, model):

+def qdevice_stop(lib_env, model, proceed_if_used=False):

     """

     stop qdevice now on local host

+    string model qdevice model to destroy

+    bool procced_if_used stop qdevice even if it is used by clusters

     """

     _ensure_not_cman(lib_env)

     _check_model(model)

+    _check_qdevice_not_used(

+        lib_env.report_processor,

+        lib_env.cmd_runner(),

+        model,

+        proceed_if_used

+    )

     _service_stop(lib_env, qdevice_net.qdevice_stop)

 def qdevice_kill(lib_env, model):

@@ -176,6 +192,19 @@ def _check_model(model):

             reports.invalid_option_value("model", model, ["net"])

         )

+def _check_qdevice_not_used(reporter, runner, model, force=False):

+    _check_model(model)

+    connected_clusters = []

+    if model == "net":

+        status = qdevice_net.qdevice_status_cluster_text(runner)

+        connected_clusters = qdevice_net.qdevice_connected_clusters(status)

+    if connected_clusters:

+        reporter.process(reports.qdevice_used_by_clusters(

+            connected_clusters,

+            ReportItemSeverity.WARNING if force else ReportItemSeverity.ERROR,

+            None if force else report_codes.FORCE_QDEVICE_USED

+        ))

+

 def _service_start(lib_env, func):

     lib_env.report_processor.process(

         reports.service_start_started("quorum device")

diff --git a/pcs/lib/commands/quorum.py b/pcs/lib/commands/quorum.py

index 7fb7bb4..8390fc6 100644

--- a/pcs/lib/commands/quorum.py

+++ b/pcs/lib/commands/quorum.py

@@ -283,14 +283,23 @@ def remove_device(lib_env, skip_offline_nodes=False):

     cfg = lib_env.get_corosync_conf()

     model, dummy_options, dummy_options = cfg.get_quorum_device_settings()

     cfg.remove_quorum_device()

+

+    if lib_env.is_corosync_conf_live:

+        # fix quorum options for SBD to work properly

+        if sbd.atb_has_to_be_enabled(lib_env.cmd_runner(), cfg):

+            lib_env.report_processor.process(reports.sbd_requires_atb())

+            cfg.set_quorum_options(

+                lib_env.report_processor, {"auto_tie_breaker": "1"}

+            )

+

     lib_env.push_corosync_conf(cfg, skip_offline_nodes)

     if lib_env.is_corosync_conf_live:

+        communicator = lib_env.node_communicator()

         # disable qdevice

         lib_env.report_processor.process(

             reports.service_disable_started("corosync-qdevice")

         )

-        communicator = lib_env.node_communicator()

         parallel_nodes_communication_helper(

             qdevice_client.remote_client_disable,

             [

@@ -304,7 +313,6 @@ def remove_device(lib_env, skip_offline_nodes=False):

         lib_env.report_processor.process(

             reports.service_stop_started("corosync-qdevice")

         )

-        communicator = lib_env.node_communicator()

         parallel_nodes_communication_helper(

             qdevice_client.remote_client_stop,

             [

diff --git a/pcs/lib/commands/test/test_booth.py b/pcs/lib/commands/test/test_booth.py

index 08d2c79..6bcab2b 100644

--- a/pcs/lib/commands/test/test_booth.py

+++ b/pcs/lib/commands/test/test_booth.py

@@ -520,7 +520,7 @@ class TicketOperationTest(TestCase):

         )

     def test_raises_when_command_fail(self):

-        mock_run = mock.Mock(return_value=("some message", 1))

+        mock_run = mock.Mock(return_value=("some message", "error", 1))

         mock_env = mock.MagicMock(

             cmd_runner=mock.Mock(return_value=mock.MagicMock(run=mock_run))

         )

@@ -533,7 +533,7 @@ class TicketOperationTest(TestCase):

                 report_codes.BOOTH_TICKET_OPERATION_FAILED,

                 {

                     "operation": "grant",

-                    "reason": "some message",

+                    "reason": "error\nsome message",

                     "site_ip": "1.2.3.4",

                     "ticket_name": "ABC",

                 }

diff --git a/pcs/lib/corosync/live.py b/pcs/lib/corosync/live.py

index 1e68c31..67aa0e4 100644

--- a/pcs/lib/corosync/live.py

+++ b/pcs/lib/corosync/live.py

@@ -8,6 +8,7 @@ from __future__ import (

 import os.path

 from pcs import settings

+from pcs.common.tools import join_multilines

 from pcs.lib import reports

 from pcs.lib.errors import LibraryError

 from pcs.lib.external import NodeCommunicator

@@ -41,42 +42,39 @@ def reload_config(runner):

     """

     Ask corosync to reload its configuration

     """

-    output, retval = runner.run([

+    stdout, stderr, retval = runner.run([

         os.path.join(settings.corosync_binaries, "corosync-cfgtool"),

         "-R"

     ])

-    if retval != 0 or "invalid option" in output:

-        raise LibraryError(

-            reports.corosync_config_reload_error(output.rstrip())

-        )

+    message = join_multilines([stderr, stdout])

+    if retval != 0 or "invalid option" in message:

+        raise LibraryError(reports.corosync_config_reload_error(message))

 def get_quorum_status_text(runner):

     """

     Get runtime quorum status from the local node

     """

-    output, retval = runner.run([

+    stdout, stderr, retval = runner.run([

         os.path.join(settings.corosync_binaries, "corosync-quorumtool"),

         "-p"

     ])

     # retval is 0 on success if node is not in partition with quorum

     # retval is 1 on error OR on success if node has quorum

-    if retval not in [0, 1]:

-        raise LibraryError(

-            reports.corosync_quorum_get_status_error(output)

-        )

-    return output

+    if retval not in [0, 1] or stderr.strip():

+        raise LibraryError(reports.corosync_quorum_get_status_error(stderr))

+    return stdout

 def set_expected_votes(runner, votes):

     """

     set expected votes in live cluster to specified value

     """

-    output, retval = runner.run([

+    stdout, stderr, retval = runner.run([

         os.path.join(settings.corosync_binaries, "corosync-quorumtool"),

         # format votes to handle the case where they are int

         "-e", "{0}".format(votes)

     ])

     if retval != 0:

         raise LibraryError(

-            reports.corosync_quorum_set_expected_votes_error(output)

+            reports.corosync_quorum_set_expected_votes_error(stderr)

         )

-    return output

+    return stdout

diff --git a/pcs/lib/corosync/qdevice_client.py b/pcs/lib/corosync/qdevice_client.py

index 98fbb0e..c9d0095 100644

--- a/pcs/lib/corosync/qdevice_client.py

+++ b/pcs/lib/corosync/qdevice_client.py

@@ -8,6 +8,7 @@ from __future__ import (

 import os.path

 from pcs import settings

+from pcs.common.tools import join_multilines

 from pcs.lib import reports

 from pcs.lib.errors import LibraryError

@@ -23,12 +24,14 @@ def get_status_text(runner, verbose=False):

     ]

     if verbose:

         cmd.append("-v")

-    output, retval = runner.run(cmd)

+    stdout, stderr, retval = runner.run(cmd)

     if retval != 0:

         raise LibraryError(

-            reports.corosync_quorum_get_status_error(output)

+            reports.corosync_quorum_get_status_error(

+                join_multilines([stderr, stdout])

+            )

         )

-    return output

+    return stdout

 def remote_client_enable(reporter, node_communicator, node):

     """

diff --git a/pcs/lib/corosync/qdevice_net.py b/pcs/lib/corosync/qdevice_net.py

index 4054592..200e45a 100644

--- a/pcs/lib/corosync/qdevice_net.py

+++ b/pcs/lib/corosync/qdevice_net.py

@@ -15,6 +15,7 @@ import shutil

 import tempfile

 from pcs import settings

+from pcs.common.tools import join_multilines

 from pcs.lib import external, reports

 from pcs.lib.errors import LibraryError

@@ -41,12 +42,15 @@ def qdevice_setup(runner):

     if external.is_dir_nonempty(settings.corosync_qdevice_net_server_certs_dir):

         raise LibraryError(reports.qdevice_already_initialized(__model))

-    output, retval = runner.run([

+    stdout, stderr, retval = runner.run([

         __qnetd_certutil, "-i"

     ])

     if retval != 0:

         raise LibraryError(

-            reports.qdevice_initialization_error(__model, output.rstrip())

+            reports.qdevice_initialization_error(

+                __model,

+                join_multilines([stderr, stdout])

+            )

         )

 def qdevice_initialized():

@@ -78,10 +82,15 @@ def qdevice_status_generic_text(runner, verbose=False):

     cmd = [__qnetd_tool, "-s"]

     if verbose:

         cmd.append("-v")

-    output, retval = runner.run(cmd)

+    stdout, stderr, retval = runner.run(cmd)

     if retval != 0:

-        raise LibraryError(reports.qdevice_get_status_error(__model, output))

-    return output

+        raise LibraryError(

+            reports.qdevice_get_status_error(

+                __model,

+                join_multilines([stderr, stdout])

+            )

+        )

+    return stdout

 def qdevice_status_cluster_text(runner, cluster=None, verbose=False):

     """

@@ -94,10 +103,24 @@ def qdevice_status_cluster_text(runner, cluster=None, verbose=False):

         cmd.append("-v")

     if cluster:

         cmd.extend(["-c", cluster])

-    output, retval = runner.run(cmd)

+    stdout, stderr, retval = runner.run(cmd)

     if retval != 0:

-        raise LibraryError(reports.qdevice_get_status_error(__model, output))

-    return output

+        raise LibraryError(

+            reports.qdevice_get_status_error(

+                __model,

+                join_multilines([stderr, stdout])

+            )

+        )

+    return stdout

+

+def qdevice_connected_clusters(status_cluster_text):

+    connected_clusters = []

+    regexp = re.compile(r'^Cluster "(?P<cluster>[^"]+)":$')

+    for line in status_cluster_text.splitlines():

+        match = regexp.search(line)

+        if match:

+            connected_clusters.append(match.group("cluster"))

+    return connected_clusters

 def qdevice_enable(runner):

     """

@@ -143,17 +166,19 @@ def qdevice_sign_certificate_request(runner, cert_request, cluster_name):