From 200559d8ca0b834f90d4f2ba70e8f7ce403b9726 Mon Sep 17 00:00:00 2001

From: Tomas Jelinek <tojeline@redhat.com>

Date: Fri, 16 Jan 2015 16:31:49 +0100

Subject: [PATCH] Warn if nodes stop will cause a loss of the quorum

---

 pcs/cluster.py         |  68 +++++++++++

 pcs/utils.py           |  77 ++++++++++--

 pcsd/public/js/pcsd.js |  41 ++++++-

 pcsd/remote.rb         |  35 ++++--

 5 files changed, 513 insertions(+), 17 deletions(-)

diff --git a/pcs/cluster.py b/pcs/cluster.py

index 05dc91a..e4829f1 100644

--- a/pcs/cluster.py

+++ b/pcs/cluster.py

@@ -676,6 +676,55 @@ def stop_cluster_all():

     stop_cluster_nodes(utils.getNodesFromCorosyncConf())

 def stop_cluster_nodes(nodes):

+    all_nodes = utils.getNodesFromCorosyncConf()

+    unknown_nodes = set(nodes) - set(all_nodes)

+    if unknown_nodes:

+        utils.err(

+            "nodes '%s' do not appear to exist in configuration"

+            % "', '".join(unknown_nodes)

+        )

+

+    stopping_all = set(nodes) >= set(all_nodes)

+    if (

+        not "--force" in utils.pcs_options

+        and

+        not stopping_all

+        and

+        not utils.is_rhel6()

+    ):

+        # we are sure we are not on cman cluster because only nodes from

+        # a local cluster can be stopped (see nodes validation above)

+        error_list = []

+        for node in nodes:

+            retval, data = utils.get_remote_quorumtool_output(node)

+            if retval != 0:

+                error_list.append(node + ": " + data)

+                continue

+            quorum_info = utils.parse_quorumtool_output(data)

+            if quorum_info:

+                if not quorum_info["quorate"]:

+                    continue

+                if utils.is_node_stop_cause_quorum_loss(

+                    quorum_info, local=False, node_list=nodes

+                ):

+                    utils.err(

+                        "Stopping the node(s) will cause a loss of the quorum"

+                        + ", use --force to override"

+                    )

+                else:

+                    # We have the info, no need to print errors

+                    error_list = []

+                    break

+            if data.strip() != "Cannot initialize CMAP service":

+                error_list.append("Unable to get quorum status")

+            # else the node seems to be stopped already

+        if error_list:

+            utils.err(

+                "Unable to determine whether stopping the nodes will cause "

+                + "a loss of the quorum, use --force to override\n"

+                + "\n".join(error_list)

+            )

+

     threads = dict()

     for node in nodes:

         threads[node] = NodeStopPacemakerThread(node)

@@ -764,6 +813,25 @@ def stop_cluster(argv):

         stop_cluster_nodes(argv)

         return

+    if not "--force" in utils.pcs_options and not utils.is_rhel6():

+        output, retval = utils.run(["corosync-quorumtool", "-p", "-s"])

+        # retval is 0 on success if node is not in partition with quorum

+        # retval is 1 on error OR on success if node has quorum

+        quorum_info = utils.parse_quorumtool_output(output)

+        if quorum_info:

+            if utils.is_node_stop_cause_quorum_loss(quorum_info, local=True):

+                utils.err(

+                    "Stopping the node will cause a loss of the quorum"

+                    + ", use --force to override"

+                )

+        else:

+            if output.strip() != "Cannot initialize CMAP service":

+                utils.err(

+                    "Unable to determine whether stopping the node will cause "

+                    + "a loss of the quorum, use --force to override"

+                )

+            # else the node seems to be stopped already, proceed to be sure

+

     stop_all = (

         "--pacemaker" not in utils.pcs_options

         and

diff --git a/pcs/utils.py b/pcs/utils.py

index 82b71c6..2f718e7 100644

--- a/pcs/utils.py

+++ b/pcs/utils.py

@@ -211,15 +211,15 @@ def setCorosyncConfig(node,config):

 def startCluster(node, quiet=False):

     return sendHTTPRequest(node, 'remote/cluster_start', None, False, not quiet)

-def stopCluster(node, quiet=False, pacemaker=True, corosync=True):

-    if (pacemaker and corosync) or (not pacemaker and not corosync):

-        data = None

-    elif pacemaker:

-        data = {"component": "pacemaker"}

-    elif corosync:

-        data = {"component": "corosync"}

-    if data:

-        data = urllib.urlencode(data)

+def stopCluster(node, quiet=False, pacemaker=True, corosync=True, force=True):

+    data = dict()

+    if pacemaker and not corosync:

+        data["component"] = "pacemaker"

+    elif corosync and not pacemaker:

+        data["component"] = "corosync"

+    if force:

+        data["force"] = 1

+    data = urllib.urlencode(data)

     return sendHTTPRequest(node, 'remote/cluster_stop', data, False, not quiet)

 def enableCluster(node):

@@ -2286,3 +2286,62 @@ def get_operations_from_transitions(transitions_dom):

     op_list = [op[1] for op in operation_list]

     return op_list

+def get_remote_quorumtool_output(node):

+    return sendHTTPRequest(node, "remote/get_quorum_info", None, False, False)

+

+def parse_quorumtool_output(quorumtool_output):

+    parsed = {}

+    in_node_list = False

+    try:

+        for line in quorumtool_output.split("\n"):

+            line = line.strip()

+            if not line:

+                continue

+            if in_node_list:

+                if line.startswith("-") or line.startswith("Nodeid"):

+                    # skip headers

+                    continue

+                parts = line.split()

+                parsed["node_list"].append({

+                    "name": parts[3],

+                    "votes": int(parts[1]),

+                    "local": len(parts) > 4 and parts[4] == "(local)"

+                })

+            else:

+                if line == "Membership information":

+                    in_node_list = True

+                    parsed["node_list"] = []

+                    continue

+                if not ":" in line:

+                    continue

+                parts = map(lambda x: x.strip(), line.split(":", 1))

+                if parts[0] == "Quorate":

+                    parsed["quorate"] = parts[1].lower() == "yes"

+                elif parts[0] == "Quorum":

+                    match = re.match("(\d+).*", parts[1])

+                    if match:

+                        parsed["quorum"] = int(match.group(1))

+                    else:

+                        return None

+    except (ValueError, IndexError):

+        return None

+    for required in ("quorum", "quorate", "node_list"):

+        if required not in parsed:

+            return None

+    return parsed

+

+# node_list - nodes to stop

+# local - local node is going to be stopped

+def is_node_stop_cause_quorum_loss(quorum_info, local=True, node_list=None):

+    if not quorum_info["quorate"]:

+        return False

+    # sum the votes of nodes that are not going to be stopped

+    votes_after_stop = 0

+    for node_info in quorum_info.get("node_list", []):

+        if local and node_info["local"]:

+            continue

+        if node_list and node_info["name"] in node_list:

+            continue

+        votes_after_stop += node_info["votes"]

+    return votes_after_stop < quorum_info["quorum"]

+

diff --git a/pcsd/public/js/pcsd.js b/pcsd/public/js/pcsd.js

index cbfc621..30473c9 100644

--- a/pcsd/public/js/pcsd.js

+++ b/pcsd/public/js/pcsd.js

@@ -481,7 +481,7 @@ function setup_node_links() {

   $("#node_stop").click(function() {

     node = $("#node_info_header_title_name").text();

     fade_in_out("#node_stop");

-    $.post('/remote/cluster_stop', {"name": $.trim(node)});

+    node_stop($.trim(node), false);

   });

   $("#node_restart").click(function() {

     node = $("#node_info_header_title_name").text();

@@ -500,6 +500,45 @@ function setup_node_links() {

   });

 }

+function node_stop(node, force) {

+  var data = {};

+  data["name"] = node;

+  if (force) {

+    data["force"] = force;

+  }

+  $.ajax({

+    type: 'POST',

+    url: '/remote/cluster_stop',

+    data: data,

+    timeout: pcs_timeout,

+    success: function() {

+    },

+    error: function(xhr, status, error) {

+      if ((status == "timeout") || ($.trim(error) == "timeout")) {

+        /*

+         We are not interested in timeout because:

+         - it can take minutes to stop a node (resources running on it have

+           to be stopped/moved and we do not need to wait for that)

+         - if pcs is not able to stop a node it returns an (forceable) error

+           immediatelly

+        */

+        return;

+      }

+      var message = "Unable to stop node '" + node + "' (" + $.trim(error) + ")";

+      message += "\n" + xhr.responseText;

+      if (message.indexOf('--force') == -1) {

+        alert(message);

+      }

+      else {

+        message = message.replace(', use --force to override', '');

+        if (confirm(message + "\n\nDo you want to force the operation?")) {

+          node_stop(node, true);

+        }

+      }

+    }

+  });

+}

+

 function setup_resource_links(link_type) {

   Ember.debug("Setup resource links");

   $("#resource_delete_link").click(function () {

diff --git a/pcsd/remote.rb b/pcsd/remote.rb

index e818288..cbbcc12 100644

--- a/pcsd/remote.rb

+++ b/pcsd/remote.rb

@@ -21,6 +21,8 @@ def remote(params,request)

     return setup_cluster(params)

   when "create_cluster"

     return create_cluster(params)

+  when "get_quorum_info"

+    return get_quorum_info(params)

   when "get_cib"

     return get_cib(params)

   when "get_corosync_conf"

@@ -156,18 +158,22 @@ def cluster_stop(params)

       params[:name], 'cluster_stop', true, params_without_name

     )

   else

-    options = ""

+    options = []

     if params.has_key?("component")

       if params["component"].downcase == "pacemaker"

-        options = "--pacemaker"

+        options << "--pacemaker"

       elsif params["component"].downcase == "corosync"

-        options = "--corosync"

+        options << "--corosync"

       end

     end

-    $logger.info "Stopping Daemons #{options}"

-    output =  `#{PCS} cluster stop #{options}`

-    $logger.debug output

-    return output

+    options << "--force" if params["force"]

+    $logger.info "Stopping Daemons"

+    stdout, stderr, retval = run_cmd(PCS, "cluster", "stop", *options)

+    if retval != 0

+      return [400, stderr.join]

+    else

+      return stdout.join

+    end

   end

 end

@@ -274,6 +280,21 @@ def cluster_disable(params)

   end

 end

+def get_quorum_info(params)

+  if ISRHEL6

+    return ''

+  else

+    stdout, stderr, retval = run_cmd("corosync-quorumtool", "-p", "-s")

+    # retval is 0 on success if node is not in partition with quorum

+    # retval is 1 on error OR on success if node has quorum

+    if stderr.length > 0

+      return stderr.join

+    else

+      return stdout.join

+    end

+  end

+end

+

 def get_cib(params)

   cib, stderr, retval = run_cmd(CIBADMIN, "-Ql")

   if retval != 0

-- 

1.9.1