From 6db8e3adef0441953ec18dd0339c0a67c5c26bdf Mon Sep 17 00:00:00 2001

From: Ken Gaillot <kgaillot@redhat.com>

Date: Tue, 14 Dec 2021 16:25:21 -0600

Subject: [PATCH 01/17] Doc: Pacemaker Development: update for recent function

 renames

---

 doc/sphinx/Pacemaker_Development/components.rst | 16 ++++++++--------

 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/doc/sphinx/Pacemaker_Development/components.rst b/doc/sphinx/Pacemaker_Development/components.rst

index a51220cac9..68158484ce 100644

--- a/doc/sphinx/Pacemaker_Development/components.rst

+++ b/doc/sphinx/Pacemaker_Development/components.rst

@@ -106,7 +106,7 @@ or messaging layer callback, which calls:

       the number of active peers), and if this is the last expected reply,

       calls

-      * ``call_remote_stonith()``, which calculates the timeout and sends

+      * ``request_peer_fencing()``, which calculates the timeout and sends

         ``STONITH_OP_FENCE`` request(s) to carry out the fencing. If the target

 	node has a fencing "topology" (which allows specifications such as

 	"this node can be fenced either with device A, or devices B and C in

@@ -156,7 +156,7 @@ returns, and calls

   * done callback (``st_child_done()``), which calls ``schedule_stonith_command()``

     for a new device if there are further required actions to execute or if the

     original action failed, then builds and sends an XML reply to the original

-    fencer (via ``stonith_send_async_reply()``), then checks whether any

+    fencer (via ``send_async_reply()``), then checks whether any

     pending actions are the same as the one just executed and merges them if so.

 Fencing replies

@@ -169,18 +169,18 @@ messaging layer callback, which calls:

   * ``handle_reply()``, which calls

-    * ``process_remote_stonith_exec()``, which calls either

-      ``call_remote_stonith()`` (to retry a failed operation, or try the next

-       device in a topology is appropriate, which issues a new

+    * ``fenced_process_fencing_reply()``, which calls either

+      ``request_peer_fencing()`` (to retry a failed operation, or try the next

+      device in a topology is appropriate, which issues a new

       ``STONITH_OP_FENCE`` request, proceeding as before) or

-      ``remote_op_done()`` (if the operation is definitively failed or

+      ``finalize_op()`` (if the operation is definitively failed or

       successful).

-      * remote_op_done() broadcasts the result to all peers.

+      * ``finalize_op()`` broadcasts the result to all peers.

 Finally, all peers receive the broadcast result and call

-* ``remote_op_done()``, which sends the result to all local clients.

+* ``finalize_op()``, which sends the result to all local clients.

 .. index::

-- 

2.27.0

From 47db9e5fb410b1e911710727d646eb7180a70c90 Mon Sep 17 00:00:00 2001

From: Ken Gaillot <kgaillot@redhat.com>

Date: Fri, 12 Nov 2021 09:58:16 -0600

Subject: [PATCH 02/17] Refactor: fencing: add full result to fence action

 callback data

stonith_callback_data_t previously only contained the legacy return code for

the action. Use its new opaque member to store the full result, along with

accessors (available only internally for now).

---

 include/crm/fencing/internal.h |  3 ++

 lib/fencing/st_client.c        | 99 ++++++++++++++++++++++++++--------

 2 files changed, 81 insertions(+), 21 deletions(-)

diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h

index f0d294a0b3..eff689e59b 100644

--- a/include/crm/fencing/internal.h

+++ b/include/crm/fencing/internal.h

@@ -187,6 +187,9 @@ bool stonith__event_state_eq(stonith_history_t *history, void *user_data);

 bool stonith__event_state_neq(stonith_history_t *history, void *user_data);

 int stonith__legacy2status(int rc);

+int stonith__exit_status(stonith_callback_data_t *data);

+int stonith__execution_status(stonith_callback_data_t *data);

+const char *stonith__exit_reason(stonith_callback_data_t *data);

 /*!

  * \internal

diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c

index 2ca094566b..9d93ffd481 100644

--- a/lib/fencing/st_client.c

+++ b/lib/fencing/st_client.c

@@ -854,20 +854,23 @@ stonith_api_del_callback(stonith_t * stonith, int call_id, bool all_callbacks)

  * \param[in] st        Fencer API connection

  * \param[in] call_id   If positive, call ID of completed fence action, otherwise

  *                      legacy return code for early action failure

- * \param[in] rc        Legacy return code for action result

+ * \param[in] result    Full result for action

  * \param[in] userdata  User data to pass to callback

  * \param[in] callback  Fence action callback to invoke

  */

 static void

-invoke_fence_action_callback(stonith_t *st, int call_id, int rc, void *userdata,

+invoke_fence_action_callback(stonith_t *st, int call_id,

+                             pcmk__action_result_t *result,

+                             void *userdata,

                              void (*callback) (stonith_t *st,

                                                stonith_callback_data_t *data))

 {

     stonith_callback_data_t data = { 0, };

     data.call_id = call_id;

-    data.rc = rc;

+    data.rc = pcmk_rc2legacy(stonith__result2rc(result));

     data.userdata = userdata;

+    data.opaque = (void *) result;

     callback(st, &data);

 }

@@ -888,7 +891,7 @@ invoke_registered_callbacks(stonith_t *stonith, xmlNode *msg, int call_id)

 {

     stonith_private_t *private = NULL;

     stonith_callback_client_t *cb_info = NULL;

-    int rc = pcmk_ok;

+    pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

     CRM_CHECK(stonith != NULL, return);

     CRM_CHECK(stonith->st_private != NULL, return);

@@ -897,20 +900,17 @@ invoke_registered_callbacks(stonith_t *stonith, xmlNode *msg, int call_id)

     if (msg == NULL) {

         // Fencer didn't reply in time

-        rc = -ETIME;

+        pcmk__set_result(&result, CRM_EX_ERROR, PCMK_EXEC_TIMEOUT,

+                         "Timeout waiting for reply from fencer");

         CRM_LOG_ASSERT(call_id > 0);

     } else {

         // We have the fencer reply

-

-        if (crm_element_value_int(msg, F_STONITH_RC, &rc) != 0) {

-            rc = -pcmk_err_generic;

-        }

-

         if ((crm_element_value_int(msg, F_STONITH_CALLID, &call_id) != 0)

             || (call_id <= 0)) {

             crm_log_xml_warn(msg, "Bad fencer reply");

         }

+        stonith__xe_get_result(msg, &result);

     }

     if (call_id > 0) {

@@ -919,27 +919,29 @@ invoke_registered_callbacks(stonith_t *stonith, xmlNode *msg, int call_id)

     }

     if ((cb_info != NULL) && (cb_info->callback != NULL)

-        && (rc == pcmk_ok || !(cb_info->only_success))) {

+        && (pcmk__result_ok(&result) || !(cb_info->only_success))) {

         crm_trace("Invoking callback %s for call %d",

                   crm_str(cb_info->id), call_id);

-        invoke_fence_action_callback(stonith, call_id, rc, cb_info->user_data,

-                                     cb_info->callback);

+        invoke_fence_action_callback(stonith, call_id, &result,

+                                     cb_info->user_data, cb_info->callback);

-    } else if ((private->op_callback == NULL) && (rc != pcmk_ok)) {

-        crm_warn("Fencing action without registered callback failed: %s",

-                 pcmk_strerror(rc));

+    } else if ((private->op_callback == NULL) && !pcmk__result_ok(&result)) {

+        crm_warn("Fencing action without registered callback failed: %d (%s)",

+                 result.exit_status,

+                 pcmk_exec_status_str(result.execution_status));

         crm_log_xml_debug(msg, "Failed fence update");

     }

     if (private->op_callback != NULL) {

         crm_trace("Invoking global callback for call %d", call_id);

-        invoke_fence_action_callback(stonith, call_id, rc, NULL,

+        invoke_fence_action_callback(stonith, call_id, &result, NULL,

                                      private->op_callback);

     }

     if (cb_info != NULL) {

         stonith_api_del_callback(stonith, call_id, FALSE);

     }

+    pcmk__reset_result(&result);

 }

 static gboolean

@@ -1252,14 +1254,18 @@ stonith_api_add_callback(stonith_t * stonith, int call_id, int timeout, int opti

     CRM_CHECK(stonith->st_private != NULL, return -EINVAL);

     private = stonith->st_private;

-    if (call_id == 0) {

+    if (call_id == 0) { // Add global callback

         private->op_callback = callback;

-    } else if (call_id < 0) {

+    } else if (call_id < 0) { // Call failed immediately, so call callback now

         if (!(options & st_opt_report_only_success)) {

+            pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

+

             crm_trace("Call failed, calling %s: %s", callback_name, pcmk_strerror(call_id));

-            invoke_fence_action_callback(stonith, call_id, call_id, user_data,

-                                         callback);

+            pcmk__set_result(&result, CRM_EX_ERROR,

+                             stonith__legacy2status(call_id), NULL);

+            invoke_fence_action_callback(stonith, call_id, &result,

+                                         user_data, callback);

         } else {

             crm_warn("Fencer call failed: %s", pcmk_strerror(call_id));

         }

@@ -2293,6 +2299,57 @@ stonith__device_parameter_flags(uint32_t *device_flags, const char *device_name,

     freeXpathObject(xpath);

 }

+/*!

+ * \internal

+ * \brief Return the exit status from an async action callback

+ *

+ * \param[in] data  Callback data

+ *

+ * \return Exit status from callback data

+ */

+int

+stonith__exit_status(stonith_callback_data_t *data)

+{

+    if ((data == NULL) || (data->opaque == NULL)) {

+        return CRM_EX_ERROR;

+    }

+    return ((pcmk__action_result_t *) data->opaque)->exit_status;

+}

+

+/*!

+ * \internal

+ * \brief Return the execution status from an async action callback

+ *

+ * \param[in] data  Callback data

+ *

+ * \return Execution status from callback data

+ */

+int

+stonith__execution_status(stonith_callback_data_t *data)

+{

+    if ((data == NULL) || (data->opaque == NULL)) {

+        return PCMK_EXEC_UNKNOWN;

+    }

+    return ((pcmk__action_result_t *) data->opaque)->execution_status;

+}

+

+/*!

+ * \internal

+ * \brief Return the exit reason from an async action callback

+ *

+ * \param[in] data  Callback data

+ *

+ * \return Exit reason from callback data

+ */

+const char *

+stonith__exit_reason(stonith_callback_data_t *data)

+{

+    if ((data == NULL) || (data->opaque == NULL)) {

+        return NULL;

+    }

+    return ((pcmk__action_result_t *) data->opaque)->exit_reason;

+}

+

 // Deprecated functions kept only for backward API compatibility

 // LCOV_EXCL_START

-- 

2.27.0

From 1e076370ef4ac7993b5ff21ed1cdfb3c4a494cf0 Mon Sep 17 00:00:00 2001

From: Ken Gaillot <kgaillot@redhat.com>

Date: Tue, 9 Nov 2021 16:16:03 -0600

Subject: [PATCH 03/17] Log: controller: improve fencing result messages

Now that fence callbacks get the full result, we can log a better message.

Also check for error conditions better, improve message wording, and ensure

only a single message is logged per result.

---

 daemons/controld/controld_fencing.c | 83 +++++++++++++++++++----------

 1 file changed, 56 insertions(+), 27 deletions(-)

diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c

index f5a252c813..f8d2fc13f4 100644

--- a/daemons/controld/controld_fencing.c

+++ b/daemons/controld/controld_fencing.c

@@ -714,45 +714,64 @@ tengine_stonith_callback(stonith_t *stonith, stonith_callback_data_t *data)

     int stonith_id = -1;

     int transition_id = -1;

     crm_action_t *action = NULL;

-    int call_id = data->call_id;

-    int rc = data->rc;

-    char *userdata = data->userdata;

-

-    CRM_CHECK(userdata != NULL, return);

-    crm_notice("Stonith operation %d/%s: %s (%d)", call_id, (char *)userdata,

-               pcmk_strerror(rc), rc);

+    const char *target = NULL;

-    if (AM_I_DC == FALSE) {

+    if ((data == NULL) || (data->userdata == NULL)) {

+        crm_err("Ignoring fence operation %d result: "

+                "No transition key given (bug?)",

+                ((data == NULL)? -1 : data->call_id));

         return;

     }

-    /* crm_info("call=%d, optype=%d, node_name=%s, result=%d, node_list=%s, action=%s", */

-    /*       op->call_id, op->optype, op->node_name, op->op_result, */

-    /*       (char *)op->node_list, op->private_data); */

+    if (!AM_I_DC) {

+        const char *reason = stonith__exit_reason(data);

+

+        if (reason == NULL) {

+           reason = pcmk_exec_status_str(stonith__execution_status(data));

+        }

+        crm_notice("Result of fence operation %d: %d (%s) " CRM_XS " key=%s",

+                   data->call_id, stonith__exit_status(data), reason,

+                   (const char *) data->userdata);

+        return;

+    }

-    /* filter out old STONITH actions */

-    CRM_CHECK(decode_transition_key(userdata, &uuid, &transition_id, &stonith_id, NULL),

+    CRM_CHECK(decode_transition_key(data->userdata, &uuid, &transition_id,

+                                    &stonith_id, NULL),

               goto bail);

-    if (transition_graph->complete || stonith_id < 0 || !pcmk__str_eq(uuid, te_uuid, pcmk__str_casei)

-        || transition_graph->id != transition_id) {

-        crm_info("Ignoring STONITH action initiated outside of the current transition");

+    if (transition_graph->complete || (stonith_id < 0)

+        || !pcmk__str_eq(uuid, te_uuid, pcmk__str_none)

+        || (transition_graph->id != transition_id)) {

+        crm_info("Ignoring fence operation %d result: "

+                 "Not from current transition " CRM_XS

+                 " complete=%s action=%d uuid=%s (vs %s) transition=%d (vs %d)",

+                 data->call_id, pcmk__btoa(transition_graph->complete),

+                 stonith_id, uuid, te_uuid, transition_id, transition_graph->id);

         goto bail;

     }

     action = controld_get_action(stonith_id);

     if (action == NULL) {

-        crm_err("Stonith action not matched");

+        crm_err("Ignoring fence operation %d result: "

+                "Action %d not found in transition graph (bug?) "

+                CRM_XS " uuid=%s transition=%d",

+                data->call_id, stonith_id, uuid, transition_id);

+        goto bail;

+    }

+

+    target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);

+    if (target == NULL) {

+        crm_err("Ignoring fence operation %d result: No target given (bug?)",

+                data->call_id);

         goto bail;

     }

     stop_te_timer(action->timer);

-    if (rc == pcmk_ok) {

-        const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);

+    if (stonith__exit_status(data) == CRM_EX_OK) {

         const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);

         const char *op = crm_meta_value(action->params, "stonith_action");

-        crm_info("Stonith operation %d for %s passed", call_id, target);

+        crm_notice("Fence operation %d for %s passed", data->call_id, target);

         if (!(pcmk_is_set(action->flags, pcmk__graph_action_confirmed))) {

             te_action_confirmed(action, NULL);

             if (pcmk__str_eq("on", op, pcmk__str_casei)) {

@@ -791,20 +810,30 @@ tengine_stonith_callback(stonith_t *stonith, stonith_callback_data_t *data)

         st_fail_count_reset(target);

     } else {

-        const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);

         enum transition_action abort_action = tg_restart;

+        int status = stonith__execution_status(data);

+        const char *reason = stonith__exit_reason(data);

+        if (reason == NULL) {

+            if (status == PCMK_EXEC_DONE) {

+                reason = "Agent returned error";

+            } else {

+                reason = pcmk_exec_status_str(status);

+            }

+        }

         crm__set_graph_action_flags(action, pcmk__graph_action_failed);

-        crm_notice("Stonith operation %d for %s failed (%s): aborting transition.",

-                   call_id, target, pcmk_strerror(rc));

         /* If no fence devices were available, there's no use in immediately

          * checking again, so don't start a new transition in that case.

          */

-        if (rc == -ENODEV) {

-            crm_warn("No devices found in cluster to fence %s, giving up",

-                     target);

+        if (status == PCMK_EXEC_NO_FENCE_DEVICE) {

+            crm_warn("Fence operation %d for %s failed: %s "

+                     "(aborting transition and giving up for now)",

+                     data->call_id, target, reason);

             abort_action = tg_stop;

+        } else {

+            crm_notice("Fence operation %d for %s failed: %s "

+                       "(aborting transition)", data->call_id, target, reason);

         }

         /* Increment the fail count now, so abort_for_stonith_failure() can

@@ -818,7 +847,7 @@ tengine_stonith_callback(stonith_t *stonith, stonith_callback_data_t *data)

     trigger_graph();

   bail:

-    free(userdata);

+    free(data->userdata);

     free(uuid);

     return;

 }

-- 

2.27.0

From 25547e3b7e6eb23efad1c359388d6e8d0df62363 Mon Sep 17 00:00:00 2001

From: Ken Gaillot <kgaillot@redhat.com>

Date: Mon, 22 Nov 2021 12:37:16 -0600

Subject: [PATCH 04/17] Refactor: executor: drop action_get_uniform_rc()

 function

action_get_uniform_rc() called stonith2uniform_rc() or services_result2ocf() as

appropriate to the action standard. However, it was called only from a place

that did not process stonith actions, so that place can just call

services_result2ocf() directly.

This will simplify planned changes.

---

 daemons/execd/execd_commands.c | 24 ++++++------------------

 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/daemons/execd/execd_commands.c b/daemons/execd/execd_commands.c

index 5bb2aab692..5e123e322e 100644

--- a/daemons/execd/execd_commands.c

+++ b/daemons/execd/execd_commands.c

@@ -780,23 +780,6 @@ stonith2uniform_rc(const char *action, int rc)

     return rc;

 }

-static int

-action_get_uniform_rc(svc_action_t *action)

-{

-    lrmd_cmd_t *cmd = action->cb_data;

-

-    if (pcmk__str_eq(action->standard, PCMK_RESOURCE_CLASS_STONITH,

-                            pcmk__str_casei)) {

-        return stonith2uniform_rc(cmd->action, action->rc);

-    } else {

-        enum ocf_exitcode code = services_result2ocf(action->standard,

-                                                     cmd->action, action->rc);

-

-        // Cast variable instead of function return to keep compilers happy

-        return (int) code;

-    }

-}

-

 struct notify_new_client_data {

     xmlNode *notify;

     pcmk__client_t *new_client;

@@ -848,6 +831,7 @@ action_complete(svc_action_t * action)

 {

     lrmd_rsc_t *rsc;

     lrmd_cmd_t *cmd = action->cb_data;

+    enum ocf_exitcode code;

 #ifdef PCMK__TIME_USE_CGT

     const char *rclass = NULL;

@@ -867,8 +851,12 @@ action_complete(svc_action_t * action)

 #endif

     cmd->last_pid = action->pid;

-    pcmk__set_result(&(cmd->result), action_get_uniform_rc(action),

+

+    // Cast variable instead of function return to keep compilers happy

+    code = services_result2ocf(action->standard, cmd->action, action->rc);

+    pcmk__set_result(&(cmd->result), (int) code,

                      action->status, services__exit_reason(action));

+

     rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;

 #ifdef PCMK__TIME_USE_CGT

-- 

2.27.0

From b5e31ba2539da4e94c124c3f0c8c72f7039f9a7a Mon Sep 17 00:00:00 2001

From: Ken Gaillot <kgaillot@redhat.com>

Date: Mon, 22 Nov 2021 12:39:30 -0600

Subject: [PATCH 05/17] Feature: executor: use full result from fencer for

 fence actions

Now that fence callbacks get the full result, we can improve the executor

command result for fence actions. stonith_action_complete() now takes a

full result, allowing the executor to use that directly rather than map a

legacy return code.

---

 daemons/execd/execd_commands.c | 140 +++++++++++++++++++--------------

 1 file changed, 80 insertions(+), 60 deletions(-)

diff --git a/daemons/execd/execd_commands.c b/daemons/execd/execd_commands.c

index 5e123e322e..e722994012 100644

--- a/daemons/execd/execd_commands.c

+++ b/daemons/execd/execd_commands.c

@@ -8,6 +8,7 @@

  */

 #include <crm_internal.h>

+#include <crm/fencing/internal.h>

 #include <glib.h>

@@ -748,38 +749,6 @@ cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc)

     }

 }

-static int

-stonith2uniform_rc(const char *action, int rc)

-{

-    switch (rc) {

-        case pcmk_ok:

-            rc = PCMK_OCF_OK;

-            break;

-

-        case -ENODEV:

-            /* This should be possible only for probes in practice, but

-             * interpret for all actions to be safe.

-             */

-            if (pcmk__str_eq(action, "monitor", pcmk__str_casei)) {

-                rc = PCMK_OCF_NOT_RUNNING;

-            } else if (pcmk__str_eq(action, "stop", pcmk__str_casei)) {

-                rc = PCMK_OCF_OK;

-            } else {

-                rc = PCMK_OCF_NOT_INSTALLED;

-            }

-            break;

-

-        case -EOPNOTSUPP:

-            rc = PCMK_OCF_UNIMPLEMENT_FEATURE;

-            break;

-

-        default:

-            rc = PCMK_OCF_UNKNOWN_ERROR;

-            break;

-    }

-    return rc;

-}

-

 struct notify_new_client_data {

     xmlNode *notify;

     pcmk__client_t *new_client;

@@ -988,46 +957,84 @@ action_complete(svc_action_t * action)

     cmd_finalize(cmd, rsc);

 }

+/*!

+ * \internal

+ * \brief Process the result of a fence device action (start, stop, or monitor)

+ *

+ * \param[in] cmd               Fence device action that completed

+ * \param[in] exit_status       Fencer API exit status for action

+ * \param[in] execution_status  Fencer API execution status for action

+ * \param[in] exit_reason       Human-friendly detail, if action failed

+ */

 static void

-stonith_action_complete(lrmd_cmd_t * cmd, int rc)

+stonith_action_complete(lrmd_cmd_t *cmd, int exit_status,

+                        enum pcmk_exec_status execution_status,

+                        const char *exit_reason)

 {

     // This can be NULL if resource was removed before command completed

     lrmd_rsc_t *rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);

-    cmd->result.exit_status = stonith2uniform_rc(cmd->action, rc);

+    // Simplify fencer exit status to uniform exit status

+    if (exit_status != CRM_EX_OK) {

+        exit_status = PCMK_OCF_UNKNOWN_ERROR;

+    }

-    /* This function may be called with status already set to cancelled, if a

-     * pending action was aborted. Otherwise, we need to determine status from

-     * the fencer return code.

-     */

-    if (cmd->result.execution_status != PCMK_EXEC_CANCELLED) {

-        cmd->result.execution_status = stonith__legacy2status(rc);

+    if (cmd->result.execution_status == PCMK_EXEC_CANCELLED) {

+        /* An in-flight fence action was cancelled. The execution status is

+         * already correct, so don't overwrite it.

+         */

+        execution_status = PCMK_EXEC_CANCELLED;

-        // Simplify status codes from fencer

-        switch (cmd->result.execution_status) {

+    } else {

+        /* Some execution status codes have specific meanings for the fencer

+         * that executor clients may not expect, so map them to a simple error

+         * status.

+         */

+        switch (execution_status) {

             case PCMK_EXEC_NOT_CONNECTED:

             case PCMK_EXEC_INVALID:

-            case PCMK_EXEC_NO_FENCE_DEVICE:

             case PCMK_EXEC_NO_SECRETS:

-                cmd->result.execution_status = PCMK_EXEC_ERROR;

+                execution_status = PCMK_EXEC_ERROR;

                 break;

-            default:

+

+            case PCMK_EXEC_NO_FENCE_DEVICE:

+                /* This should be possible only for probes in practice, but

+                 * interpret for all actions to be safe.

+                 */

+                if (pcmk__str_eq(cmd->action, CRMD_ACTION_STATUS,

+                                 pcmk__str_none)) {

+                    exit_status = PCMK_OCF_NOT_RUNNING;

+

+                } else if (pcmk__str_eq(cmd->action, CRMD_ACTION_STOP,

+                                        pcmk__str_none)) {

+                    exit_status = PCMK_OCF_OK;

+

+                } else {

+                    exit_status = PCMK_OCF_NOT_INSTALLED;

+                }

+                execution_status = PCMK_EXEC_ERROR;

                 break;

-        }

-        // Certain successful actions change the known state of the resource

-        if ((rsc != NULL) && pcmk__result_ok(&(cmd->result))) {

-            if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)) {

-                rsc->st_probe_rc = pcmk_ok; // maps to PCMK_OCF_OK

-            } else if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {

-                rsc->st_probe_rc = -ENODEV; // maps to PCMK_OCF_NOT_RUNNING

-            }

+            case PCMK_EXEC_NOT_SUPPORTED:

+                exit_status = PCMK_OCF_UNIMPLEMENT_FEATURE;

+                break;

+

+            default:

+                break;

         }

     }

-    // Give the user more detail than an OCF code

-    if (rc != -pcmk_err_generic) {

-        cmd->result.exit_reason = strdup(pcmk_strerror(rc));

+    pcmk__set_result(&cmd->result, exit_status, execution_status, exit_reason);

+

+    // Certain successful actions change the known state of the resource

+    if ((rsc != NULL) && pcmk__result_ok(&(cmd->result))) {

+

+        if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)) {

+            rsc->st_probe_rc = pcmk_ok; // maps to PCMK_OCF_OK

+

+        } else if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {

+            rsc->st_probe_rc = -ENODEV; // maps to PCMK_OCF_NOT_RUNNING

+        }

     }

     /* The recurring timer should not be running at this point in any case, but

@@ -1050,7 +1057,15 @@ stonith_action_complete(lrmd_cmd_t * cmd, int rc)

 static void

 lrmd_stonith_callback(stonith_t * stonith, stonith_callback_data_t * data)

 {

-    stonith_action_complete(data->userdata, data->rc);

+    if ((data == NULL) || (data->userdata == NULL)) {

+        crm_err("Ignoring fence action result: "

+                "Invalid callback arguments (bug?)");

+    } else {

+        stonith_action_complete((lrmd_cmd_t *) data->userdata,

+                                stonith__exit_status(data),

+                                stonith__execution_status(data),

+                                stonith__exit_reason(data));

+    }

 }

 void

@@ -1097,7 +1112,9 @@ stonith_connection_failed(void)

     crm_err("Connection to fencer failed, finalizing %d pending operations",

             g_list_length(cmd_list));

     for (cmd_iter = cmd_list; cmd_iter; cmd_iter = cmd_iter->next) {

-        stonith_action_complete(cmd_iter->data, -ENOTCONN);

+        stonith_action_complete((lrmd_cmd_t *) cmd_iter->data,

+                                CRM_EX_ERROR, PCMK_EXEC_NOT_CONNECTED,

+                                "Lost connection to fencer");

     }

     g_list_free(cmd_list);

 }

@@ -1210,7 +1227,7 @@ lrmd_rsc_execute_stonith(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)

     } else if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)) {

         rc = execd_stonith_start(stonith_api, rsc, cmd);

-        if (rc == 0) {

+        if (rc == pcmk_ok) {

             do_monitor = TRUE;

         }

@@ -1233,7 +1250,10 @@ lrmd_rsc_execute_stonith(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)

         }

     }

-    stonith_action_complete(cmd, rc);

+    stonith_action_complete(cmd,

+                            ((rc == pcmk_ok)? CRM_EX_OK : CRM_EX_ERROR),

+                            stonith__legacy2status(rc),

+                            rc == -pcmk_err_generic? NULL : pcmk_strerror(rc));

 }

 static int