From: Andrew Beekhof <andrew@beekhof.net>
Date: Tue, 1 Sep 2015 13:17:45 +1000
Subject: [PATCH] Feature: crmd: Implement reliable event notifications
(cherry picked from commit 0cd1b8f02b403976afe106e0ca3a8a8a16864c6c)
---
crmd/Makefile.am | 2 +-
crmd/callbacks.c | 4 +
crmd/control.c | 67 +++++++++++++---
crmd/crmd_utils.h | 1 +
crmd/lrm.c | 2 +
crmd/notify.c | 188 ++++++++++++++++++++++++++++++++++++++++++++
crmd/notify.h | 30 +++++++
crmd/te_utils.c | 2 +
cts/CIB.py | 2 +
extra/pcmk_notify_sample.sh | 68 ++++++++++++++++
include/crm_internal.h | 1 +
lib/common/utils.c | 27 +++++++
12 files changed, 380 insertions(+), 14 deletions(-)
create mode 100644 crmd/notify.c
create mode 100644 crmd/notify.h
create mode 100755 extra/pcmk_notify_sample.sh
diff --git a/crmd/Makefile.am b/crmd/Makefile.am
index 8e5e1df..984f5d0 100644
--- a/crmd/Makefile.am
+++ b/crmd/Makefile.am
@@ -28,7 +28,7 @@ noinst_HEADERS = crmd.h crmd_fsa.h crmd_messages.h fsa_defines.h \
fsa_matrix.h fsa_proto.h crmd_utils.h crmd_callbacks.h \
crmd_lrm.h te_callbacks.h tengine.h
-crmd_SOURCES = main.c crmd.c corosync.c \
+crmd_SOURCES = main.c crmd.c corosync.c notify.c \
fsa.c control.c messages.c membership.c callbacks.c \
election.c join_client.c join_dc.c subsystems.c throttle.c \
cib.c pengine.c tengine.c lrm.c lrm_state.c remote_lrmd_ra.c \
diff --git a/crmd/callbacks.c b/crmd/callbacks.c
index f646927..38fb30b 100644
--- a/crmd/callbacks.c
+++ b/crmd/callbacks.c
@@ -126,6 +126,7 @@ peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *d
case crm_status_nstate:
crm_info("%s is now %s (was %s)",
node->uname, state_text(node->state), state_text(data));
+
if (safe_str_eq(data, node->state)) {
/* State did not change */
return;
@@ -147,7 +148,10 @@ peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *d
}
}
}
+
+ crmd_notify_node_event(node);
break;
+
case crm_status_processes:
if (data) {
old = *(const uint32_t *)data;
diff --git a/crmd/control.c b/crmd/control.c
index f4add49..d92f46b 100644
--- a/crmd/control.c
+++ b/crmd/control.c
@@ -873,28 +873,64 @@ do_recover(long long action,
/* *INDENT-OFF* */
pe_cluster_option crmd_opts[] = {
- /* name, old-name, validate, default, description */
- { "dc-version", NULL, "string", NULL, "none", NULL, "Version of Pacemaker on the cluster's DC.", "Includes the hash which identifies the exact Mercurial changeset it was built from. Used for diagnostic purposes." },
- { "cluster-infrastructure", NULL, "string", NULL, "heartbeat", NULL, "The messaging stack on which Pacemaker is currently running.", "Used for informational and diagnostic purposes." },
- { XML_CONFIG_ATTR_DC_DEADTIME, "dc_deadtime", "time", NULL, "20s", &check_time, "How long to wait for a response from other nodes during startup.", "The \"correct\" value will depend on the speed/load of your network and the type of switches used." },
+ /* name, old-name, validate, values, default, short description, long description */
+ { "dc-version", NULL, "string", NULL, "none", NULL,
+ "Version of Pacemaker on the cluster's DC.",
+ "Includes the hash which identifies the exact changeset it was built from. Used for diagnostic purposes."
+ },
+ { "cluster-infrastructure", NULL, "string", NULL, "heartbeat", NULL,
+ "The messaging stack on which Pacemaker is currently running.",
+ "Used for informational and diagnostic purposes." },
+ { XML_CONFIG_ATTR_DC_DEADTIME, "dc_deadtime", "time", NULL, "20s", &check_time,
+ "How long to wait for a response from other nodes during startup.",
+ "The \"correct\" value will depend on the speed/load of your network and the type of switches used."
+ },
{ XML_CONFIG_ATTR_RECHECK, "cluster_recheck_interval", "time",
- "Zero disables polling. Positive values are an interval in seconds (unless other SI units are specified. eg. 5min)", "15min", &check_timer,
+ "Zero disables polling. Positive values are an interval in seconds (unless other SI units are specified. eg. 5min)",
+ "15min", &check_timer,
"Polling interval for time based changes to options, resource parameters and constraints.",
"The Cluster is primarily event driven, however the configuration can have elements that change based on time."
- " To ensure these changes take effect, we can optionally poll the cluster's status for changes." },
+ " To ensure these changes take effect, we can optionally poll the cluster's status for changes."
+ },
+
+ { "notification-script", NULL, "string", NULL, "/dev/null", &check_script,
+ "Notification script to be called after significant cluster events",
+ "Full path to a script that will be invoked when resources start/stop/fail, fencing occurs or nodes join/leave the cluster.\n"
+ "Must exist on all nodes in the cluster."
+ },
+ { "notification-target", NULL, "string", NULL, "", NULL,
+ "Destination for notifications (Optional)",
+ "Where should the supplied script send notifications to. Useful to avoid hard-coding this in the script."
+ },
+
{ "load-threshold", NULL, "percentage", NULL, "80%", &check_utilization,
"The maximum amount of system resources that should be used by nodes in the cluster",
"The cluster will slow down its recovery process when the amount of system resources used"
- " (currently CPU) approaches this limit", },
+ " (currently CPU) approaches this limit",
+ },
{ "node-action-limit", NULL, "integer", NULL, "0", &check_number,
"The maximum number of jobs that can be scheduled per node. Defaults to 2x cores"},
- { XML_CONFIG_ATTR_ELECTION_FAIL, "election_timeout", "time", NULL, "2min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." },
- { XML_CONFIG_ATTR_FORCE_QUIT, "shutdown_escalation", "time", NULL, "20min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." },
- { "crmd-integration-timeout", NULL, "time", NULL, "3min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." },
- { "crmd-finalization-timeout", NULL, "time", NULL, "30min", &check_timer, "*** Advanced Use Only ***.", "If you need to adjust this value, it probably indicates the presence of a bug." },
- { "crmd-transition-delay", NULL, "time", NULL, "0s", &check_timer, "*** Advanced Use Only ***\nEnabling this option will slow down cluster recovery under all conditions", "Delay cluster recovery for the configured interval to allow for additional/related events to occur.\nUseful if your configuration is sensitive to the order in which ping updates arrive." },
+ { XML_CONFIG_ATTR_ELECTION_FAIL, "election_timeout", "time", NULL, "2min", &check_timer,
+ "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug."
+ },
+ { XML_CONFIG_ATTR_FORCE_QUIT, "shutdown_escalation", "time", NULL, "20min", &check_timer,
+ "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug."
+ },
+ { "crmd-integration-timeout", NULL, "time", NULL, "3min", &check_timer,
+ "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug."
+ },
+ { "crmd-finalization-timeout", NULL, "time", NULL, "30min", &check_timer,
+ "*** Advanced Use Only ***.", "If you need to adjust this value, it probably indicates the presence of a bug."
+ },
+ { "crmd-transition-delay", NULL, "time", NULL, "0s", &check_timer,
+ "*** Advanced Use Only ***\n"
+ "Enabling this option will slow down cluster recovery under all conditions",
+ "Delay cluster recovery for the configured interval to allow for additional/related events to occur.\n"
+ "Useful if your configuration is sensitive to the order in which ping updates arrive."
+ },
{ "stonith-watchdog-timeout", NULL, "time", NULL, NULL, &check_timer,
- "How long to wait before we can assume nodes are safely down", NULL },
+ "How long to wait before we can assume nodes are safely down", NULL
+ },
{ "no-quorum-policy", "no_quorum_policy", "enum", "stop, freeze, ignore, suicide", "stop", &check_quorum, NULL, NULL },
#if SUPPORT_PLUGIN
@@ -927,6 +963,7 @@ crmd_pref(GHashTable * options, const char *name)
static void
config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
+ const char *script = NULL;
const char *value = NULL;
GHashTable *config_hash = NULL;
crm_time_t *now = crm_time_new(NULL);
@@ -955,6 +992,10 @@ config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void
verify_crmd_options(config_hash);
+ script = crmd_pref(config_hash, "notification-script");
+ value = crmd_pref(config_hash, "notification-target");
+ crmd_enable_notifications(script, value);
+
value = crmd_pref(config_hash, XML_CONFIG_ATTR_DC_DEADTIME);
election_trigger->period_ms = crm_get_msec(value);
diff --git a/crmd/crmd_utils.h b/crmd/crmd_utils.h
index 78214bf..7e8c3e6 100644
--- a/crmd/crmd_utils.h
+++ b/crmd/crmd_utils.h
@@ -21,6 +21,7 @@
# include <crm/crm.h>
# include <crm/common/xml.h>
# include <crm/cib/internal.h> /* For CIB_OP_MODIFY */
+# include "notify.h"
# define CLIENT_EXIT_WAIT 30
# define FAKE_TE_ID "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
diff --git a/crmd/lrm.c b/crmd/lrm.c
index 418e7cf..48195e8 100644
--- a/crmd/lrm.c
+++ b/crmd/lrm.c
@@ -2415,6 +2415,8 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
free(prefix);
}
+ crmd_notify_resource_op(lrm_state->node_name, op);
+
if (op->rsc_deleted) {
crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
diff --git a/crmd/notify.c b/crmd/notify.c
new file mode 100644
index 0000000..980bfa6
--- /dev/null
+++ b/crmd/notify.c
@@ -0,0 +1,188 @@
+/*
+ * Copyright (C) 2015 Andrew Beekhof <andrew@beekhof.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <crm_internal.h>
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include "notify.h"
+
+char *notify_script = NULL;
+char *notify_target = NULL;
+
+
+static const char *notify_keys[] =
+{
+ "CRM_notify_recipient",
+ "CRM_notify_node",
+ "CRM_notify_rsc",
+ "CRM_notify_task",
+ "CRM_notify_interval",
+ "CRM_notify_desc",
+ "CRM_notify_status",
+ "CRM_notify_target_rc",
+ "CRM_notify_rc",
+ "CRM_notify_kind",
+ "CRM_notify_version",
+};
+
+
+void
+crmd_enable_notifications(const char *script, const char *target)
+{
+ free(notify_script);
+ notify_script = NULL;
+
+ free(notify_target);
+ notify_target = NULL;
+
+ if(safe_str_eq(script, "/dev/null")) {
+ crm_notice("Notifications disabled");
+ return;
+ }
+
+ notify_script = strdup(script);
+ notify_target = strdup(target);
+ crm_notice("Notifications enabled");
+}
+
+static void
+set_notify_key(const char *name, const char *cvalue, char *value)
+{
+ int lpc;
+ bool found = 0;
+
+ if(cvalue == NULL) {
+ cvalue = value;
+ }
+
+ for(lpc = 0; lpc < DIMOF(notify_keys); lpc++) {
+ if(safe_str_eq(name, notify_keys[lpc])) {
+ found = 1;
+ crm_trace("Setting notify key %s = '%s'", name, cvalue);
+ setenv(name, cvalue, 1);
+ break;
+ }
+ }
+
+ CRM_ASSERT(found != 0);
+ free(value);
+}
+
+
+static void
+send_notification(const char *kind)
+{
+ int lpc;
+ pid_t pid;
+
+ crm_debug("Sending '%s' notification to '%s' via '%s'", kind, notify_target, notify_script);
+
+ set_notify_key("CRM_notify_recipient", notify_target, NULL);
+ set_notify_key("CRM_notify_kind", kind, NULL);
+ set_notify_key("CRM_notify_version", VERSION, NULL);
+
+ pid = fork();
+ if (pid == -1) {
+ crm_perror(LOG_ERR, "notification failed");
+ }
+
+ if (pid == 0) {
+ /* crm_debug("notification: I am the child. Executing the nofitication program."); */
+ execl(notify_script, notify_script, NULL);
+ exit(EXIT_FAILURE);
+
+ } else {
+ for(lpc = 0; lpc < DIMOF(notify_keys); lpc++) {
+ unsetenv(notify_keys[lpc]);
+ }
+ }
+}
+
+void crmd_notify_node_event(crm_node_t *node)
+{
+ if(notify_script == NULL) {
+ return;
+ }
+
+ set_notify_key("CRM_notify_node", node->uname, NULL);
+ set_notify_key("CRM_notify_desc", node->state, NULL);
+
+ send_notification("node");
+}
+
+void
+crmd_notify_fencing_op(stonith_event_t * e)
+{
+ char *desc = NULL;
+
+ if(notify_script) {
+ return;
+ }
+
+ desc = crm_strdup_printf("Operation %s requested by %s for peer %s: %s (ref=%s)",
+ e->operation, e->origin, e->target, pcmk_strerror(e->result),
+ e->id);
+
+ set_notify_key("CRM_notify_node", e->target, NULL);
+ set_notify_key("CRM_notify_task", e->operation, NULL);
+ set_notify_key("CRM_notify_desc", NULL, desc);
+ set_notify_key("CRM_notify_rc", NULL, crm_itoa(e->result));
+
+ send_notification("fencing");
+}
+
+void
+crmd_notify_resource_op(const char *node, lrmd_event_data_t * op)
+{
+ int target_rc = 0;
+
+ if(notify_script == NULL) {
+ return;
+ }
+
+ target_rc = rsc_op_expected_rc(op);
+ if(op->interval == 0 && target_rc == op->rc && safe_str_eq(op->op_type, RSC_STATUS)) {
+ /* Leave it up to the script if they want to notify for
+ * 'failed' probes, only swallow ones for which the result was
+ * unexpected.
+ *
+ * Even if we find a resource running, it was probably because
+ * someone erased the status section.
+ */
+ return;
+ }
+
+ set_notify_key("CRM_notify_node", node, NULL);
+
+ set_notify_key("CRM_notify_rsc", op->rsc_id, NULL);
+ set_notify_key("CRM_notify_task", op->op_type, NULL);
+ set_notify_key("CRM_notify_interval", NULL, crm_itoa(op->interval));
+
+ set_notify_key("CRM_notify_target_rc", NULL, crm_itoa(target_rc));
+ set_notify_key("CRM_notify_status", NULL, crm_itoa(op->op_status));
+ set_notify_key("CRM_notify_rc", NULL, crm_itoa(op->rc));
+
+ if(op->op_status == PCMK_LRM_OP_DONE) {
+ set_notify_key("CRM_notify_desc", services_ocf_exitcode_str(op->rc), NULL);
+ } else {
+ set_notify_key("CRM_notify_desc", services_lrm_status_str(op->op_status), NULL);
+ }
+
+ send_notification("resource");
+}
+
diff --git a/crmd/notify.h b/crmd/notify.h
new file mode 100644
index 0000000..4b138ea
--- /dev/null
+++ b/crmd/notify.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2015 Andrew Beekhof <andrew@beekhof.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#ifndef CRMD_NOTIFY__H
+# define CRMD_NOTIFY__H
+
+# include <crm/crm.h>
+# include <crm/cluster.h>
+# include <crm/stonith-ng.h>
+
+void crmd_enable_notifications(const char *script, const char *target);
+void crmd_notify_node_event(crm_node_t *node);
+void crmd_notify_fencing_op(stonith_event_t * e);
+void crmd_notify_resource_op(const char *node, lrmd_event_data_t * op);
+
+#endif
diff --git a/crmd/te_utils.c b/crmd/te_utils.c
index a1d29f6..22551ba 100644
--- a/crmd/te_utils.c
+++ b/crmd/te_utils.c
@@ -124,6 +124,8 @@ tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event)
return;
}
+ crmd_notify_fencing_op(st_event);
+
if (st_event->result == pcmk_ok && safe_str_eq("on", st_event->action)) {
crm_notice("%s was successfully unfenced by %s (at the request of %s)",
st_event->target, st_event->executioner ? st_event->executioner : "<anyone>", st_event->origin);
diff --git a/cts/CIB.py b/cts/CIB.py
index 8fbba6c..cd3a6a1 100644
--- a/cts/CIB.py
+++ b/cts/CIB.py
@@ -219,6 +219,8 @@ class CIB11(ConfigBase):
o["dc-deadtime"] = "5s"
o["no-quorum-policy"] = no_quorum
o["expected-quorum-votes"] = self.num_nodes
+ o["notification-script"] = "/var/lib/pacemaker/notify.sh"
+ o["notification-target"] = "/var/lib/pacemaker/notify.log"
if self.CM.Env["DoBSC"] == 1:
o["ident-string"] = "Linux-HA TEST configuration file - REMOVEME!!"
diff --git a/extra/pcmk_notify_sample.sh b/extra/pcmk_notify_sample.sh
new file mode 100755
index 0000000..83cf8e9
--- /dev/null
+++ b/extra/pcmk_notify_sample.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+#
+# Copyright (C) 2015 Andrew Beekhof <andrew@beekhof.net>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
+if [ -z $CRM_notify_version ]; then
+ echo "Pacemaker version 1.1.14 is required" >> ${CRM_notify_recipient}
+ exit 0
+fi
+
+case $CRM_notify_kind in
+ node)
+ echo "Node '${CRM_notify_node}' is now '${CRM_notify_desc}'" >> ${CRM_notify_recipient}
+ ;;
+ fencing)
+ # Other keys:
+ #
+ # CRM_notify_node
+ # CRM_notify_task
+ # CRM_notify_rc
+ #
+ echo "Fencing ${CRM_notify_desc}" >> ${CRM_notify_recipient}
+ ;;
+ resource)
+ # Other keys:
+ #
+ # CRM_notify_target_rc
+ # CRM_notify_status
+ # CRM_notify_rc
+ #
+ if [ ${CRM_notify_interval} = "0" ]; then
+ CRM_notify_interval=""
+ else
+ CRM_notify_interval=" (${CRM_notify_interval})"
+ fi
+
+ if [ ${CRM_notify_target_rc} = "0" ]; then
+ CRM_notify_target_rc=""
+ else
+ CRM_notify_target_rc=" (target: ${CRM_notify_target_rc})"
+ fi
+
+ case ${CRM_notify_desc} in
+ Cancelled) ;;
+ *)
+ echo "Resource operation '${CRM_notify_task}${CRM_notify_interval}' for '${CRM_notify_rsc}' on '${CRM_notify_node}': ${CRM_notify_desc}${CRM_notify_target_rc}" >> ${CRM_notify_recipient}
+ ;;
+ esac
+ ;;
+ *)
+ echo "Unhandled $CRM_notify_kind notification" >> ${CRM_notify_recipient}
+ env | grep CRM_notify >> ${CRM_notify_recipient}
+ ;;
+
+esac
diff --git a/include/crm_internal.h b/include/crm_internal.h
index c13bc7b..fb03537 100644
--- a/include/crm_internal.h
+++ b/include/crm_internal.h
@@ -127,6 +127,7 @@ gboolean check_timer(const char *value);
gboolean check_boolean(const char *value);
gboolean check_number(const char *value);
gboolean check_quorum(const char *value);
+gboolean check_script(const char *value);
gboolean check_utilization(const char *value);
/* Shared PE/crmd functionality */
diff --git a/lib/common/utils.c b/lib/common/utils.c
index 6a234dc..628cf2f 100644
--- a/lib/common/utils.c
+++ b/lib/common/utils.c
@@ -180,6 +180,33 @@ check_quorum(const char *value)
}
gboolean
+check_script(const char *value)
+{
+ struct stat st;
+
+ if(safe_str_eq(value, "/dev/null")) {
+ return TRUE;
+ }
+
+ if(stat(value, &st) != 0) {
+ crm_err("Script %s does not exist", value);
+ return FALSE;
+ }
+
+ if(S_ISREG(st.st_mode) == 0) {
+ crm_err("Script %s is not a regular file", value);
+ return FALSE;
+ }
+
+ if( (st.st_mode & (S_IXUSR | S_IXGRP )) == 0) {
+ crm_err("Script %s is not executable", value);
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+gboolean
check_utilization(const char *value)
{
char *end = NULL;