From 4c8e4432ad8749f5fc9ecc47e348ac5e6521c35a Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Nov 05 2019 19:42:16 +0000 Subject: import pacemaker-2.0.2-3.el8 --- diff --git a/.gitignore b/.gitignore index 879ad60..4bfe1e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ SOURCES/nagios-agents-metadata-105ab8a.tar.gz -SOURCES/pacemaker-0eb7991.tar.gz +SOURCES/pacemaker-744a30d.tar.gz diff --git a/.pacemaker.metadata b/.pacemaker.metadata index 89d7ae7..31dac21 100644 --- a/.pacemaker.metadata +++ b/.pacemaker.metadata @@ -1,2 +1,2 @@ ea6c0a27fd0ae8ce02f84a11f08a0d79377041c3 SOURCES/nagios-agents-metadata-105ab8a.tar.gz -0800ac9e89dcaae479976a54dc8819745b074030 SOURCES/pacemaker-0eb7991.tar.gz +98d783c49fa894c5bdc30f907f5355539030578d SOURCES/pacemaker-744a30d.tar.gz diff --git a/SOURCES/001-rc4.patch b/SOURCES/001-rc4.patch deleted file mode 100644 index 330f0ff..0000000 --- a/SOURCES/001-rc4.patch +++ /dev/null @@ -1,8608 +0,0 @@ -From 824738210258e8d0486f54726de5fe657b648a96 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Fri, 14 Dec 2018 16:58:30 -0500 -Subject: [PATCH 01/69] Bug: tests: Add CRM_EX_DIGEST definition to cts-cli.in. - -This is the value that's returned when unexpected output happens in the -CLI regression tests. However since it's not defined in cts-cli, 0 is -returned instead. Add a definition for it to the block with all the -other error codes. ---- - cts/cts-cli.in | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/cts/cts-cli.in b/cts/cts-cli.in -index 2ce912d..880cebb 100755 ---- a/cts/cts-cli.in -+++ b/cts/cts-cli.in -@@ -55,6 +55,7 @@ CRM_EX_INSUFFICIENT_PRIV=4 - CRM_EX_USAGE=64 - CRM_EX_CONFIG=78 - CRM_EX_OLD=103 -+CRM_EX_DIGEST=104 - CRM_EX_NOSUCH=105 - CRM_EX_UNSAFE=107 - CRM_EX_EXISTS=108 --- -1.8.3.1 - - -From f05828dc774059dc39444e186839753a8d5bf8f3 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Mon, 17 Dec 2018 18:25:12 +0100 -Subject: [PATCH 02/69] Build: spec: Provides: not dissected for extra arch - qualification - -That means that with arch-qualified-only virtual Provides: (introduced -as of 73e2c94a3 for which this is a follow-up), the dependants would -need to track Requires:/Recommends: etc. in the same qualified form -unconditionally (which was not previously true in pacemaker packaging -itself as arranged with the said commit, which led to this discovery), -since only that would match string-wise. Solution is to declare -Provides: for both forms in parallel, providing the depending packages -with the benefit of choice (or conversely, to prevent hard to debug -install transaction unresolvability issues; Ken did a great work -on this front). - -See also: -https://bugzilla.redhat.com/show_bug.cgi?id=1659259 ---- - pacemaker.spec.in | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/pacemaker.spec.in b/pacemaker.spec.in -index bfe68bf..dbc8360 100644 ---- a/pacemaker.spec.in -+++ b/pacemaker.spec.in -@@ -248,6 +248,7 @@ BuildRequires: inkscape asciidoc publican - %endif - %endif - -+Provides: pcmk-cluster-manager = %{version}-%{release} - Provides: pcmk-cluster-manager%{?_isa} = %{version}-%{release} - - %description -@@ -330,6 +331,7 @@ Requires: procps-ng - %endif - # -remote can be fully independent of systemd - %{?systemd_ordering}%{!?systemd_ordering:%{?systemd_requires}} -+Provides: pcmk-cluster-manager = %{version}-%{release} - Provides: pcmk-cluster-manager%{?_isa} = %{version}-%{release} - - %description remote --- -1.8.3.1 - - -From 2a24cd38aea749c94d9c5b2670ed9f52056abeef Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 12 Dec 2018 16:27:36 -0600 -Subject: [PATCH 03/69] Refactor: cts-exec: remove dead code - ---- - cts/cts-exec.in | 11 ----------- - 1 file changed, 11 deletions(-) - -diff --git a/cts/cts-exec.in b/cts/cts-exec.in -index c451841..235a966 100644 ---- a/cts/cts-exec.in -+++ b/cts/cts-exec.in -@@ -117,17 +117,6 @@ def output_from_command(command): - return pipe_output(test).split("\n") - - --def write_file(filename, contents, executable=False): -- """ Create a file. """ -- -- print("Installing %s ..." % (filename)) -- f = io.open(filename, "w+") -- f.write(contents) -- f.close() -- if executable: -- os.chmod(filename, EXECMODE) -- -- - class TestError(Exception): - """ Base class for exceptions in this module """ - pass --- -1.8.3.1 - - -From 7a749048e5462430c38a0b233b8e0e51f346f98e Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 13 Dec 2018 11:27:57 -0600 -Subject: [PATCH 04/69] Fix: tools: stonith_admin -I doesn't require an agent - -regression since cc4c8411b (2.0.1-rc1) ---- - tools/stonith_admin.c | 7 +++++-- - 1 file changed, 5 insertions(+), 2 deletions(-) - -diff --git a/tools/stonith_admin.c b/tools/stonith_admin.c -index c9ed971..5160505 100644 ---- a/tools/stonith_admin.c -+++ b/tools/stonith_admin.c -@@ -495,14 +495,17 @@ main(int argc, char **argv) - case '?': - crm_help(flag, CRM_EX_OK); - break; -- case 'I': -+ - case 'K': -- no_connect = 1; - required_agent = true; - /* fall through */ -+ case 'I': -+ no_connect = 1; -+ /* fall through */ - case 'L': - action = flag; - break; -+ - case 'q': - quiet = 1; - break; --- -1.8.3.1 - - -From 5f75a663a1b5ea13a4cef114dc13169b01b2b29a Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 18 Dec 2018 12:24:34 -0600 -Subject: [PATCH 05/69] Fix: libpe_status: avoid double free of stop_needed - list - -regression in 2.0.1-rc1 introduced by bf69beb3 ---- - lib/pengine/unpack.c | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c -index ac27121..a445442 100644 ---- a/lib/pengine/unpack.c -+++ b/lib/pengine/unpack.c -@@ -1153,6 +1153,7 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set) - } - } - g_list_free(data_set->stop_needed); -+ data_set->stop_needed = NULL; - } - - for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { --- -1.8.3.1 - - -From b8c592cb5fe431215b2f7755fe3a9c3a256f8d70 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 17 Dec 2018 14:16:48 -0600 -Subject: [PATCH 06/69] Build: GNUmakefile: update change log substitutions - ---- - GNUmakefile | 10 +++++++++- - 1 file changed, 9 insertions(+), 1 deletion(-) - -diff --git a/GNUmakefile b/GNUmakefile -index b9f7590..8e5bdd7 100644 ---- a/GNUmakefile -+++ b/GNUmakefile -@@ -333,7 +333,15 @@ changes: summary - @printf "\n- Features added since $(LAST_RELEASE)\n" - @git log --pretty=format:' +%s' --abbrev-commit $(LAST_RELEASE)..HEAD | grep -e Feature: | sed -e 's@Feature:@@' | sort -uf - @printf "\n- Changes since $(LAST_RELEASE)\n" -- @git log --pretty=format:' +%s' --abbrev-commit $(LAST_RELEASE)..HEAD | grep -e High: -e Fix: -e Bug | sed -e 's@Fix:@@' -e s@High:@@ -e s@Fencing:@fencing:@ -e 's@Bug@ Bug@' -e s@PE:@scheduler:@ -e s@pengine:@scheduler:@ | sort -uf -+ @git log --pretty=format:' +%s' --no-merges --abbrev-commit $(LAST_RELEASE)..HEAD \ -+ | grep -e High: -e Fix: -e Bug | sed \ -+ -e 's@\(Fix\|High\|Bug\):@@' \ -+ -e 's@\(cib\|pacemaker-based\|based\):@CIB:@' \ -+ -e 's@\(crmd\|pacemaker-controld\|controld\):@controller:@' \ -+ -e 's@\(lrmd\|pacemaker-execd\|execd\):@executor:@' \ -+ -e 's@\(Fencing\|stonithd\|stonith\|pacemaker-fenced\|fenced\):@fencing:@' \ -+ -e 's@\(PE\|pengine\|pacemaker-schedulerd\|schedulerd\):@scheduler:@' \ -+ | sort -uf - - changelog: - @make changes > ChangeLog --- -1.8.3.1 - - -From dc51b0e7e3c910c1d6233a259c66f5be3f93eb73 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 17 Dec 2018 17:29:07 -0600 -Subject: [PATCH 07/69] Build: replace: remove uuid_parse.c - -doesn't appear to have ever been used, likely was carried over from heartbeat ---- - replace/uuid_parse.c | 515 --------------------------------------------------- - 1 file changed, 515 deletions(-) - delete mode 100644 replace/uuid_parse.c - -diff --git a/replace/uuid_parse.c b/replace/uuid_parse.c -deleted file mode 100644 -index 5da6223..0000000 ---- a/replace/uuid_parse.c -+++ /dev/null -@@ -1,515 +0,0 @@ --/* -- * uuid: emulation of e2fsprogs interface if implementation lacking. -- * -- * This library is free software; you can redistribute it and/or -- * modify it under the terms of the GNU Lesser General Public -- * License as published by the Free Software Foundation; either -- * version 2.1 of the License, or (at your option) any later version. -- * -- * This library is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- * Lesser General Public License for more details. -- * -- * You should have received a copy of the GNU Lesser General Public -- * License along with this library; if not, write to the Free Software -- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -- * -- * Original uuid implementation: copyright (C) Theodore Ts'o -- * -- * This importation into heartbeat: -- * Copyright (C) 2004 David Lee -- * -- */ -- --#include --#include --#ifdef HAVE_UNISTD_H --# include --#endif --#ifdef HAVE_STDLIB_H --# include --#endif --#include --#include -- --#include -- --/* -- * Local "replace" implementation of uuid functions. -- */ -- --#include --#include --#include -- --/* UUID Variant definitions */ --#define UUID_VARIANT_NCS 0 --#define UUID_VARIANT_DCE 1 --#define UUID_VARIANT_MICROSOFT 2 --#define UUID_VARIANT_OTHER 3 -- --/* UUID Type definitions */ --#define UUID_TYPE_DCE_TIME 1 --#define UUID_TYPE_DCE_RANDOM 4 -- --/* For uuid_compare() */ --#define UUCMP(u1,u2) if (u1 != u2) return((u1 < u2) ? -1 : 1); -- --/************************************ -- * Private types -- ************************************/ -- --#define longlong long long -- --/* -- * Offset between 15-Oct-1582 and 1-Jan-70 -- */ --#define TIME_OFFSET_HIGH 0x01B21DD2 --#define TIME_OFFSET_LOW 0x13814000 -- --#if (SIZEOF_INT == 4) --typedef unsigned int __u32; --#elif (SIZEOF_LONG == 4) --typedef unsigned long __u32; --#endif -- --#if (SIZEOF_INT == 2) --typedef int __s16; --typedef unsigned int __u16; --#elif (SIZEOF_SHORT == 2) --typedef short __s16; --typedef unsigned short __u16; --#endif -- --typedef unsigned char __u8; -- --struct uuid { -- __u32 time_low; -- __u16 time_mid; -- __u16 time_hi_and_version; -- __u16 clock_seq; -- __u8 node[6]; --}; -- --/************************************ -- * internal routines -- ************************************/ --static void --uuid_pack(const struct uuid *uu, uuid_t ptr) --{ -- __u32 tmp; -- unsigned char *out = ptr; -- -- tmp = uu->time_low; -- out[3] = (unsigned char)tmp; -- tmp >>= 8; -- out[2] = (unsigned char)tmp; -- tmp >>= 8; -- out[1] = (unsigned char)tmp; -- tmp >>= 8; -- out[0] = (unsigned char)tmp; -- -- tmp = uu->time_mid; -- out[5] = (unsigned char)tmp; -- tmp >>= 8; -- out[4] = (unsigned char)tmp; -- -- tmp = uu->time_hi_and_version; -- out[7] = (unsigned char)tmp; -- tmp >>= 8; -- out[6] = (unsigned char)tmp; -- -- tmp = uu->clock_seq; -- out[9] = (unsigned char)tmp; -- tmp >>= 8; -- out[8] = (unsigned char)tmp; -- -- memcpy(out + 10, uu->node, 6); --} -- --static void --uuid_unpack(const uuid_t in, struct uuid *uu) --{ -- const __u8 *ptr = in; -- __u32 tmp; -- -- tmp = *ptr++; -- tmp = (tmp << 8) | *ptr++; -- tmp = (tmp << 8) | *ptr++; -- tmp = (tmp << 8) | *ptr++; -- uu->time_low = tmp; -- -- tmp = *ptr++; -- tmp = (tmp << 8) | *ptr++; -- uu->time_mid = tmp; -- -- tmp = *ptr++; -- tmp = (tmp << 8) | *ptr++; -- uu->time_hi_and_version = tmp; -- -- tmp = *ptr++; -- tmp = (tmp << 8) | *ptr++; -- uu->clock_seq = tmp; -- -- memcpy(uu->node, ptr, 6); --} -- --/************************************ -- * Main routines, except uuid_generate*() -- ************************************/ --void --uuid_clear(uuid_t uu) --{ -- memset(uu, 0, 16); --} -- --int --uuid_compare(const uuid_t uu1, const uuid_t uu2) --{ -- struct uuid uuid1, uuid2; -- -- uuid_unpack(uu1, &uuid1); -- uuid_unpack(uu2, &uuid2); -- -- UUCMP(uuid1.time_low, uuid2.time_low); -- UUCMP(uuid1.time_mid, uuid2.time_mid); -- UUCMP(uuid1.time_hi_and_version, uuid2.time_hi_and_version); -- UUCMP(uuid1.clock_seq, uuid2.clock_seq); -- return memcmp(uuid1.node, uuid2.node, 6); --} -- --void --uuid_copy(uuid_t dst, const uuid_t src) --{ -- unsigned char *cp1; -- const unsigned char *cp2; -- int i; -- -- for (i = 0, cp1 = dst, cp2 = src; i < 16; i++) -- *cp1++ = *cp2++; --} -- --/* if uu is the null uuid, return 1 else 0 */ --int --uuid_is_null(const uuid_t uu) --{ -- const unsigned char *cp; -- int i; -- -- for (i = 0, cp = uu; i < 16; i++) -- if (*cp++) -- return 0; -- return 1; --} -- --/* 36byte-string=>uuid */ --int --uuid_parse(const char *in, uuid_t uu) --{ -- struct uuid uuid; -- int i; -- const char *cp; -- char buf[3]; -- -- if (strlen(in) != 36) -- return -1; -- for (i = 0, cp = in; i <= 36; i++, cp++) { -- if ((i == 8) || (i == 13) || (i == 18) || (i == 23)) { -- if (*cp == '-') -- continue; -- else -- return -1; -- } -- if (i == 36) -- if (*cp == 0) -- continue; -- if (!isxdigit((int)*cp)) -- return -1; -- } -- uuid.time_low = strtoul(in, NULL, 16); -- uuid.time_mid = strtoul(in + 9, NULL, 16); -- uuid.time_hi_and_version = strtoul(in + 14, NULL, 16); -- uuid.clock_seq = strtoul(in + 19, NULL, 16); -- cp = in + 24; -- buf[2] = 0; -- for (i = 0; i < 6; i++) { -- buf[0] = *cp++; -- buf[1] = *cp++; -- uuid.node[i] = strtoul(buf, NULL, 16); -- } -- -- uuid_pack(&uuid, uu); -- return 0; --} -- --/* uuid=>36byte-string-with-null */ --void --uuid_unparse(const uuid_t uu, char *out) --{ -- struct uuid uuid; -- -- uuid_unpack(uu, &uuid); -- sprintf(out, -- "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x", -- uuid.time_low, uuid.time_mid, uuid.time_hi_and_version, -- uuid.clock_seq >> 8, uuid.clock_seq & 0xFF, -- uuid.node[0], uuid.node[1], uuid.node[2], uuid.node[3], uuid.node[4], uuid.node[5]); --} -- --/************************************ -- * Main routines: uuid_generate*() -- ************************************/ -- --#include --#include --#include -- --#ifdef HAVE_SYS_IOCTL_H --# include --#endif --#ifdef HAVE_SYS_SOCKET_H --# include --#endif --#ifdef HAVE_SYS_SOCKIO_H --# include --#endif --#ifdef HAVE_NET_IF_H --# include --#endif --#ifdef HAVE_NETINET_IN_H --# include --#endif -- --#ifdef HAVE_SRANDOM --# define srand(x) srandom(x) --# define rand() random() --#endif -- --static int --get_random_fd(void) --{ -- struct timeval tv; -- static int fd = -2; -- int i; -- -- if (fd == -2) { -- gettimeofday(&tv, 0); -- fd = open("/dev/urandom", O_RDONLY); -- if (fd == -1) -- fd = open("/dev/random", O_RDONLY | O_NONBLOCK); -- srand((getpid() << 16) ^ getuid() ^ tv.tv_sec ^ tv.tv_usec); -- } -- /* Crank the random number generator a few times */ -- gettimeofday(&tv, 0); -- for (i = (tv.tv_sec ^ tv.tv_usec) & 0x1F; i > 0; i--) -- rand(); -- return fd; --} -- --/* -- * Generate a series of random bytes. Use /dev/urandom if possible, -- * and if not, use srandom/random. -- */ --static void --get_random_bytes(void *buf, int nbytes) --{ -- int i, n = nbytes, fd = get_random_fd(); -- int lose_counter = 0; -- unsigned char *cp = (unsigned char *)buf; -- -- if (fd >= 0) { -- while (n > 0) { -- i = read(fd, cp, n); -- if (i <= 0) { -- if (lose_counter++ > 16) -- break; -- continue; -- } -- n -= i; -- cp += i; -- lose_counter = 0; -- } -- } -- -- /* -- * We do this all the time, but this is the only source of -- * randomness if /dev/random/urandom is out to lunch. -- */ -- for (cp = buf, i = 0; i < nbytes; i++) -- *cp++ ^= (rand() >> 7) & 0xFF; -- return; --} -- --/* -- * Get the ethernet hardware address, if we can find it... -- */ --static int --get_node_id(unsigned char *node_id) --{ --#ifdef HAVE_NET_IF_H -- int sd; -- struct ifreq ifr, *ifrp; -- struct ifconf ifc; -- char buf[1024]; -- int n, i; -- unsigned char *a; -- --/* -- * BSD 4.4 defines the size of an ifreq to be -- * max(sizeof(ifreq), sizeof(ifreq.ifr_name)+ifreq.ifr_addr.sa_len -- * However, under earlier systems, sa_len isn't present, so the size is -- * just sizeof(struct ifreq) -- */ --# ifdef HAVE_SA_LEN --# ifndef max --# define max(a,b) ((a) > (b) ? (a) : (b)) --# endif --# define ifreq_size(i) max(sizeof(struct ifreq),\ -- sizeof((i).ifr_name)+(i).ifr_addr.sa_len) --# else --# define ifreq_size(i) sizeof(struct ifreq) --# endif /* HAVE_SA_LEN */ -- -- sd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP); -- if (sd < 0) { -- return -1; -- } -- memset(buf, 0, sizeof(buf)); -- ifc.ifc_len = sizeof(buf); -- ifc.ifc_buf = buf; -- if (ioctl(sd, SIOCGIFCONF, (char *)&ifc) < 0) { -- close(sd); -- return -1; -- } -- n = ifc.ifc_len; -- for (i = 0; i < n; i += ifreq_size(*ifr)) { -- ifrp = (struct ifreq *)((char *)ifc.ifc_buf + i); -- strncpy(ifr.ifr_name, ifrp->ifr_name, IFNAMSIZ); --# ifdef SIOCGIFHWADDR -- if (ioctl(sd, SIOCGIFHWADDR, &ifr) < 0) -- continue; -- a = (unsigned char *)&ifr.ifr_hwaddr.sa_data; --# else --# ifdef SIOCGENADDR -- if (ioctl(sd, SIOCGENADDR, &ifr) < 0) -- continue; -- a = (unsigned char *)ifr.ifr_enaddr; --# else -- /* -- * XXX we don't have a way of getting the hardware -- * address -- */ -- close(sd); -- return 0; --# endif /* SIOCGENADDR */ --# endif /* SIOCGIFHWADDR */ -- if (!a[0] && !a[1] && !a[2] && !a[3] && !a[4] && !a[5]) -- continue; -- if (node_id) { -- memcpy(node_id, a, 6); -- close(sd); -- return 1; -- } -- } -- close(sd); --#endif -- return 0; --} -- --/* Assume that the gettimeofday() has microsecond granularity */ --#define MAX_ADJUSTMENT 10 -- --static int --get_clock(__u32 * clock_high, __u32 * clock_low, __u16 * ret_clock_seq) --{ -- static int adjustment = 0; -- static struct timeval last = { 0, 0 }; -- static __u16 clock_seq; -- struct timeval tv; -- unsigned longlong clock_reg; -- -- try_again: -- gettimeofday(&tv, 0); -- if ((last.tv_sec == 0) && (last.tv_usec == 0)) { -- get_random_bytes(&clock_seq, sizeof(clock_seq)); -- clock_seq &= 0x1FFF; -- last = tv; -- last.tv_sec--; -- } -- if ((tv.tv_sec < last.tv_sec) || ((tv.tv_sec == last.tv_sec) && (tv.tv_usec < last.tv_usec))) { -- clock_seq = (clock_seq + 1) & 0x1FFF; -- adjustment = 0; -- last = tv; -- } else if ((tv.tv_sec == last.tv_sec) && (tv.tv_usec == last.tv_usec)) { -- if (adjustment >= MAX_ADJUSTMENT) -- goto try_again; -- adjustment++; -- } else { -- adjustment = 0; -- last = tv; -- } -- -- clock_reg = tv.tv_usec * 10 + adjustment; -- clock_reg += ((unsigned longlong)tv.tv_sec) * 10000000; -- clock_reg += (((unsigned longlong)0x01B21DD2) << 32) + 0x13814000; -- -- *clock_high = clock_reg >> 32; -- *clock_low = clock_reg; -- *ret_clock_seq = clock_seq; -- return 0; --} -- --/* create a new uuid, based on randomness */ --void --uuid_generate_random(uuid_t out) --{ -- uuid_t buf; -- struct uuid uu; -- -- get_random_bytes(buf, sizeof(buf)); -- uuid_unpack(buf, &uu); -- -- uu.clock_seq = (uu.clock_seq & 0x3FFF) | 0x8000; -- uu.time_hi_and_version = (uu.time_hi_and_version & 0x0FFF) | 0x4000; -- uuid_pack(&uu, out); --} -- --/* create a new uuid, based on time */ --static void --uuid_generate_time(uuid_t out) --{ -- static unsigned char node_id[6]; -- static int has_init = 0; -- struct uuid uu; -- __u32 clock_mid; -- -- if (!has_init) { -- if (get_node_id(node_id) <= 0) { -- get_random_bytes(node_id, 6); -- /* -- * Set multicast bit, to prevent conflicts -- * with IEEE 802 addresses obtained from -- * network cards -- */ -- node_id[0] |= 0x80; -- } -- has_init = 1; -- } -- get_clock(&clock_mid, &uu.time_low, &uu.clock_seq); -- uu.clock_seq |= 0x8000; -- uu.time_mid = (__u16) clock_mid; -- uu.time_hi_and_version = (clock_mid >> 16) | 0x1000; -- memcpy(uu.node, node_id, 6); -- uuid_pack(&uu, out); --} -- --void --uuid_generate(uuid_t out) --{ -- if (get_random_fd() >= 0) { -- uuid_generate_random(out); -- } else { -- uuid_generate_time(out); -- } --} --- -1.8.3.1 - - -From d57aa84c1ce8b432c2a90615b67564f881d3883f Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 17 Dec 2018 17:46:06 -0600 -Subject: [PATCH 08/69] Build: spec: declare bundled gnulib - ---- - pacemaker.spec.in | 3 +++ - 1 file changed, 3 insertions(+) - -diff --git a/pacemaker.spec.in b/pacemaker.spec.in -index dbc8360..6b2b268 100644 ---- a/pacemaker.spec.in -+++ b/pacemaker.spec.in -@@ -251,6 +251,9 @@ BuildRequires: inkscape asciidoc publican - Provides: pcmk-cluster-manager = %{version}-%{release} - Provides: pcmk-cluster-manager%{?_isa} = %{version}-%{release} - -+# Pacemaker uses the crypto/md5 module from gnulib -+Provides: bundled(gnulib) -+ - %description - Pacemaker is an advanced, scalable High-Availability cluster resource - manager. --- -1.8.3.1 - - -From 15814c6c0dc844cac87023ddcebf083a35beef4d Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 13 Dec 2018 10:25:14 -0600 -Subject: [PATCH 09/69] Doc: ChangeLog: update for 2.0.1-rc2 release - ---- - ChangeLog | 12 +++++++++++- - 1 file changed, 11 insertions(+), 1 deletion(-) - -diff --git a/ChangeLog b/ChangeLog -index 875fe5c..23d8307 100644 ---- a/ChangeLog -+++ b/ChangeLog -@@ -1,3 +1,11 @@ -+* Wed Dec 19 2018 Ken Gaillot Pacemaker-2.0.1-rc2 -+- Changesets: 12 -+- Diff: 2 files changed, 6 insertions(+), 2 deletions(-) -+ -+- Changes since Pacemaker-2.0.1-rc1 -+ + libpe_status: avoid double free of stop_needed list (regression in 2.0.1-rc1) -+ + tools: stonith_admin -I doesn't require an agent (regression in 2.0.1-rc1) -+ - * Wed Dec 12 2018 Ken Gaillot Pacemaker-2.0.1-rc1 - - Changesets: 481 - 164 files changed, 7717 insertions(+), 4398 deletions(-) -@@ -15,6 +23,9 @@ - (regression since 1.1.12) - + Pacemaker Remote: avoid unnecessary downtime when moving resource to - Pacemaker Remote node that fails to come up (regression since 1.1.18) -+ + scheduler: clone notifications could be scheduled for a stopped -+ Pacemaker Remote node and block all further cluster actions -+ (regression since 2.0.0) - + tools: crm_resource -C could fail to clean up all failures in one run - (regression since 2.0.0) - + build: spec file now puts XML schemas in new pacemaker-schemas package -@@ -38,7 +49,6 @@ - + scheduler: start unique clone instances in numerical order - + scheduler: convert unique clones to anonymous clones if not supported by standard - + scheduler: associate pending tasks with correct clone instance -- + scheduler: don't send clone notifications to a stopped remote node - + scheduler: ensure bundle clone notifications are directed to correct host - + scheduler: avoid improper monitor rescheduling or fail count clearing for bundles - + scheduler: honor asymmetric orderings even when restarting --- -1.8.3.1 - - -From 74262eb5602bbb4d6406fc0a92766d5fdae9b303 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 20 Dec 2018 10:48:10 -0600 -Subject: [PATCH 10/69] Refactor: controller: remove dead code - ---- - daemons/controld/controld_fsa.h | 15 --------------- - daemons/controld/controld_messages.h | 2 -- - 2 files changed, 17 deletions(-) - -diff --git a/daemons/controld/controld_fsa.h b/daemons/controld/controld_fsa.h -index 1da88ff..a1f4dfd 100644 ---- a/daemons/controld/controld_fsa.h -+++ b/daemons/controld/controld_fsa.h -@@ -530,11 +530,6 @@ void do_pe_invoke(long long action, enum crmd_fsa_cause cause, - enum crmd_fsa_state cur_state, - enum crmd_fsa_input current_input, fsa_data_t *msg_data); - --/* A_ERROR */ --void do_error(long long action, enum crmd_fsa_cause cause, -- enum crmd_fsa_state cur_state, -- enum crmd_fsa_input cur_input, fsa_data_t *msg_data); -- - /* A_LOG */ - void do_log(long long action, enum crmd_fsa_cause cause, - enum crmd_fsa_state cur_state, -@@ -665,11 +660,6 @@ void do_cl_join_finalize_respond(long long action, enum crmd_fsa_cause cause, - enum crmd_fsa_input current_input, - fsa_data_t *msg_data); - --/* A_UPDATE_NODESTATUS */ --void do_update_node_status(long long action, enum crmd_fsa_cause cause, -- enum crmd_fsa_state cur_state, -- enum crmd_fsa_input cur_input, fsa_data_t *msg_data); -- - /* A_LRM_INVOKE */ - void do_lrm_invoke(long long action, enum crmd_fsa_cause cause, - enum crmd_fsa_state cur_state, -@@ -685,11 +675,6 @@ void do_te_invoke(long long action, enum crmd_fsa_cause cause, - enum crmd_fsa_state cur_state, - enum crmd_fsa_input cur_input, fsa_data_t *msg_data); - --/* A_TE_INVOKE */ --void do_te_copyto(long long action, enum crmd_fsa_cause cause, -- enum crmd_fsa_state cur_state, -- enum crmd_fsa_input cur_input, fsa_data_t *msg_data); -- - /* A_SHUTDOWN_REQ */ - void do_shutdown_req(long long action, enum crmd_fsa_cause cause, - enum crmd_fsa_state cur_state, -diff --git a/daemons/controld/controld_messages.h b/daemons/controld/controld_messages.h -index 8c80b5c..4c7e777 100644 ---- a/daemons/controld/controld_messages.h -+++ b/daemons/controld/controld_messages.h -@@ -74,8 +74,6 @@ gboolean is_message(void); - - extern gboolean relay_message(xmlNode * relay_message, gboolean originated_locally); - --extern void process_message(xmlNode * msg, gboolean originated_locally, const char *src_node_name); -- - extern gboolean send_msg_via_ipc(xmlNode * msg, const char *sys); - - gboolean crmd_is_proxy_session(const char *session); --- -1.8.3.1 - - -From 005d76b7038f94d7ed4067a1e7c936baac2f0aba Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 20 Dec 2018 10:51:06 -0600 -Subject: [PATCH 11/69] Refactor: libstonithd: make function declaration - C++-compatible - -just an argument name, so still backward-compatible ---- - include/crm/stonith-ng.h | 2 +- - lib/fencing/st_client.c | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/include/crm/stonith-ng.h b/include/crm/stonith-ng.h -index aa225e2..62a72d9 100644 ---- a/include/crm/stonith-ng.h -+++ b/include/crm/stonith-ng.h -@@ -83,7 +83,7 @@ enum stonith_namespace { - }; - - enum stonith_namespace stonith_text2namespace(const char *namespace_s); --const char *stonith_namespace2text(enum stonith_namespace namespace); -+const char *stonith_namespace2text(enum stonith_namespace st_namespace); - enum stonith_namespace stonith_get_namespace(const char *agent, - const char *namespace_s); - -diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c -index 456bc80..9376105 100644 ---- a/lib/fencing/st_client.c -+++ b/lib/fencing/st_client.c -@@ -152,9 +152,9 @@ stonith_text2namespace(const char *namespace_s) - * \return Namespace name as string - */ - const char * --stonith_namespace2text(enum stonith_namespace namespace) -+stonith_namespace2text(enum stonith_namespace st_namespace) - { -- switch (namespace) { -+ switch (st_namespace) { - case st_namespace_any: return "any"; - case st_namespace_rhcs: return "stonith-ng"; - case st_namespace_internal: return "internal"; --- -1.8.3.1 - - -From 1a83b5834b20fea1ff1d509659b13776b75a167f Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 21 Dec 2018 11:55:07 -0600 -Subject: [PATCH 12/69] Low: libpe_status: avoid use-after-free when logging at - trace level - ---- - lib/pengine/status.c | 59 ++++++++++++++++++++++++++++++++-------------------- - 1 file changed, 37 insertions(+), 22 deletions(-) - -diff --git a/lib/pengine/status.c b/lib/pengine/status.c -index dcbdc16..0684f52 100644 ---- a/lib/pengine/status.c -+++ b/lib/pengine/status.c -@@ -125,6 +125,17 @@ cluster_status(pe_working_set_t * data_set) - return TRUE; - } - -+/*! -+ * \internal -+ * \brief Free a list of pe_resource_t -+ * -+ * \param[in] resources List to free -+ * -+ * \note When a working set's resource list is freed, that includes the original -+ * storage for the uname and id of any Pacemaker Remote nodes in the -+ * working set's node list, so take care not to use those afterward. -+ * \todo Refactor pe_node_t to strdup() the node name. -+ */ - static void - pe_free_resources(GListPtr resources) - { -@@ -158,32 +169,36 @@ pe_free_actions(GListPtr actions) - static void - pe_free_nodes(GListPtr nodes) - { -- GListPtr iterator = nodes; -+ for (GList *iterator = nodes; iterator != NULL; iterator = iterator->next) { -+ pe_node_t *node = (pe_node_t *) iterator->data; - -- while (iterator != NULL) { -- node_t *node = (node_t *) iterator->data; -- struct pe_node_shared_s *details = node->details; -+ // Shouldn't be possible, but to be safe ... -+ if (node == NULL) { -+ continue; -+ } -+ if (node->details == NULL) { -+ free(node); -+ continue; -+ } - -- iterator = iterator->next; -+ /* This is called after pe_free_resources(), which means that we can't -+ * use node->details->uname for Pacemaker Remote nodes. -+ */ -+ crm_trace("Freeing node %s", (is_remote_node(node)? -+ "(Pacemaker Remote)" : node->details->uname)); - -- crm_trace("deleting node"); -- print_node("delete", node, FALSE); -- -- if (details != NULL) { -- crm_trace("%s is being deleted", details->uname); -- if (details->attrs != NULL) { -- g_hash_table_destroy(details->attrs); -- } -- if (details->utilization != NULL) { -- g_hash_table_destroy(details->utilization); -- } -- if (details->digest_cache != NULL) { -- g_hash_table_destroy(details->digest_cache); -- } -- g_list_free(details->running_rsc); -- g_list_free(details->allocated_rsc); -- free(details); -+ if (node->details->attrs != NULL) { -+ g_hash_table_destroy(node->details->attrs); -+ } -+ if (node->details->utilization != NULL) { -+ g_hash_table_destroy(node->details->utilization); -+ } -+ if (node->details->digest_cache != NULL) { -+ g_hash_table_destroy(node->details->digest_cache); - } -+ g_list_free(node->details->running_rsc); -+ g_list_free(node->details->allocated_rsc); -+ free(node->details); - free(node); - } - if (nodes != NULL) { --- -1.8.3.1 - - -From 847ff91992ad1e657bfda7b3f4e7446e0e91b398 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 21 Dec 2018 14:46:59 -0600 -Subject: [PATCH 13/69] Log: libpe_status: improve trace messages when finding - actions - -Previously, "Node mismatch" would be logged for matches. ---- - lib/pengine/utils.c | 34 +++++++++++++++++++++------------- - 1 file changed, 21 insertions(+), 13 deletions(-) - -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index 924085b..8b44c8b 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -1465,27 +1465,35 @@ find_actions(GListPtr input, const char *key, const node_t *on_node) - GListPtr - find_actions_exact(GListPtr input, const char *key, node_t * on_node) - { -- GListPtr gIter = input; -- GListPtr result = NULL; -+ GList *result = NULL; - - CRM_CHECK(key != NULL, return NULL); - -- for (; gIter != NULL; gIter = gIter->next) { -- action_t *action = (action_t *) gIter->data; -+ if (on_node == NULL) { -+ crm_trace("Not searching for action %s because node not specified", -+ key); -+ return NULL; -+ } - -- crm_trace("Matching %s against %s", key, action->uuid); -- if (safe_str_neq(key, action->uuid)) { -- crm_trace("Key mismatch: %s vs. %s", key, action->uuid); -- continue; -+ for (GList *gIter = input; gIter != NULL; gIter = gIter->next) { -+ pe_action_t *action = (pe_action_t *) gIter->data; - -- } else if (on_node == NULL || action->node == NULL) { -- crm_trace("on_node=%p, action->node=%p", on_node, action->node); -- continue; -+ if (action->node == NULL) { -+ crm_trace("Skipping comparison of %s vs action %s without node", -+ key, action->uuid); - -- } else if (safe_str_eq(on_node->details->id, action->node->details->id)) { -+ } else if (safe_str_neq(key, action->uuid)) { -+ crm_trace("Desired action %s doesn't match %s", key, action->uuid); -+ -+ } else if (safe_str_neq(on_node->details->id, -+ action->node->details->id)) { -+ crm_trace("Action %s desired node ID %s doesn't match %s", -+ key, on_node->details->id, action->node->details->id); -+ -+ } else { -+ crm_trace("Action %s matches", key); - result = g_list_prepend(result, action); - } -- crm_trace("Node mismatch: %s vs. %s", on_node->details->id, action->node->details->id); - } - - return result; --- -1.8.3.1 - - -From 61ecbc3e059e84691dc23908fd9d5663ba828bb7 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 21 Dec 2018 16:34:44 -0600 -Subject: [PATCH 14/69] Log: libpe_status: downgrade remote node - fence-before-clear message - -Previously, check_operation_expiry() would order remote node fencing -before clearing of the connection fail count (and log a notice) when -the remote node was unclean. - -However, at the point that is called, the nodes are always unclean -(link_rsc2remotenode() sets unclean after unpacking resources, and -unpack_node_loop() won't clear it until after unpacking status). - -Now, skip the unclean check (it is always true, and not completely -necessary since the fence op is created as optional), and lower -the log message to info. ---- - lib/pengine/unpack.c | 26 +++++++++++++++++++------- - 1 file changed, 19 insertions(+), 7 deletions(-) - -diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c -index a445442..77fd79d 100644 ---- a/lib/pengine/unpack.c -+++ b/lib/pengine/unpack.c -@@ -2932,19 +2932,31 @@ static bool check_operation_expiry(resource_t *rsc, node_t *node, int rc, xmlNod - } - - if (clear_reason != NULL) { -- node_t *remote_node = pe_find_node(data_set->nodes, rsc->id); -+ // Schedule clearing of the fail count - pe_action_t *clear_op = pe__clear_failcount(rsc, node, clear_reason, - data_set); - - if (is_set(data_set->flags, pe_flag_stonith_enabled) -- && rsc->remote_reconnect_ms -- && remote_node -- && remote_node->details->unclean) { -+ && rsc->remote_reconnect_ms) { - -- action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL, data_set); -- crm_notice("Waiting for %s to complete before clearing %s failure for remote node %s", fence?fence->uuid:"nil", task, rsc->id); -+ pe_node_t *remote_node = pe_find_node(data_set->nodes, rsc->id); - -- order_actions(fence, clear_op, pe_order_implies_then); -+ if (remote_node) { -+ /* If we're clearing a remote connection due to a reconnect -+ * interval, we want to wait until any scheduled fencing -+ * completes. -+ * -+ * We could limit this to remote_node->details->unclean, but at -+ * this point, that's always true (it won't be reliable until -+ * after unpack_node_loop() is done). -+ */ -+ pe_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL, -+ data_set); -+ -+ crm_info("Clearing %s failure will wait until any scheduled " -+ "fencing of %s completes", task, rsc->id); -+ order_actions(fence, clear_op, pe_order_implies_then); -+ } - } - } - --- -1.8.3.1 - - -From 3ecfacd6d0b2f26769058f627f0c0df28af13a6c Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 3 Jan 2019 10:40:32 -0600 -Subject: [PATCH 15/69] Log: scheduler: downgrade clone pre-allocation message - -When allocating clone instances, we pre-allocate instances to their existing -location if possible. If not, we would previously log a notice about -"Pre-allocation failed", which might make this seem more significant than it -is (it's normal when the instance is moving). So, downgrade it to info, -and don't say it "failed". - -This also includes some trivial refactoring for efficiency and clarity. ---- - daemons/schedulerd/sched_clone.c | 23 ++++++++++++----------- - 1 file changed, 12 insertions(+), 11 deletions(-) - -diff --git a/daemons/schedulerd/sched_clone.c b/daemons/schedulerd/sched_clone.c -index 9b42a83..8f01f56 100644 ---- a/daemons/schedulerd/sched_clone.c -+++ b/daemons/schedulerd/sched_clone.c -@@ -442,18 +442,19 @@ color_instance(resource_t * rsc, node_t * prefer, gboolean all_coloc, int limit, - - backup = node_hash_dup(rsc->allowed_nodes); - chosen = rsc->cmds->allocate(rsc, prefer, data_set); -+ if (chosen && prefer && (chosen->details != prefer->details)) { -+ crm_info("Not pre-allocating %s to %s because %s is better", -+ rsc->id, prefer->details->uname, chosen->details->uname); -+ g_hash_table_destroy(rsc->allowed_nodes); -+ rsc->allowed_nodes = backup; -+ native_deallocate(rsc); -+ chosen = NULL; -+ backup = NULL; -+ } - if (chosen) { -- node_t *local_node = parent_node_instance(rsc, chosen); -- if (prefer && (chosen->details != prefer->details)) { -- crm_notice("Pre-allocation failed: got %s instead of %s", -- chosen->details->uname, prefer->details->uname); -- g_hash_table_destroy(rsc->allowed_nodes); -- rsc->allowed_nodes = backup; -- native_deallocate(rsc); -- chosen = NULL; -- backup = NULL; -- -- } else if (local_node) { -+ pe_node_t *local_node = parent_node_instance(rsc, chosen); -+ -+ if (local_node) { - local_node->count++; - - } else if (is_set(rsc->flags, pe_rsc_managed)) { --- -1.8.3.1 - - -From 6678aa6cc2c3d515f851002b725e2f96ce8ce634 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 3 Jan 2019 10:24:35 -0600 -Subject: [PATCH 16/69] Build: move project maintenance helpers to new - subdirectory - -so their purpose is clear ---- - GNUmakefile | 1 - - bumplibs.sh | 128 ---------------------------------- - maint/README | 2 + - maint/bumplibs.sh | 128 ++++++++++++++++++++++++++++++++++ - maint/travisci_build_coverity_scan.sh | 102 +++++++++++++++++++++++++++ - 8 files changed, 234 insertions(+), 233 deletions(-) - delete mode 100755 bumplibs.sh - create mode 100644 maint/README - create mode 100755 maint/bumplibs.sh - create mode 100644 maint/travisci_build_coverity_scan.sh - -diff --git a/GNUmakefile b/GNUmakefile -index 8e5bdd7..60de623 100644 ---- a/GNUmakefile -+++ b/GNUmakefile -@@ -347,7 +347,6 @@ changelog: - @make changes > ChangeLog - @printf "\n">> ChangeLog - git show $(LAST_RELEASE):ChangeLog >> ChangeLog -- @echo -e "\033[1;35m -- Don't forget to run the bumplibs.sh script! --\033[0m" - - DO_NOT_INDENT = lib/gnu daemons/controld/controld_fsa.h - -diff --git a/bumplibs.sh b/bumplibs.sh -deleted file mode 100755 -index 2044efa..0000000 ---- a/bumplibs.sh -+++ /dev/null -@@ -1,128 +0,0 @@ --#!/bin/bash -- --declare -A headers --headers[crmcommon]="include/crm/common include/crm/crm.h" --headers[crmcluster]="include/crm/cluster.h" --headers[crmservice]="include/crm/services.h" --headers[transitioner]="include/crm/transition.h" --headers[cib]="include/crm/cib.h include/crm/cib/util.h" --headers[pe_rules]="include/crm/pengine/rules.h" --headers[pe_status]="include/crm/pengine/common.h include/crm/pengine/complex.h include/crm/pengine/rules.h include/crm/pengine/status.h" --headers[pengine]="include/crm/pengine/common.h include/crm/pengine/complex.h include/crm/pengine/rules.h include/crm/pengine/status.h" --headers[stonithd]="include/crm/stonith-ng.h" --headers[lrmd]="include/crm/lrmd.h" -- --if [ ! -z $1 ]; then -- LAST_RELEASE=$1 --else -- LAST_RELEASE=`test -e /Volumes || git tag -l | grep Pacemaker | grep -v rc | sort -Vr | head -n 1` --fi --libs=$(find . -name "*.am" -exec grep "lib.*_la_LDFLAGS.*version-info" \{\} \; | sed -e s/_la_LDFLAGS.*// -e s/^lib//) --for lib in $libs; do -- if [ -z "${headers[$lib]}" ]; then -- echo "Unknown headers for lib$lib" -- exit 0 -- fi -- git diff -w $LAST_RELEASE..HEAD ${headers[$lib]} -- echo "" -- -- am=`find . -name Makefile.am -exec grep -lr "lib${lib}_la.*version-info" \{\} \;` -- am_dir=`dirname $am` -- -- if -- grep "lib${lib}_la_SOURCES.*\\\\" $am -- then -- echo -e "\033[1;35m -- Sources list for lib$lib is probably truncated! --\033[0m" -- echo "" -- fi -- -- sources=`grep "lib${lib}_la_SOURCES" $am | sed s/.*=// | sed 's:$(top_builddir)/::' | sed 's:$(top_srcdir)/::' | sed 's:\\\::' | sed 's:$(libpe_rules_la_SOURCES):rules.c\ common.c:'` -- -- full_sources="" -- for f in $sources; do -- if -- echo $f | grep -q "/" -- then -- full_sources="$full_sources $f" -- else -- full_sources="$full_sources $am_dir/$f" -- fi -- done -- -- lines=`git diff -w $LAST_RELEASE..HEAD ${headers[$lib]} $full_sources | wc -l` -- -- if [ $lines -gt 0 ]; then -- echo "- Headers: ${headers[$lib]}" -- echo "- Sources: $full_sources" -- echo "- Changed Sources since $LAST_RELEASE:" -- git diff -w $LAST_RELEASE..HEAD --stat $full_sources -- echo "" -- echo "New arguments to functions or changes to the middle of structs are incompatible additions" -- echo "" -- echo "Where possible:" -- echo "- move new fields to the end of structs" -- echo "- use bitfields instead of booleans" -- echo "- when adding arguments, create new functions that the old version can call" -- echo "" -- read -p "Are the changes to lib$lib: [a]dditions, [i]ncompatible additions, [r]emovals or [f]ixes? [None]: " CHANGE -- -- git show $LAST_RELEASE:$am | grep version-info -- VER=`git show $LAST_RELEASE:$am | grep "lib.*${lib}_la.*version-info" | sed s/.*version-info// | awk '{print $1}'` -- VER_NOW=`grep "lib.*${lib}_la.*version-info" $am | sed s/.*version-info// | awk '{print $1}'` -- VER_1=`echo $VER | awk -F: '{print $1}'` -- VER_2=`echo $VER | awk -F: '{print $2}'` -- VER_3=`echo $VER | awk -F: '{print $3}'` -- VER_1_NOW=`echo $VER_NOW | awk -F: '{print $1}'` -- -- case $CHANGE in -- i|I) -- echo "New version with incompatible extensions: x+1:0:0" -- VER_1=`expr $VER_1 + 1` -- VER_2=0 -- VER_3=0 -- for h in ${headers[$lib]}; do -- sed -i.sed "s/lib${lib}.so.${VER_1_NOW}/lib${lib}.so.${VER_1}/" $h -- done -- ;; -- a|A) -- echo "New version with backwards compatible extensions: x+1:0:z+1" -- VER_1=`expr $VER_1 + 1` -- VER_2=0 -- VER_3=`expr $VER_3 + 1` -- ;; -- R|r) -- echo "New backwards incompatible version: x+1:0:0" -- VER_1=`expr $VER_1 + 1` -- VER_2=0 -- VER_3=0 -- for h in ${headers[$lib]}; do -- sed -i.sed "s/lib${lib}.so.${VER_1_NOW}/lib${lib}.so.${VER_1}/" $h -- done -- ;; -- F|f) -- echo "Bugfix: x:y+1:z" -- VER_2=`expr $VER_2 + 1` -- ;; -- esac -- VER_NEW=$VER_1:$VER_2:$VER_3 -- -- if [ ! -z $CHANGE ]; then -- if [ $VER_NEW != $VER_NOW ]; then -- echo "Updating $lib library version: $VER -> $VER_NEW" -- sed -i.sed "s/version-info\ $VER_NOW/version-info\ $VER_NEW/" $am -- else -- echo "No further version changes needed" -- sed -i.sed "s/version-info\ $VER_NOW/version-info\ $VER_NEW/" $am -- fi -- else -- echo "Skipping $lib version" -- fi -- else -- echo "No changes to $lib interface" -- fi -- -- read -p "Continue?" -- echo "" --done -- --git diff -w -diff --git a/maint/README b/maint/README -new file mode 100644 -index 0000000..738e7db ---- /dev/null -+++ b/maint/README -@@ -0,0 +1,2 @@ -+This directory contains helpers for the project maintainers. -+Everyone else can safely ignore it. -diff --git a/maint/bumplibs.sh b/maint/bumplibs.sh -new file mode 100755 -index 0000000..2044efa ---- /dev/null -+++ b/maint/bumplibs.sh -@@ -0,0 +1,128 @@ -+#!/bin/bash -+ -+declare -A headers -+headers[crmcommon]="include/crm/common include/crm/crm.h" -+headers[crmcluster]="include/crm/cluster.h" -+headers[crmservice]="include/crm/services.h" -+headers[transitioner]="include/crm/transition.h" -+headers[cib]="include/crm/cib.h include/crm/cib/util.h" -+headers[pe_rules]="include/crm/pengine/rules.h" -+headers[pe_status]="include/crm/pengine/common.h include/crm/pengine/complex.h include/crm/pengine/rules.h include/crm/pengine/status.h" -+headers[pengine]="include/crm/pengine/common.h include/crm/pengine/complex.h include/crm/pengine/rules.h include/crm/pengine/status.h" -+headers[stonithd]="include/crm/stonith-ng.h" -+headers[lrmd]="include/crm/lrmd.h" -+ -+if [ ! -z $1 ]; then -+ LAST_RELEASE=$1 -+else -+ LAST_RELEASE=`test -e /Volumes || git tag -l | grep Pacemaker | grep -v rc | sort -Vr | head -n 1` -+fi -+libs=$(find . -name "*.am" -exec grep "lib.*_la_LDFLAGS.*version-info" \{\} \; | sed -e s/_la_LDFLAGS.*// -e s/^lib//) -+for lib in $libs; do -+ if [ -z "${headers[$lib]}" ]; then -+ echo "Unknown headers for lib$lib" -+ exit 0 -+ fi -+ git diff -w $LAST_RELEASE..HEAD ${headers[$lib]} -+ echo "" -+ -+ am=`find . -name Makefile.am -exec grep -lr "lib${lib}_la.*version-info" \{\} \;` -+ am_dir=`dirname $am` -+ -+ if -+ grep "lib${lib}_la_SOURCES.*\\\\" $am -+ then -+ echo -e "\033[1;35m -- Sources list for lib$lib is probably truncated! --\033[0m" -+ echo "" -+ fi -+ -+ sources=`grep "lib${lib}_la_SOURCES" $am | sed s/.*=// | sed 's:$(top_builddir)/::' | sed 's:$(top_srcdir)/::' | sed 's:\\\::' | sed 's:$(libpe_rules_la_SOURCES):rules.c\ common.c:'` -+ -+ full_sources="" -+ for f in $sources; do -+ if -+ echo $f | grep -q "/" -+ then -+ full_sources="$full_sources $f" -+ else -+ full_sources="$full_sources $am_dir/$f" -+ fi -+ done -+ -+ lines=`git diff -w $LAST_RELEASE..HEAD ${headers[$lib]} $full_sources | wc -l` -+ -+ if [ $lines -gt 0 ]; then -+ echo "- Headers: ${headers[$lib]}" -+ echo "- Sources: $full_sources" -+ echo "- Changed Sources since $LAST_RELEASE:" -+ git diff -w $LAST_RELEASE..HEAD --stat $full_sources -+ echo "" -+ echo "New arguments to functions or changes to the middle of structs are incompatible additions" -+ echo "" -+ echo "Where possible:" -+ echo "- move new fields to the end of structs" -+ echo "- use bitfields instead of booleans" -+ echo "- when adding arguments, create new functions that the old version can call" -+ echo "" -+ read -p "Are the changes to lib$lib: [a]dditions, [i]ncompatible additions, [r]emovals or [f]ixes? [None]: " CHANGE -+ -+ git show $LAST_RELEASE:$am | grep version-info -+ VER=`git show $LAST_RELEASE:$am | grep "lib.*${lib}_la.*version-info" | sed s/.*version-info// | awk '{print $1}'` -+ VER_NOW=`grep "lib.*${lib}_la.*version-info" $am | sed s/.*version-info// | awk '{print $1}'` -+ VER_1=`echo $VER | awk -F: '{print $1}'` -+ VER_2=`echo $VER | awk -F: '{print $2}'` -+ VER_3=`echo $VER | awk -F: '{print $3}'` -+ VER_1_NOW=`echo $VER_NOW | awk -F: '{print $1}'` -+ -+ case $CHANGE in -+ i|I) -+ echo "New version with incompatible extensions: x+1:0:0" -+ VER_1=`expr $VER_1 + 1` -+ VER_2=0 -+ VER_3=0 -+ for h in ${headers[$lib]}; do -+ sed -i.sed "s/lib${lib}.so.${VER_1_NOW}/lib${lib}.so.${VER_1}/" $h -+ done -+ ;; -+ a|A) -+ echo "New version with backwards compatible extensions: x+1:0:z+1" -+ VER_1=`expr $VER_1 + 1` -+ VER_2=0 -+ VER_3=`expr $VER_3 + 1` -+ ;; -+ R|r) -+ echo "New backwards incompatible version: x+1:0:0" -+ VER_1=`expr $VER_1 + 1` -+ VER_2=0 -+ VER_3=0 -+ for h in ${headers[$lib]}; do -+ sed -i.sed "s/lib${lib}.so.${VER_1_NOW}/lib${lib}.so.${VER_1}/" $h -+ done -+ ;; -+ F|f) -+ echo "Bugfix: x:y+1:z" -+ VER_2=`expr $VER_2 + 1` -+ ;; -+ esac -+ VER_NEW=$VER_1:$VER_2:$VER_3 -+ -+ if [ ! -z $CHANGE ]; then -+ if [ $VER_NEW != $VER_NOW ]; then -+ echo "Updating $lib library version: $VER -> $VER_NEW" -+ sed -i.sed "s/version-info\ $VER_NOW/version-info\ $VER_NEW/" $am -+ else -+ echo "No further version changes needed" -+ sed -i.sed "s/version-info\ $VER_NOW/version-info\ $VER_NEW/" $am -+ fi -+ else -+ echo "Skipping $lib version" -+ fi -+ else -+ echo "No changes to $lib interface" -+ fi -+ -+ read -p "Continue?" -+ echo "" -+done -+ -+git diff -w -diff --git a/maint/travisci_build_coverity_scan.sh b/maint/travisci_build_coverity_scan.sh -new file mode 100644 -index 0000000..b8b24d2 ---- /dev/null -+++ b/maint/travisci_build_coverity_scan.sh -@@ -0,0 +1,102 @@ -+#!/bin/sh -+ -+set -e -+ -+export RED="\033[33;1m" -+export NONE="\033[0m" -+ -+if [ -z "$PROJECT_NAME" ]; then -+ PROJECT_NAME=${TRAVIS_REPO_SLUG} -+fi -+ -+# Environment check -+echo -e "${RED}Note: PROJECT_NAME and COVERITY_SCAN_TOKEN are available on Project Settings page on scan.coverity.com${NONE}" -+[ -z "$PROJECT_NAME" ] && echo "ERROR: PROJECT_NAME must be set" && exit 1 -+[ -z "$OWNER_EMAIL" ] && echo "ERROR: OWNER_EMAIL must be set" && exit 1 -+[ -z "$COVERITY_SCAN_BRANCH_PATTERN" ] && echo "ERROR: COVERITY_SCAN_BRANCH_PATTERN must be set" && exit 1 -+[ -z "$COVERITY_SCAN_BUILD_COMMAND" ] && echo "ERROR: COVERITY_SCAN_BUILD_COMMAND must be set" && exit 1 -+ -+PLATFORM=`uname` -+TOOL_ARCHIVE=/tmp/cov-analysis-${PLATFORM}.tgz -+TOOL_URL=https://scan.coverity.com/download/${PLATFORM} -+TOOL_BASE=/tmp/coverity-scan-analysis -+UPLOAD_URL="http://scan5.coverity.com/cgi-bin/upload.py" -+SCAN_URL="https://scan.coverity.com" -+ -+# Do not run on pull requests -+if [ "${TRAVIS_PULL_REQUEST}" = "true" ]; then -+ echo -e "${RED}INFO: Skipping Coverity Analysis: branch is a pull request.${NONE}" -+ exit 0 -+fi -+ -+# Verify this branch should run -+IS_COVERITY_SCAN_BRANCH=`ruby -e "puts '${TRAVIS_BRANCH}' =~ /\\A$COVERITY_SCAN_BRANCH_PATTERN\\z/ ? 1 : 0"` -+if [ "$IS_COVERITY_SCAN_BRANCH" = "1" ]; then -+ echo -e "${RED}Coverity Scan configured to run on branch ${TRAVIS_BRANCH}${NONE}" -+else -+ echo -e "${RED}Coverity Scan NOT configured to run on branch ${TRAVIS_BRANCH}${NONE}" -+ exit 0 # Nothing to do, exit with success otherwise the build will be considered failed -+fi -+ -+# If COVERITY_SCAN_TOKEN isn't set, then we're probably running from somewhere -+# other than ClusterLabs/pacemaker and coverity shouldn't be running anyway -+[ -z "$COVERITY_SCAN_TOKEN" ] && echo "${RED}ERROR: COVERITY_SCAN_TOKEN must be set${NONE}" && exit 0 -+ -+# Verify upload is permitted -+AUTH_RES=`curl -s --form project="$PROJECT_NAME" --form token="$COVERITY_SCAN_TOKEN" $SCAN_URL/api/upload_permitted` -+if [ "$AUTH_RES" = "Access denied" ]; then -+ echo -e "${RED}Coverity Scan API access denied. Check PROJECT_NAME and COVERITY_SCAN_TOKEN.${NONE}" -+ exit 1 -+else -+ AUTH=`echo $AUTH_RES | ruby -e "require 'rubygems'; require 'json'; puts JSON[STDIN.read]['upload_permitted']"` -+ if [ "$AUTH" = "true" ]; then -+ echo -e "${RED}Coverity Scan analysis authorized per quota.${NONE}" -+ else -+ WHEN=`echo $AUTH_RES | ruby -e "require 'rubygems'; require 'json'; puts JSON[STDIN.read]['next_upload_permitted_at']"` -+ echo -e "${RED}Coverity Scan analysis NOT authorized until $WHEN.${NONE}" -+ exit 1 -+ fi -+fi -+ -+if [ ! -d $TOOL_BASE ]; then -+ # Download Coverity Scan Analysis Tool -+ if [ ! -e $TOOL_ARCHIVE ]; then -+ echo -e "${RED}Downloading Coverity Scan Analysis Tool...${NONE}" -+ wget -nv -O $TOOL_ARCHIVE $TOOL_URL --post-data "project=$PROJECT_NAME&token=$COVERITY_SCAN_TOKEN" -+ fi -+ -+ # Extract Coverity Scan Analysis Tool -+ echo -e "${RED}Extracting Coverity Scan Analysis Tool...${NONE}" -+ mkdir -p $TOOL_BASE -+ pushd $TOOL_BASE -+ tar xzf $TOOL_ARCHIVE -+ popd -+fi -+ -+TOOL_DIR=`find $TOOL_BASE -type d -name 'cov-analysis*'` -+export PATH=$TOOL_DIR/bin:$PATH -+ -+# Build -+echo -e "${RED}Running Coverity Scan Analysis Tool...${NONE}" -+COV_BUILD_OPTIONS="" -+#COV_BUILD_OPTIONS="--return-emit-failures 8 --parse-error-threshold 85" -+RESULTS_DIR="cov-int" -+eval "${COVERITY_SCAN_BUILD_COMMAND_PREPEND}" -+COVERITY_UNSUPPORTED=1 cov-build --dir $RESULTS_DIR $COV_BUILD_OPTIONS $COVERITY_SCAN_BUILD_COMMAND -+ -+# Upload results -+echo -e "${RED}Tarring Coverity Scan Analysis results...${NONE}" -+RESULTS_ARCHIVE=analysis-results.tgz -+tar czf $RESULTS_ARCHIVE $RESULTS_DIR -+SHA=`git rev-parse --short HEAD` -+ -+echo -e "${RED}Uploading Coverity Scan Analysis results...${NONE}" -+curl \ -+ --progress-bar \ -+ --form project=$PROJECT_NAME \ -+ --form token=$COVERITY_SCAN_TOKEN \ -+ --form email=$OWNER_EMAIL \ -+ --form file=@$RESULTS_ARCHIVE \ -+ --form version=$SHA \ -+ --form description="Travis CI build" \ -+ $UPLOAD_URL --- -1.8.3.1 - - -From d63ced718d0d9f28f711a234307ec534cb76d7fa Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 20 Dec 2018 16:26:01 -0600 -Subject: [PATCH 17/69] Doc: ChangeLog: podman support was added in 2.0.1-rc1 - ---- - ChangeLog | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/ChangeLog b/ChangeLog -index 23d8307..e348333 100644 ---- a/ChangeLog -+++ b/ChangeLog -@@ -11,6 +11,7 @@ - 164 files changed, 7717 insertions(+), 4398 deletions(-) - - - Features added since Pacemaker-2.0.0 -+ + Pacemaker bundles now support podman container management - + fencing: SBD may now be used in a cluster that has guest nodes or bundles - + fencing: synchronize fencing history among all nodes - + fencing: stonith_admin now has option to clear fence history --- -1.8.3.1 - - -From 8b58ea39391e876647ff76d81ac55d05b7cd8880 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 2 Oct 2018 15:26:23 -0500 -Subject: [PATCH 18/69] Low: attrd: connect to the CIB before connecting the - cluster - -This avoids a (highly unlikely) race: If the local node wins a writer election -and receives a peer update at start-up, between the cluster connection and the -CIB connection, we could delay the write due to "cib not connected". However, -we would not re-attempt this write once the connection was established, so the -value might never be written out (or delayed an unreasonably long time). - -Rearranging the connections is a simple solution, and also allows us to assume -the CIB is connected everywhere else. ---- - daemons/attrd/attrd_alerts.c | 23 ++++++++---------- - daemons/attrd/attrd_commands.c | 7 ++---- - daemons/attrd/attrd_utils.c | 9 +++---- - daemons/attrd/pacemaker-attrd.c | 54 ++++++++++++++++++++++++++--------------- - 4 files changed, 50 insertions(+), 43 deletions(-) - -diff --git a/daemons/attrd/attrd_alerts.c b/daemons/attrd/attrd_alerts.c -index 377e27a..e63d635 100644 ---- a/daemons/attrd/attrd_alerts.c -+++ b/daemons/attrd/attrd_alerts.c -@@ -111,19 +111,16 @@ attrd_read_options(gpointer user_data) - { - int call_id; - -- if (the_cib) { -- call_id = the_cib->cmds->query(the_cib, XPATH_ALERTS, NULL, -- cib_xpath | cib_scope_local); -- -- the_cib->cmds->register_callback_full(the_cib, call_id, 120, FALSE, -- NULL, -- "config_query_callback", -- config_query_callback, free); -- -- crm_trace("Querying the CIB... call %d", call_id); -- } else { -- crm_err("Could not check for alerts configuration: CIB connection not active"); -- } -+ CRM_CHECK(the_cib != NULL, return TRUE); -+ -+ call_id = the_cib->cmds->query(the_cib, XPATH_ALERTS, NULL, -+ cib_xpath | cib_scope_local); -+ -+ the_cib->cmds->register_callback_full(the_cib, call_id, 120, FALSE, NULL, -+ "config_query_callback", -+ config_query_callback, free); -+ -+ crm_trace("Querying the CIB... call %d", call_id); - return TRUE; - } - -diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c -index f07d503..1cd7aaf 100644 ---- a/daemons/attrd/attrd_commands.c -+++ b/daemons/attrd/attrd_commands.c -@@ -1163,11 +1163,8 @@ write_attribute(attribute_t *a, bool ignore_delay) - if (!a->is_private) { - - /* Defer the write if now's not a good time */ -- if (the_cib == NULL) { -- crm_info("Write out of '%s' delayed: cib not connected", a->id); -- return; -- -- } else if (a->update && (a->update < last_cib_op_done)) { -+ CRM_CHECK(the_cib != NULL, return); -+ if (a->update && (a->update < last_cib_op_done)) { - crm_info("Write out of '%s' continuing: update %d considered lost", a->id, a->update); - - } else if (a->update) { -diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c -index 9ce784c..8245af6 100644 ---- a/daemons/attrd/attrd_utils.c -+++ b/daemons/attrd/attrd_utils.c -@@ -175,11 +175,10 @@ attrd_init_ipc(qb_ipcs_service_t **ipcs, qb_ipcs_msg_process_fn dispatch_fn) - void - attrd_cib_disconnect() - { -- if (the_cib) { -- the_cib->cmds->signoff(the_cib); -- cib_delete(the_cib); -- the_cib = NULL; -- } -+ CRM_CHECK(the_cib != NULL, return); -+ the_cib->cmds->signoff(the_cib); -+ cib_delete(the_cib); -+ the_cib = NULL; - } - - /* strlen("value") */ -diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c -index d96e666..69d02c0 100644 ---- a/daemons/attrd/pacemaker-attrd.c -+++ b/daemons/attrd/pacemaker-attrd.c -@@ -198,6 +198,22 @@ attrd_cib_connect(int max_retry) - goto cleanup; - } - -+ return pcmk_ok; -+ -+ cleanup: -+ the_cib->cmds->signoff(the_cib); -+ cib_delete(the_cib); -+ the_cib = NULL; -+ return -ENOTCONN; -+} -+ -+/*! -+ * \internal -+ * \brief Prepare the CIB after cluster is connected -+ */ -+static void -+attrd_cib_init() -+{ - // We have no attribute values in memory, wipe the CIB to match - attrd_erase_attrs(); - -@@ -206,21 +222,6 @@ attrd_cib_connect(int max_retry) - - // Always read the CIB at start-up - mainloop_set_trigger(attrd_config_read); -- -- /* Set a private attribute for ourselves with the protocol version we -- * support. This lets all nodes determine the minimum supported version -- * across all nodes. It also ensures that the writer learns our node name, -- * so it can send our attributes to the CIB. -- */ -- attrd_broadcast_protocol(); -- -- return pcmk_ok; -- -- cleanup: -- the_cib->cmds->signoff(the_cib); -- cib_delete(the_cib); -- the_cib = NULL; -- return -ENOTCONN; - } - - static int32_t -@@ -361,19 +362,32 @@ main(int argc, char **argv) - crm_info("Starting up"); - attributes = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_attribute); - -+ /* Connect to the CIB before connecting to the cluster or listening for IPC. -+ * This allows us to assume the CIB is connected whenever we process a -+ * cluster or IPC message (which also avoids start-up race conditions). -+ */ -+ if (attrd_cib_connect(10) != pcmk_ok) { -+ attrd_exit_status = CRM_EX_FATAL; -+ goto done; -+ } -+ crm_info("CIB connection active"); -+ - if (attrd_cluster_connect() != pcmk_ok) { - attrd_exit_status = CRM_EX_FATAL; - goto done; - } - crm_info("Cluster connection active"); - -+ // Initialization that requires the cluster to be connected - attrd_election_init(); -+ attrd_cib_init(); - -- if (attrd_cib_connect(10) != pcmk_ok) { -- attrd_exit_status = CRM_EX_FATAL; -- goto done; -- } -- crm_info("CIB connection active"); -+ /* Set a private attribute for ourselves with the protocol version we -+ * support. This lets all nodes determine the minimum supported version -+ * across all nodes. It also ensures that the writer learns our node name, -+ * so it can send our attributes to the CIB. -+ */ -+ attrd_broadcast_protocol(); - - attrd_init_ipc(&ipcs, attrd_ipc_dispatch); - crm_info("Accepting attribute updates"); --- -1.8.3.1 - - -From 61ae92650c4c3065e5a7dc92e4d1c806d3f24f9e Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 3 Oct 2018 16:50:12 -0500 -Subject: [PATCH 19/69] Low: attrd: don't delay re-attempted writes unless - original failed - -73e5b6d3 was too aggressive in delaying re-attempted writes; a write could be -re-attempted because the value changed since the original write was submitted. -Limit the delay to cases where the original write failed. - -Also, lower a trivial debug log to trace. ---- - daemons/attrd/attrd_commands.c | 12 +++++++++--- - 1 file changed, 9 insertions(+), 3 deletions(-) - -diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c -index 1cd7aaf..fa0459a 100644 ---- a/daemons/attrd/attrd_commands.c -+++ b/daemons/attrd/attrd_commands.c -@@ -1018,7 +1018,13 @@ attrd_cib_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *u - } - - if (a->changed && attrd_election_won()) { -- /* If we're re-attempting a write because the original failed, delay -+ if (rc == pcmk_ok) { -+ /* We deferred a write of a new update because this update was in -+ * progress. Write out the new value without additional delay. -+ */ -+ write_attribute(a, FALSE); -+ -+ /* We're re-attempting a write because the original failed; delay - * the next attempt so we don't potentially flood the CIB manager - * and logs with a zillion attempts per second. - * -@@ -1027,7 +1033,7 @@ attrd_cib_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *u - * if all peers similarly fail to write this attribute (which may - * indicate a corrupted attribute entry rather than a CIB issue). - */ -- if (a->timer) { -+ } else if (a->timer) { - // Attribute has a dampening value, so use that as delay - if (!mainloop_timer_running(a->timer)) { - crm_trace("Delayed re-attempted write (%dms) for %s", -@@ -1067,7 +1073,7 @@ write_attributes(bool all, bool ignore_delay) - /* When forced write flag is set, ignore delay. */ - write_attribute(a, (a->force_write ? TRUE : ignore_delay)); - } else { -- crm_debug("Skipping unchanged attribute %s", a->id); -+ crm_trace("Skipping unchanged attribute %s", a->id); - } - } - } --- -1.8.3.1 - - -From 19b18a549013e7ce081d0874c92fdbb6885e0e2c Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 18 Oct 2018 09:45:03 -0500 -Subject: [PATCH 20/69] Log: attrd: clear lost updates - -If an attribute's CIB update is determined to be lost, set the update ID to 0, -to avoid logging the same message repeatedly in the (unlikely) chance that the -a new write couldn't be scheduled (e.g. if the peer UUIDs are unknown for the -values that need to be written). ---- - daemons/attrd/attrd_commands.c | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c -index fa0459a..8e81a66 100644 ---- a/daemons/attrd/attrd_commands.c -+++ b/daemons/attrd/attrd_commands.c -@@ -1172,6 +1172,7 @@ write_attribute(attribute_t *a, bool ignore_delay) - CRM_CHECK(the_cib != NULL, return); - if (a->update && (a->update < last_cib_op_done)) { - crm_info("Write out of '%s' continuing: update %d considered lost", a->id, a->update); -+ a->update = 0; // Don't log this message again - - } else if (a->update) { - crm_info("Write out of '%s' delayed: update %d in progress", a->id, a->update); --- -1.8.3.1 - - -From e1ff25d0d637664ad8954121f0b9a530eff1a5f9 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 3 Oct 2018 16:52:48 -0500 -Subject: [PATCH 21/69] Low: attrd: don't start a new election when receiving a - client update - -We already start an election (if needed) when an attribute needs to be written. ---- - daemons/attrd/attrd_commands.c | 2 -- - 1 file changed, 2 deletions(-) - -diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c -index 8e81a66..6daacba 100644 ---- a/daemons/attrd/attrd_commands.c -+++ b/daemons/attrd/attrd_commands.c -@@ -297,8 +297,6 @@ attrd_client_update(xmlNode *xml) - } - } - -- attrd_start_election_if_needed(); -- - crm_debug("Broadcasting %s[%s]=%s%s", attr, host, value, - (attrd_election_won()? " (writer)" : "")); - --- -1.8.3.1 - - -From 546f9f255a0ee53453927fa32847541b1f36c6f1 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 18 Oct 2018 13:44:02 -0500 -Subject: [PATCH 22/69] Refactor: attrd: use defined constant for CIB op - timeout - -for readability and possible future reuse ---- - daemons/attrd/attrd_commands.c | 3 ++- - daemons/attrd/pacemaker-attrd.h | 2 ++ - 2 files changed, 4 insertions(+), 1 deletion(-) - -diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c -index 6daacba..4caf76c 100644 ---- a/daemons/attrd/attrd_commands.c -+++ b/daemons/attrd/attrd_commands.c -@@ -1277,7 +1277,8 @@ write_attribute(attribute_t *a, bool ignore_delay) - a->update, cib_updates, s_if_plural(cib_updates), - a->id, (a->uuid? a->uuid : "n/a"), (a->set? a->set : "n/a")); - -- the_cib->cmds->register_callback_full(the_cib, a->update, 120, FALSE, -+ the_cib->cmds->register_callback_full(the_cib, a->update, -+ CIB_OP_TIMEOUT_S, FALSE, - strdup(a->id), - "attrd_cib_callback", - attrd_cib_callback, free); -diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h -index bc4947b..2276c73 100644 ---- a/daemons/attrd/pacemaker-attrd.h -+++ b/daemons/attrd/pacemaker-attrd.h -@@ -111,6 +111,8 @@ GHashTable *attributes; - #define attrd_send_ack(client, id, flags) \ - crm_ipcs_send_ack((client), (id), (flags), "ack", __FUNCTION__, __LINE__) - -+#define CIB_OP_TIMEOUT_S 120 -+ - void write_attributes(bool all, bool ignore_delay); - void attrd_broadcast_protocol(void); - void attrd_peer_message(crm_node_t *client, xmlNode *msg); --- -1.8.3.1 - - -From 677f6680526e2636646127f36a3d049e3e81e64b Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 18 Oct 2018 13:48:12 -0500 -Subject: [PATCH 23/69] Refactor: attrd: functionize closing IPC server - -for readability, isolation, and possible future reuse ---- - daemons/attrd/pacemaker-attrd.c | 21 ++++++++++++++------- - daemons/attrd/pacemaker-attrd.h | 1 + - 2 files changed, 15 insertions(+), 7 deletions(-) - -diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c -index 69d02c0..4088938 100644 ---- a/daemons/attrd/pacemaker-attrd.c -+++ b/daemons/attrd/pacemaker-attrd.c -@@ -224,6 +224,8 @@ attrd_cib_init() - mainloop_set_trigger(attrd_config_read); - } - -+static qb_ipcs_service_t *ipcs = NULL; -+ - static int32_t - attrd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size) - { -@@ -289,6 +291,16 @@ attrd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size) - return 0; - } - -+void -+attrd_ipc_fini() -+{ -+ if (ipcs != NULL) { -+ crm_client_disconnect_all(ipcs); -+ qb_ipcs_destroy(ipcs); -+ ipcs = NULL; -+ } -+} -+ - static int - attrd_cluster_connect() - { -@@ -323,7 +335,6 @@ main(int argc, char **argv) - int flag = 0; - int index = 0; - int argerr = 0; -- qb_ipcs_service_t *ipcs = NULL; - - attrd_init_mainloop(); - crm_log_preinit(NULL, argc, argv); -@@ -397,14 +408,10 @@ main(int argc, char **argv) - crm_info("Shutting down attribute manager"); - - attrd_election_fini(); -- if (ipcs) { -- crm_client_disconnect_all(ipcs); -- qb_ipcs_destroy(ipcs); -- g_hash_table_destroy(attributes); -- } -- -+ attrd_ipc_fini(); - attrd_lrmd_disconnect(); - attrd_cib_disconnect(); -+ g_hash_table_destroy(attributes); - - return crm_exit(attrd_exit_status); - } -diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h -index 2276c73..cc8e29e 100644 ---- a/daemons/attrd/pacemaker-attrd.h -+++ b/daemons/attrd/pacemaker-attrd.h -@@ -22,6 +22,7 @@ gboolean attrd_shutting_down(void); - void attrd_shutdown(int nsig); - void attrd_init_ipc(qb_ipcs_service_t **ipcs, - qb_ipcs_msg_process_fn dispatch_fn); -+void attrd_ipc_fini(void); - - void attrd_cib_disconnect(void); - --- -1.8.3.1 - - -From 6ddb87f6a364bd5c3be651d0ede0ec1c8f48666c Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 18 Oct 2018 14:33:24 -0500 -Subject: [PATCH 24/69] Low: attrd: handle shutdown more cleanly - -Once shutdown has started, avoid wasting time on such things as updating the -alert configuration, responding to most peer messages, starting a new election, -or writing out attributes after a CIB replace. - -This doesn't really matter much since shutdown is self-contained at the moment. -There were plans to change that, but they wound up being unnecessary. These -changes still seem worthwhile, though. ---- - daemons/attrd/attrd_alerts.c | 2 +- - daemons/attrd/attrd_commands.c | 8 ++++++++ - daemons/attrd/attrd_elections.c | 9 +++++++-- - daemons/attrd/attrd_utils.c | 25 +++++++++++++++++++++---- - daemons/attrd/pacemaker-attrd.c | 6 +++++- - 5 files changed, 42 insertions(+), 8 deletions(-) - -diff --git a/daemons/attrd/attrd_alerts.c b/daemons/attrd/attrd_alerts.c -index e63d635..611f270 100644 ---- a/daemons/attrd/attrd_alerts.c -+++ b/daemons/attrd/attrd_alerts.c -@@ -127,7 +127,7 @@ attrd_read_options(gpointer user_data) - void - attrd_cib_updated_cb(const char *event, xmlNode * msg) - { -- if (crm_patchset_contains_alert(msg, FALSE)) { -+ if (!attrd_shutting_down() && crm_patchset_contains_alert(msg, FALSE)) { - mainloop_set_trigger(attrd_config_read); - } - } -diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c -index 4caf76c..64ab9f1 100644 ---- a/daemons/attrd/attrd_commands.c -+++ b/daemons/attrd/attrd_commands.c -@@ -569,6 +569,14 @@ attrd_peer_message(crm_node_t *peer, xmlNode *xml) - return; - } - -+ if (attrd_shutting_down()) { -+ /* If we're shutting down, we want to continue responding to election -+ * ops as long as we're a cluster member (because our vote may be -+ * needed). Ignore all other messages. -+ */ -+ return; -+ } -+ - peer_won = attrd_check_for_new_writer(peer, xml); - - if (safe_str_eq(op, ATTRD_OP_UPDATE) || safe_str_eq(op, ATTRD_OP_UPDATE_BOTH) || safe_str_eq(op, ATTRD_OP_UPDATE_DELAY)) { -diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c -index 05e1d84..cfe2d89 100644 ---- a/daemons/attrd/attrd_elections.c -+++ b/daemons/attrd/attrd_elections.c -@@ -32,7 +32,9 @@ void - attrd_start_election_if_needed() - { - if ((peer_writer == NULL) -- && (election_state(writer) != election_in_progress)) { -+ && (election_state(writer) != election_in_progress) -+ && !attrd_shutting_down()) { -+ - crm_info("Starting an election to determine the writer"); - election_vote(writer); - } -@@ -51,7 +53,10 @@ attrd_handle_election_op(const crm_node_t *peer, xmlNode *xml) - enum election_result previous = election_state(writer); - - crm_xml_add(xml, F_CRM_HOST_FROM, peer->uname); -- rc = election_count_vote(writer, xml, TRUE); -+ -+ // Don't become writer if we're shutting down -+ rc = election_count_vote(writer, xml, !attrd_shutting_down()); -+ - switch(rc) { - case election_start: - free(peer_writer); -diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c -index 8245af6..6096dcc 100644 ---- a/daemons/attrd/attrd_utils.c -+++ b/daemons/attrd/attrd_utils.c -@@ -8,6 +8,7 @@ - #include - - #include -+#include - #include - #include - #include -@@ -21,7 +22,9 @@ - - cib_t *the_cib = NULL; - --static gboolean shutting_down = FALSE; -+// volatile because attrd_shutdown() can be called for a signal -+static volatile bool shutting_down = FALSE; -+ - static GMainLoop *mloop = NULL; - - /*! -@@ -45,11 +48,25 @@ attrd_shutting_down() - void - attrd_shutdown(int nsig) - { -+ // Tell various functions not to do anthing - shutting_down = TRUE; -- if ((mloop != NULL) && g_main_loop_is_running(mloop)) { -- g_main_loop_quit(mloop); -- } else { -+ -+ // Don't respond to signals while shutting down -+ mainloop_destroy_signal(SIGTERM); -+ mainloop_destroy_signal(SIGCHLD); -+ mainloop_destroy_signal(SIGPIPE); -+ mainloop_destroy_signal(SIGUSR1); -+ mainloop_destroy_signal(SIGUSR2); -+ mainloop_destroy_signal(SIGTRAP); -+ -+ if ((mloop == NULL) || !g_main_loop_is_running(mloop)) { -+ /* If there's no main loop active, just exit. This should be possible -+ * only if we get SIGTERM in brief windows at start-up and shutdown. -+ */ - crm_exit(CRM_EX_OK); -+ } else { -+ g_main_loop_quit(mloop); -+ g_main_loop_unref(mloop); - } - } - -diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c -index 4088938..5097b53 100644 ---- a/daemons/attrd/pacemaker-attrd.c -+++ b/daemons/attrd/pacemaker-attrd.c -@@ -81,8 +81,12 @@ attrd_cpg_destroy(gpointer unused) - static void - attrd_cib_replaced_cb(const char *event, xmlNode * msg) - { -- crm_notice("Updating all attributes after %s event", event); -+ if (attrd_shutting_down()) { -+ return; -+ } -+ - if (attrd_election_won()) { -+ crm_notice("Updating all attributes after %s event", event); - write_attributes(TRUE, FALSE); - } - } --- -1.8.3.1 - - -From 435b7728688010f057681fdd276109a0c6b06ba0 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 3 Jan 2019 15:13:42 -0600 -Subject: [PATCH 25/69] Fix: attrd: start new election if writer is lost - -If a writer is shutting down when it receives an attribute update, it will not -write it to the CIB. Previously, a new election wouldn't be held until another -attribute needed to be written, which may not happen in a reasonable time (or -at all). Now, we trigger a new election when the writer leaves the cluster. - -rhbz#1535221 ---- - daemons/attrd/attrd_elections.c | 12 ++++++++++-- - 1 file changed, 10 insertions(+), 2 deletions(-) - -diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c -index cfe2d89..a7816a9 100644 ---- a/daemons/attrd/attrd_elections.c -+++ b/daemons/attrd/attrd_elections.c -@@ -1,5 +1,5 @@ - /* -- * Copyright 2013-2018 Andrew Beekhof -+ * Copyright 2013-2019 Andrew Beekhof - * - * This source code is licensed under the GNU General Public License version 2 - * or later (GPLv2+) WITHOUT ANY WARRANTY. -@@ -131,12 +131,20 @@ attrd_declare_winner() - void - attrd_remove_voter(const crm_node_t *peer) - { -+ election_remove(writer, peer->uname); - if (peer_writer && safe_str_eq(peer->uname, peer_writer)) { - free(peer_writer); - peer_writer = NULL; - crm_notice("Lost attribute writer %s", peer->uname); -+ -+ /* If the writer received attribute updates during its shutdown, it will -+ * not have written them to the CIB. Ensure we get a new writer so they -+ * are written out. This means that every node that sees the writer -+ * leave will start a new election, but that's better than losing -+ * attributes. -+ */ -+ attrd_start_election_if_needed(); - } -- election_remove(writer, peer->uname); - } - - void --- -1.8.3.1 - - -From 4dcea7ea009c36ac7f3ebbb65fc02fb5479c3543 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 4 Jan 2019 18:25:57 -0600 -Subject: [PATCH 26/69] Low: attrd: check for alert changes after CIB is - replaced - -Previously, we checked for alert changes when the CIB was updated, but not when -it was replaced entirely. ---- - daemons/attrd/pacemaker-attrd.c | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c -index 5097b53..358e841 100644 ---- a/daemons/attrd/pacemaker-attrd.c -+++ b/daemons/attrd/pacemaker-attrd.c -@@ -1,5 +1,5 @@ - /* -- * Copyright 2013-2018 Andrew Beekhof -+ * Copyright 2013-2019 Andrew Beekhof - * - * This source code is licensed under the GNU General Public License version 2 - * or later (GPLv2+) WITHOUT ANY WARRANTY. -@@ -89,6 +89,9 @@ attrd_cib_replaced_cb(const char *event, xmlNode * msg) - crm_notice("Updating all attributes after %s event", event); - write_attributes(TRUE, FALSE); - } -+ -+ // Check for changes in alerts -+ mainloop_set_trigger(attrd_config_read); - } - - static void --- -1.8.3.1 - - -From c53b2832ecba09012e3a598d3a36655097352328 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Wed, 12 Dec 2018 13:38:39 -0500 -Subject: [PATCH 27/69] Bug: tools: Fix moving a resource with a lifetime - constraint - -A node can be specified two different ways - with attribute="#uname" -and value= on an expression XML node, or with node= on a rsc_location -XML node. Both possibilities need to be considered when attempting to -clear an existing constraint. - -cli_resource_clear was previously only considering the second case. -This patch rearranges the code to allow for trying both, and then adds -the code to try the first case by constructing a different blob of XML -to match. - -See rhbz#1648620 ---- - tools/crm_resource_ban.c | 104 +++++++++++++++++++++++++++++++++++------------ - 1 file changed, 79 insertions(+), 25 deletions(-) - -diff --git a/tools/crm_resource_ban.c b/tools/crm_resource_ban.c -index ef1413f..b74fc0d 100644 ---- a/tools/crm_resource_ban.c -+++ b/tools/crm_resource_ban.c -@@ -191,37 +191,56 @@ cli_resource_prefer(const char *rsc_id, const char *host, cib_t * cib_conn) - return rc; - } - --int --cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn) -+/* Nodes can be specified two different ways in the CIB, so we have two different -+ * functions to try clearing out any constraints on them: -+ * -+ * (1) The node could be given by attribute=/value= in an expression XML node. -+ * That's what resource_clear_node_in_expr handles. That XML looks like this: -+ * -+ * -+ * -+ * -+ * -+ * -+ * -+ * -+ * (2) The mode could be given by node= in an rsc_location XML node. That's -+ * what resource_clear_node_in_location handles. That XML looks like this: -+ * -+ * -+ */ -+static int -+resource_clear_node_in_expr(const char *rsc_id, const char *host, cib_t * cib_conn) - { - int rc = pcmk_ok; -- xmlNode *fragment = NULL; -- xmlNode *location = NULL; -+ char *xpath_string = NULL; - -- if(cib_conn == NULL) { -- return -ENOTCONN; -- } -+ xpath_string = crm_strdup_printf("//rsc_location[@id='cli-prefer-%s'][rule[@id='cli-prefer-rule-%s']/expression[@attribute='#uname' and @value='%s']]", -+ rsc_id, rsc_id, host); - -- fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS); -+ rc = cib_conn->cmds->remove(cib_conn, xpath_string, NULL, cib_xpath | cib_options); -+ if (rc == -ENXIO) { -+ rc = pcmk_ok; -+ } - -- if(host) { -- location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); -- crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host); -+ free(xpath_string); -+ return rc; -+} - -- } else { -- GListPtr n = allnodes; -- for(; n; n = n->next) { -- node_t *target = n->data; -+static int -+resource_clear_node_in_location(const char *rsc_id, const char *host, cib_t * cib_conn) -+{ -+ int rc = pcmk_ok; -+ xmlNode *fragment = NULL; -+ xmlNode *location = NULL; - -- location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); -- crm_xml_set_id(location, "cli-ban-%s-on-%s", -- rsc_id, target->details->uname); -- } -- } -+ fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS); -+ location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); -+ crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host); - - location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); - crm_xml_set_id(location, "cli-prefer-%s", rsc_id); -- if(host && do_force == FALSE) { -+ if (do_force == FALSE) { - crm_xml_add(location, XML_CIB_TAG_NODE, host); - } - -@@ -229,13 +248,48 @@ cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_ - rc = cib_conn->cmds->remove(cib_conn, XML_CIB_TAG_CONSTRAINTS, fragment, cib_options); - if (rc == -ENXIO) { - rc = pcmk_ok; -+ } -+ -+ free(fragment); -+ return rc; -+} - -- } else if (rc != pcmk_ok) { -- goto bail; -+int -+cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn) -+{ -+ int rc = pcmk_ok; -+ -+ if(cib_conn == NULL) { -+ return -ENOTCONN; -+ } -+ -+ if (host) { -+ rc = resource_clear_node_in_expr(rsc_id, host, cib_conn); -+ -+ /* rc does not tell us whether the previous operation did anything, only -+ * whether it failed or not. Thus, as long as it did not fail, we need -+ * to try the second clear method. -+ */ -+ if (rc == pcmk_ok) { -+ rc = resource_clear_node_in_location(rsc_id, host, cib_conn); -+ } -+ -+ } else { -+ GListPtr n = allnodes; -+ -+ /* Iterate over all nodes, attempting to clear the constraint from each. -+ * On the first error, abort. -+ */ -+ for(; n; n = n->next) { -+ node_t *target = n->data; -+ -+ rc = cli_resource_clear(rsc_id, target->details->uname, NULL, cib_conn); -+ if (rc != pcmk_ok) { -+ break; -+ } -+ } - } - -- bail: -- free_xml(fragment); - return rc; - } - --- -1.8.3.1 - - -From af3012388849f4ca5c021494dbf7b1b9260c2155 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Mon, 7 Jan 2019 13:21:32 -0500 -Subject: [PATCH 28/69] Bug: tools: Clear all prefer constraints when - performing a move - -A move is implemented in terms of perfer constraints. If those -constraints contain something like a lifetime expression, and older -prefer constraints are not cleared out, the result is a mess. The XML -that is attempted to insert into the CIB will contain both the older -constraint and then the new lifetime expression as sub-nodes of that -constraint. This is invalid, so the CIB will throw it out. - -The fix is to make sure there are no prefer constraints for any nodes -when a move is done. - -Most ban constraints are left alone, because they may still be valid - -you may want to move a resource to one node while preserving the ban on -another node. Taking care of this is the bulk of the complexity in this -patch. - -One further note - any ban constraints on the destination still need to -be removed. Having both a ban and a prefer constraint on the same node -may technically be valid XML, but doesn't make any sense. - -See rhbz#1648620 ---- - tools/crm_resource.c | 4 ++-- - tools/crm_resource.h | 3 ++- - tools/crm_resource_ban.c | 17 +++++++++++------ - tools/crm_resource_runtime.c | 11 +++++++---- - 4 files changed, 22 insertions(+), 13 deletions(-) - -diff --git a/tools/crm_resource.c b/tools/crm_resource.c -index 2e98999..294c091 100644 ---- a/tools/crm_resource.c -+++ b/tools/crm_resource.c -@@ -982,10 +982,10 @@ main(int argc, char **argv) - rc = -pcmk_err_node_unknown; - goto bail; - } -- rc = cli_resource_clear(rsc_id, dest->details->uname, NULL, cib_conn); -+ rc = cli_resource_clear(rsc_id, dest->details->uname, NULL, cib_conn, TRUE); - - } else { -- rc = cli_resource_clear(rsc_id, NULL, data_set->nodes, cib_conn); -+ rc = cli_resource_clear(rsc_id, NULL, data_set->nodes, cib_conn, TRUE); - } - - } else if (rsc_cmd == 'M' && host_uname) { -diff --git a/tools/crm_resource.h b/tools/crm_resource.h -index dd902be..c371ddc 100644 ---- a/tools/crm_resource.h -+++ b/tools/crm_resource.h -@@ -38,7 +38,8 @@ extern const char *attr_set_type; - /* ban */ - int cli_resource_prefer(const char *rsc_id, const char *host, cib_t * cib_conn); - int cli_resource_ban(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn); --int cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn); -+int cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn, -+ bool clear_ban_constraints); - int cli_resource_clear_all_expired(xmlNode *root, cib_t *cib_conn, const char *rsc, const char *node, bool scope_master); - - /* print */ -diff --git a/tools/crm_resource_ban.c b/tools/crm_resource_ban.c -index b74fc0d..50843f0 100644 ---- a/tools/crm_resource_ban.c -+++ b/tools/crm_resource_ban.c -@@ -228,15 +228,19 @@ resource_clear_node_in_expr(const char *rsc_id, const char *host, cib_t * cib_co - } - - static int --resource_clear_node_in_location(const char *rsc_id, const char *host, cib_t * cib_conn) -+resource_clear_node_in_location(const char *rsc_id, const char *host, cib_t * cib_conn, -+ bool clear_ban_constraints) - { - int rc = pcmk_ok; - xmlNode *fragment = NULL; - xmlNode *location = NULL; - - fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS); -- location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); -- crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host); -+ -+ if (clear_ban_constraints == TRUE) { -+ location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); -+ crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host); -+ } - - location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); - crm_xml_set_id(location, "cli-prefer-%s", rsc_id); -@@ -255,7 +259,8 @@ resource_clear_node_in_location(const char *rsc_id, const char *host, cib_t * ci - } - - int --cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn) -+cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn, -+ bool clear_ban_constraints) - { - int rc = pcmk_ok; - -@@ -271,7 +276,7 @@ cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_ - * to try the second clear method. - */ - if (rc == pcmk_ok) { -- rc = resource_clear_node_in_location(rsc_id, host, cib_conn); -+ rc = resource_clear_node_in_location(rsc_id, host, cib_conn, clear_ban_constraints); - } - - } else { -@@ -283,7 +288,7 @@ cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_ - for(; n; n = n->next) { - node_t *target = n->data; - -- rc = cli_resource_clear(rsc_id, target->details->uname, NULL, cib_conn); -+ rc = cli_resource_clear(rsc_id, target->details->uname, NULL, cib_conn, clear_ban_constraints); - if (rc != pcmk_ok) { - break; - } -diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c -index 27ca3b1..759ca92 100644 ---- a/tools/crm_resource_runtime.c -+++ b/tools/crm_resource_runtime.c -@@ -1378,7 +1378,7 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, int timeout_ms, - } - - if (stop_via_ban) { -- rc = cli_resource_clear(rsc_id, host, NULL, cib); -+ rc = cli_resource_clear(rsc_id, host, NULL, cib, TRUE); - - } else if (orig_target_role) { - rc = cli_resource_update_attribute(rsc, rsc_id, NULL, NULL, -@@ -1460,7 +1460,7 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, int timeout_ms, - - failure: - if (stop_via_ban) { -- cli_resource_clear(rsc_id, host, NULL, cib); -+ cli_resource_clear(rsc_id, host, NULL, cib, TRUE); - } else if (orig_target_role) { - cli_resource_update_attribute(rsc, rsc_id, NULL, NULL, - XML_RSC_ATTR_TARGET_ROLE, -@@ -1874,8 +1874,11 @@ cli_resource_move(resource_t *rsc, const char *rsc_id, const char *host_name, - } - } - -- /* Clear any previous constraints for 'dest' */ -- cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib); -+ /* Clear any previous prefer constraints across all nodes. */ -+ cli_resource_clear(rsc_id, NULL, data_set->nodes, cib, FALSE); -+ -+ /* Clear any previous ban constraints on 'dest'. */ -+ cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib, TRUE); - - /* Record an explicit preference for 'dest' */ - rc = cli_resource_prefer(rsc_id, dest->details->uname, cib); --- -1.8.3.1 - - -From e73a86abff33d9fb4e866c42ead02be7cce38d35 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Tue, 18 Dec 2018 13:10:17 -0500 -Subject: [PATCH 29/69] Tests: cts-cli: Add tests for more crm_resource - options. - ---- - cts/cli/regression.tools.exp | 200 +++++++++++++++++++++++++++++++++++++++++++ - cts/cts-cli.in | 19 ++++ - 2 files changed, 219 insertions(+) - -diff --git a/cts/cli/regression.tools.exp b/cts/cli/regression.tools.exp -index 7a9acc4..b121096 100644 ---- a/cts/cli/regression.tools.exp -+++ b/cts/cli/regression.tools.exp -@@ -2968,3 +2968,203 @@ Deleted 'test-primitive' option: id=test-primitive-meta_attributes-is-managed na - - =#=#=#= End test: Delete resource child meta attribute - OK (0) =#=#=#= - * Passed: crm_resource - Delete resource child meta attribute -+=#=#=#= Begin test: Specify a lifetime when moving a resource =#=#=#= -+Migration will take effect until: -+=#=#=#= Current cib after: Specify a lifetime when moving a resource =#=#=#= -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+=#=#=#= End test: Specify a lifetime when moving a resource - OK (0) =#=#=#= -+* Passed: crm_resource - Specify a lifetime when moving a resource -+=#=#=#= Begin test: Try to move a resource previously moved with a lifetime =#=#=#= -+=#=#=#= Current cib after: Try to move a resource previously moved with a lifetime =#=#=#= -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+=#=#=#= End test: Try to move a resource previously moved with a lifetime - OK (0) =#=#=#= -+* Passed: crm_resource - Try to move a resource previously moved with a lifetime -+=#=#=#= Begin test: Ban dummy from node1 for a short time =#=#=#= -+WARNING: Creating rsc_location constraint 'cli-ban-dummy-on-node1' with a score of -INFINITY for resource dummy on node1. -+ This will prevent dummy from running on node1 until the constraint is removed using the clear option or by editing the CIB with an appropriate tool -+ This will be the case even if node1 is the last node in the cluster -+Migration will take effect until: -+=#=#=#= Current cib after: Ban dummy from node1 for a short time =#=#=#= -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+=#=#=#= End test: Ban dummy from node1 for a short time - OK (0) =#=#=#= -+* Passed: crm_resource - Ban dummy from node1 for a short time -+=#=#=#= Begin test: Remove expired constraints =#=#=#= -+=#=#=#= Current cib after: Remove expired constraints =#=#=#= -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+=#=#=#= End test: Remove expired constraints - OK (0) =#=#=#= -+* Passed: crm_resource - Remove expired constraints -diff --git a/cts/cts-cli.in b/cts/cts-cli.in -index 880cebb..34c094e 100755 ---- a/cts/cts-cli.in -+++ b/cts/cts-cli.in -@@ -394,6 +394,23 @@ function test_tools() { - cmd="crm_resource -r test-primitive --meta -d is-managed" - test_assert $CRM_EX_OK - -+ desc="Specify a lifetime when moving a resource" -+ cmd="crm_resource -r dummy --move --node node2 --lifetime=PT1H" -+ test_assert $CRM_EX_OK -+ -+ desc="Try to move a resource previously moved with a lifetime" -+ cmd="crm_resource -r dummy --move --node node1" -+ test_assert $CRM_EX_OK -+ -+ desc="Ban dummy from node1 for a short time" -+ cmd="crm_resource -r dummy -B -N node1 --lifetime=PT1S" -+ test_assert $CRM_EX_OK -+ -+ desc="Remove expired constraints" -+ sleep 2 -+ cmd="crm_resource --clear --expired" -+ test_assert $CRM_EX_OK -+ - unset CIB_shadow_dir - rm -f "$TMPXML" "$TMPORIG" - } -@@ -920,6 +937,8 @@ for t in $tests; do - -e 's|^/tmp/cts-cli\.validity\.bad.xml\.[^:]*:|validity.bad.xml:|'\ - -e 's/^Entity: line [0-9][0-9]*: //'\ - -e 's/\(validation ([0-9][0-9]* of \)[0-9][0-9]*\().*\)/\1X\2/' \ -+ -e 's/^Migration will take effect until: .*/Migration will take effect until:/' \ -+ -e 's/ end=\"[-: 0123456789]*Z\?\"/ end=\"\"/' \ - "$TMPFILE" > "${TMPFILE}.$$" - mv -- "${TMPFILE}.$$" "$TMPFILE" - --- -1.8.3.1 - - -From 48f59e65223e018f4a639c4c9c155cb62e7806a9 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Thu, 10 Jan 2019 11:13:08 -0500 -Subject: [PATCH 30/69] Tests: cts-cli: Use extended regular expressions. - -FreeBSD sed requires the use of extended regular expressions to use the -+ operator in a regex. The -E command line argument turns that on. GNU -sed also supports this. Doing so flips the meaning of parens, though. -An unescaped paren is now used for grouping, and an escaped paren is for -a literal paren. ---- - cts/cts-cli.in | 15 ++++++++------- - 1 file changed, 8 insertions(+), 7 deletions(-) - -diff --git a/cts/cts-cli.in b/cts/cts-cli.in -index 34c094e..e48d644 100755 ---- a/cts/cts-cli.in -+++ b/cts/cts-cli.in -@@ -924,21 +924,22 @@ for t in $tests; do - eval TMPFILE_$t="$TMPFILE" - test_$t > "$TMPFILE" - -- sed -e 's/cib-last-written.*>/>/'\ -+ sed -E \ -+ -e 's/cib-last-written.*>/>/'\ - -e 's/ last-run=\"[0-9]*\"//'\ - -e 's/crm_feature_set="[^"]*" //'\ - -e 's/validate-with="[^"]*" //'\ - -e 's/Created new pacemaker-.* configuration/Created new pacemaker configuration/'\ -- -e 's/.*\(pcmk__.*\)@.*\.c:[0-9][0-9]*)/\1/g' \ -- -e 's/.*\(unpack_.*\)@.*\.c:[0-9][0-9]*)/\1/g' \ -- -e 's/.*\(update_validation\)@.*\.c:[0-9][0-9]*)/\1/g' \ -- -e 's/.*\(apply_upgrade\)@.*\.c:[0-9][0-9]*)/\1/g' \ -+ -e 's/.*(pcmk__.*)@.*.c:[0-9][0-9]*\)/\1/g' \ -+ -e 's/.*(unpack_.*)@.*.c:[0-9][0-9]*\)/\1/g' \ -+ -e 's/.*(update_validation)@.*\.c:[0-9][0-9]*\)/\1/g' \ -+ -e 's/.*(apply_upgrade)@.*\.c:[0-9][0-9]*\)/\1/g' \ - -e 's/ last-rc-change=\"[0-9]*\"//'\ - -e 's|^/tmp/cts-cli\.validity\.bad.xml\.[^:]*:|validity.bad.xml:|'\ - -e 's/^Entity: line [0-9][0-9]*: //'\ -- -e 's/\(validation ([0-9][0-9]* of \)[0-9][0-9]*\().*\)/\1X\2/' \ -+ -e 's/(validation \([0-9][0-9]* of )[0-9][0-9]*(\).*)/\1X\2/' \ - -e 's/^Migration will take effect until: .*/Migration will take effect until:/' \ -- -e 's/ end=\"[-: 0123456789]*Z\?\"/ end=\"\"/' \ -+ -e 's/ end=\"[-: 0123456789]+Z?\"/ end=\"\"/' \ - "$TMPFILE" > "${TMPFILE}.$$" - mv -- "${TMPFILE}.$$" "$TMPFILE" - --- -1.8.3.1 - - -From bf2463c6a1d95e5a281ff324f0fc6416aa28e7b3 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 8 Jan 2019 16:18:13 -0600 -Subject: [PATCH 31/69] Log: libcrmcommon: downgrade empty output logging to - trace level - -nothing is not very interesting, so reduce clutter ---- - lib/common/logging.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/lib/common/logging.c b/lib/common/logging.c -index bfd82b7..ff5ae74 100644 ---- a/lib/common/logging.c -+++ b/lib/common/logging.c -@@ -997,7 +997,7 @@ crm_log_output_fn(const char *file, const char *function, int line, int level, c - const char *offset = NULL; - - if (output == NULL) { -- level = LOG_DEBUG; -+ level = LOG_TRACE; - output = "-- empty --"; - } - --- -1.8.3.1 - - -From 490ef9f6ed30427b3617236c514dd076ffd0e88f Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 10 Jan 2019 14:36:26 -0600 -Subject: [PATCH 32/69] Low: resources: clean serialized file on SIGTERM in - Dummy - -Otherwise it could give a false probe error at next start, -confusing whatever else is being tested with a dummy resource. -Unfortunately this doesn't help if an in-flight monitor gets -cancelled with a SIGKILL, but there's no obvious solution there. ---- - extra/resources/Dummy | 5 +++++ - 1 file changed, 5 insertions(+) - -diff --git a/extra/resources/Dummy b/extra/resources/Dummy -index faa0e08..75e4cf5 100755 ---- a/extra/resources/Dummy -+++ b/extra/resources/Dummy -@@ -114,6 +114,10 @@ END - trap sigterm_handler TERM - sigterm_handler() { - ocf_log info "They use TERM to bring us down. No such luck." -+ -+ # Since we're likely going to get KILLed, clean up any monitor -+ # serialization in progress, so the next probe doesn't return an error. -+ rm -f "${VERIFY_SERIALIZED_FILE}" - return - } - -@@ -171,6 +175,7 @@ dummy_monitor() { - # two monitor ops have occurred at the same time. - # This verifies a condition in pacemaker-execd regression tests. - ocf_log err "$VERIFY_SERIALIZED_FILE exists already" -+ ocf_exit_reason "alternate universe collision" - return $OCF_ERR_GENERIC - fi - --- -1.8.3.1 - - -From fe7172ff757996300c41d3d382d050a69c944bfc Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 8 Jan 2019 15:31:14 -0600 -Subject: [PATCH 33/69] Fix: controller: directly acknowledge unrecordable - operation results - -Regression introduced in 2.0.1-rc1 by 0363985dd - -Before that commit, if an operation result arrived when there was no resource -information available, a warning would be logged and the operation would be -directly acknowledged. This could occur, for example, if resource history were -cleaned while an operation was pending on that resource. - -After that commit, in that situation, an assertion and error would be logged, -and no acknowledgement would be sent, leading to a transition timeout. - -Restore the direct ack. Also improve related log messages. ---- - daemons/controld/controld_execd.c | 80 +++++++++++++++++++++++++++------------ - 1 file changed, 55 insertions(+), 25 deletions(-) - -diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c -index f7c5cde..26fcced 100644 ---- a/daemons/controld/controld_execd.c -+++ b/daemons/controld/controld_execd.c -@@ -2483,6 +2483,7 @@ process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, - int update_id = 0; - gboolean remove = FALSE; - gboolean removed = FALSE; -+ bool need_direct_ack = FALSE; - lrmd_rsc_info_t *rsc = NULL; - const char *node_name = NULL; - -@@ -2513,7 +2514,6 @@ process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, - op_key, op->rsc_id); - } - } -- CRM_LOG_ASSERT(rsc != NULL); // If it's still NULL, there's a bug somewhere - - // Get node name if available (from executor state or action XML) - if (lrm_state) { -@@ -2545,51 +2545,81 @@ process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, - } - - if (op->op_status != PCMK_LRM_OP_CANCELLED) { -+ /* We might not record the result, so directly acknowledge it to the -+ * originator instead, so it doesn't time out waiting for the result -+ * (especially important if part of a transition). -+ */ -+ need_direct_ack = TRUE; -+ - if (controld_action_is_recordable(op->op_type)) { - if (node_name && rsc) { -+ // We should record the result, and happily, we can - update_id = do_update_resource(node_name, rsc, op); -+ need_direct_ack = FALSE; -+ -+ } else if (op->rsc_deleted) { -+ /* We shouldn't record the result (likely the resource was -+ * refreshed, cleaned, or removed while this operation was -+ * in flight). -+ */ -+ crm_notice("Not recording %s result in CIB because " -+ "resource information was removed since it was initiated", -+ op_key); - } else { -- // @TODO Should we direct ack? -- crm_err("Unable to record %s result in CIB: %s", -- op_key, -+ /* This shouldn't be possible; the executor didn't consider the -+ * resource deleted, but we couldn't find resource or node -+ * information. -+ */ -+ crm_err("Unable to record %s result in CIB: %s", op_key, - (node_name? "No resource information" : "No node name")); - } -- } else { -- send_direct_ack(NULL, NULL, NULL, op, op->rsc_id); - } -+ - } else if (op->interval_ms == 0) { -- /* This will occur when "crm resource cleanup" is called while actions are in-flight */ -- crm_err("Op %s (call=%d): Cancelled", op_key, op->call_id); -- send_direct_ack(NULL, NULL, NULL, op, op->rsc_id); -+ /* A non-recurring operation was cancelled. Most likely, the -+ * never-initiated action was removed from the executor's pending -+ * operations list upon resource removal. -+ */ -+ need_direct_ack = TRUE; - - } else if (pending == NULL) { -- /* We don't need to do anything for cancelled ops -- * that are not in our pending op list. There are no -- * transition actions waiting on these operations. */ -+ /* This recurring operation was cancelled, but was not pending. No -+ * transition actions are waiting on it, nothing needs to be done. -+ */ - - } else if (op->user_data == NULL) { -- /* At this point we have a pending entry, but no transition -- * key present in the user_data field. report this */ -- crm_err("Op %s (call=%d): No user data", op_key, op->call_id); -+ /* This recurring operation was cancelled and pending, but we don't -+ * have a transition key. This should never happen. -+ */ -+ crm_err("Recurring operation %s was cancelled without transition information", -+ op_key); - - } else if (pending->remove) { -- /* The tengine canceled this op, we have been waiting for the cancel to finish. */ -+ /* This recurring operation was cancelled (by us) and pending, and we -+ * have been waiting for it to finish. -+ */ - if (lrm_state) { - erase_lrm_history_by_op(lrm_state, op); - } - - } else if (op->rsc_deleted) { -- /* The tengine initiated this op, but it was cancelled outside of the -- * tengine's control during a resource cleanup/re-probe request. The tengine -- * must be alerted that this operation completed, otherwise the tengine -- * will continue waiting for this update to occur until it is timed out. -- * We don't want this update going to the cib though, so use a direct ack. */ -- crm_trace("Op %s (call=%d): cancelled due to rsc deletion", op_key, op->call_id); -- send_direct_ack(NULL, NULL, NULL, op, op->rsc_id); -+ /* This recurring operation was cancelled (but not by us, and the -+ * executor does not have resource information, likely due to resource -+ * cleanup, refresh, or removal) and pending. -+ */ -+ crm_debug("Recurring op %s was cancelled due to resource deletion", -+ op_key); -+ need_direct_ack = TRUE; - - } else { -- /* Before a stop is called, no need to direct ack */ -- crm_trace("Op %s (call=%d): no delete event required", op_key, op->call_id); -+ /* This recurring operation was cancelled (but not by us, likely by the -+ * executor before stopping the resource) and pending. We don't need to -+ * do anything special. -+ */ -+ } -+ -+ if (need_direct_ack) { -+ send_direct_ack(NULL, NULL, NULL, op, op->rsc_id); - } - - if(remove == FALSE) { --- -1.8.3.1 - - -From cf64fdd8c842a365f90a28cdf3f374a4ba1e62c2 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 10 Jan 2019 15:10:50 -0600 -Subject: [PATCH 34/69] Doc: ChangeLog: update for 2.0.1-rc3 release - ---- - ChangeLog | 13 +++++++++++++ - 1 file changed, 13 insertions(+) - -diff --git a/ChangeLog b/ChangeLog -index e348333..ea65fbe 100644 ---- a/ChangeLog -+++ b/ChangeLog -@@ -1,3 +1,16 @@ -+* Thu Jan 10 2019 Ken Gaillot Pacemaker-2.0.1-rc3 -+- Changesets: 27 -+- Diff: 20 files changed, 375 insertions(+), 195 deletions(-) -+ -+- Changes since Pacemaker-2.0.1-rc2 -+ + attrd: start new election immediately if writer is lost -+ + attrd: detect alert configuration changes when CIB is entirely replaced -+ + controller: avoid transition timeout if resource cleaned while operation -+ is in-flight (regression in 2.0.1-rc1) -+ + libstonithd: restore C++ compatibility (regression in 2.0.1-rc1) -+ + tools: fix crm_resource --clear when lifetime was used with ban/move -+ + tools: fix crm_resource --move when lifetime was used with previous move -+ - * Wed Dec 19 2018 Ken Gaillot Pacemaker-2.0.1-rc2 - - Changesets: 12 - - Diff: 2 files changed, 6 insertions(+), 2 deletions(-) --- -1.8.3.1 - - -From bddfb0d3373951be9a200c1b1968e7d925d7fa53 Mon Sep 17 00:00:00 2001 -From: "Gao,Yan" -Date: Fri, 11 Jan 2019 11:43:49 +0100 -Subject: [PATCH 35/69] Test: cts-exec: still run the tests for the other - resource classes even without python systemd bindings - ---- - cts/cts-exec.in | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - -diff --git a/cts/cts-exec.in b/cts/cts-exec.in -index 235a966..30b87dd 100644 ---- a/cts/cts-exec.in -+++ b/cts/cts-exec.in -@@ -387,9 +387,9 @@ class Tests(object): - # all systemd tests to fail. - import systemd.daemon - except ImportError: -- print("Fatal error: python systemd bindings not found. Is package installed?", -- file=sys.stderr) -- sys.exit(CrmExit.ERROR) -+ print("Python systemd bindings not found.") -+ print("The tests for systemd class are not going to be run.") -+ self.rsc_classes.remove("systemd") - - print("Testing resource classes", repr(self.rsc_classes)) - --- -1.8.3.1 - - -From ca267fc163d9f3fc4373d558cc9ab7c096a67171 Mon Sep 17 00:00:00 2001 -From: "Gao,Yan" -Date: Fri, 18 Jan 2019 17:02:06 +0100 -Subject: [PATCH 36/69] Test: cts: service counts as enabled only if it's - explicitly enabled - -With "systemctl is-enabled", we should check if the service is -explicitly "enabled" instead of the return code. For example it returns -0 if the service is "static" or "indirect", but they don't really count -as "enabled". - -This also functionizes the check part for further use. ---- - cts/environment.py | 27 ++++++++++++++++----------- - 1 file changed, 16 insertions(+), 11 deletions(-) - -diff --git a/cts/environment.py b/cts/environment.py -index 0ce7513..611d0c3 100644 ---- a/cts/environment.py -+++ b/cts/environment.py -@@ -180,20 +180,25 @@ class Environment(object): - # default - self["syslogd"] = "rsyslog" - -+ def service_is_enabled(self, node, service): -+ if self["have_systemd"]: -+ # Systemd -+ -+ # With "systemctl is-enabled", we should check if the service is -+ # explicitly "enabled" instead of the return code. For example it returns -+ # 0 if the service is "static" or "indirect", but they don't really count -+ # as "enabled". -+ return not self.rsh(node, "systemctl is-enabled %s | grep enabled" % service) -+ -+ else: -+ # SYS-V -+ return not self.rsh(node, "chkconfig --list | grep -e %s.*on" % service) -+ - def detect_at_boot(self): - # Detect if the cluster starts at boot - if not "at-boot" in self.data: -- atboot = 0 -- -- if self["have_systemd"]: -- # Systemd -- atboot = atboot or not self.rsh(self.target, "systemctl is-enabled corosync.service") -- atboot = atboot or not self.rsh(self.target, "systemctl is-enabled pacemaker.service") -- else: -- # SYS-V -- atboot = atboot or not self.rsh(self.target, "chkconfig --list | grep -e corosync.*on -e pacemaker.*on") -- -- self["at-boot"] = atboot -+ self["at-boot"] = self.service_is_enabled(self.target, "corosync") \ -+ or self.service_is_enabled(self.target, "pacemaker") - - def detect_ip_offset(self): - # Try to determine an offset for IPaddr resources --- -1.8.3.1 - - -From 286569c1a2359a97ad33f77b3b6b63b04d520b76 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Sat, 19 Jan 2019 22:18:37 +0100 -Subject: [PATCH 37/69] Revert "Tests: cts-cli: Use extended regular - expressions." - -This reverts commit 48f59e65223e018f4a639c4c9c155cb62e7806a9, because -while it's true that GNU sed supports -E switch _now_, it wasn't the -case until about v4.1.5+ (silently, see 3a8e165, and explicitly since -v4.3, see 8b65e07), meaning that, for instance, RHEL 7 doesn't -officially supports it (as sad as it is): -https://bugzilla.redhat.com/show_bug.cgi?id=1564789 - -Also note that while there was an effort to standardize -E switch -in POSIX (http://austingroupbugs.net/view.php?id=528) it may have passed -without any affect so far (speaking of "2018 edition"): -https://pubs.opengroup.org/onlinepubs/9699919799/utilities/sed.html -Definitely, it wasn't in IEEE Std 1003.1-2008 that, moreover, we don't -take for granted even 10+ years later (cf. ongoing parallel discussion -whether and how to check for %m specifier to scanf(3)), the change at -hand is not anything else but invalid. Also, -E implementation may -have been faulty until sed v4.5: -https://lists.gnu.org/archive/html/info-gnu/2018-04/msg00000.html - -Note that one can make do without extended regular expressions, and in -turn, without '+' (or '\+' that is just a GNU-specific extension into -basic regular expressions syntax), since the respective substitute -can be used: "a+" ~ "aa*" (can be hefty for long patterns, but that's -what we reliably have). ---- - cts/cts-cli.in | 15 +++++++-------- - 1 file changed, 7 insertions(+), 8 deletions(-) - -diff --git a/cts/cts-cli.in b/cts/cts-cli.in -index e48d644..34c094e 100755 ---- a/cts/cts-cli.in -+++ b/cts/cts-cli.in -@@ -924,22 +924,21 @@ for t in $tests; do - eval TMPFILE_$t="$TMPFILE" - test_$t > "$TMPFILE" - -- sed -E \ -- -e 's/cib-last-written.*>/>/'\ -+ sed -e 's/cib-last-written.*>/>/'\ - -e 's/ last-run=\"[0-9]*\"//'\ - -e 's/crm_feature_set="[^"]*" //'\ - -e 's/validate-with="[^"]*" //'\ - -e 's/Created new pacemaker-.* configuration/Created new pacemaker configuration/'\ -- -e 's/.*(pcmk__.*)@.*.c:[0-9][0-9]*\)/\1/g' \ -- -e 's/.*(unpack_.*)@.*.c:[0-9][0-9]*\)/\1/g' \ -- -e 's/.*(update_validation)@.*\.c:[0-9][0-9]*\)/\1/g' \ -- -e 's/.*(apply_upgrade)@.*\.c:[0-9][0-9]*\)/\1/g' \ -+ -e 's/.*\(pcmk__.*\)@.*\.c:[0-9][0-9]*)/\1/g' \ -+ -e 's/.*\(unpack_.*\)@.*\.c:[0-9][0-9]*)/\1/g' \ -+ -e 's/.*\(update_validation\)@.*\.c:[0-9][0-9]*)/\1/g' \ -+ -e 's/.*\(apply_upgrade\)@.*\.c:[0-9][0-9]*)/\1/g' \ - -e 's/ last-rc-change=\"[0-9]*\"//'\ - -e 's|^/tmp/cts-cli\.validity\.bad.xml\.[^:]*:|validity.bad.xml:|'\ - -e 's/^Entity: line [0-9][0-9]*: //'\ -- -e 's/(validation \([0-9][0-9]* of )[0-9][0-9]*(\).*)/\1X\2/' \ -+ -e 's/\(validation ([0-9][0-9]* of \)[0-9][0-9]*\().*\)/\1X\2/' \ - -e 's/^Migration will take effect until: .*/Migration will take effect until:/' \ -- -e 's/ end=\"[-: 0123456789]+Z?\"/ end=\"\"/' \ -+ -e 's/ end=\"[-: 0123456789]*Z\?\"/ end=\"\"/' \ - "$TMPFILE" > "${TMPFILE}.$$" - mv -- "${TMPFILE}.$$" "$TMPFILE" - --- -1.8.3.1 - - -From 9394c8a585138be0a83adde5c7adb3f3f2032fd6 Mon Sep 17 00:00:00 2001 -From: "Gao,Yan" -Date: Fri, 18 Jan 2019 18:14:13 +0100 -Subject: [PATCH 38/69] Test: cts: temporarily disable any enabled cluster - serivces when running remote tests - -Cluster nodes are reused as remote nodes in remote tests. If cluster -services were enabled at boot, in case the remote node got fenced, the -cluster node would join instead of the expected remote one. Meanwhile -pacemaker_remote would not be able to start. Depending on the chances, -the situations might not be able to be orchestrated gracefully any more. ---- - cts/CTStests.py | 27 +++++++++++++++++++++++++++ - cts/environment.py | 18 ++++++++++++++++++ - 2 files changed, 45 insertions(+) - -diff --git a/cts/CTStests.py b/cts/CTStests.py -index 6a4aa51..3d5d88c 100644 ---- a/cts/CTStests.py -+++ b/cts/CTStests.py -@@ -2674,6 +2674,22 @@ class RemoteDriver(CTSTest): - if not self.failed: - self.remote_node_added = 1 - -+ def disable_services(self, node): -+ self.corosync_enabled = self.Env.service_is_enabled(node, "corosync") -+ if self.corosync_enabled: -+ self.Env.disable_service(node, "corosync") -+ -+ self.pacemaker_enabled = self.Env.service_is_enabled(node, "pacemaker") -+ if self.pacemaker_enabled: -+ self.Env.disable_service(node, "pacemaker") -+ -+ def restore_services(self, node): -+ if self.corosync_enabled: -+ self.Env.enable_service(node, "corosync") -+ -+ if self.pacemaker_enabled: -+ self.Env.enable_service(node, "pacemaker") -+ - def stop_pcmk_remote(self, node): - # disable pcmk remote - for i in range(10): -@@ -2703,6 +2719,15 @@ class RemoteDriver(CTSTest): - self.rsh(node, "killall -CONT pacemaker-remoted") - - def start_metal(self, node): -+ # Cluster nodes are reused as remote nodes in remote tests. If cluster -+ # services were enabled at boot, in case the remote node got fenced, the -+ # cluster node would join instead of the expected remote one. Meanwhile -+ # pacemaker_remote would not be able to start. Depending on the chances, -+ # the situations might not be able to be orchestrated gracefully any more. -+ # -+ # Temporarily disable any enabled cluster serivces. -+ self.disable_services(node) -+ - pcmk_started = 0 - - # make sure the resource doesn't already exist for some reason -@@ -2875,6 +2900,8 @@ class RemoteDriver(CTSTest): - return - - def cleanup_metal(self, node): -+ self.restore_services(node) -+ - if self.pcmk_started == 0: - return - -diff --git a/cts/environment.py b/cts/environment.py -index 611d0c3..d152578 100644 ---- a/cts/environment.py -+++ b/cts/environment.py -@@ -180,6 +180,24 @@ class Environment(object): - # default - self["syslogd"] = "rsyslog" - -+ def disable_service(self, node, service): -+ if self["have_systemd"]: -+ # Systemd -+ return self.rsh(node, "systemctl disable %s" % service) -+ -+ else: -+ # SYS-V -+ return self.rsh(node, "chkconfig %s off" % service) -+ -+ def enable_service(self, node, service): -+ if self["have_systemd"]: -+ # Systemd -+ return self.rsh(node, "systemctl enable %s" % service) -+ -+ else: -+ # SYS-V -+ return self.rsh(node, "chkconfig %s on" % service) -+ - def service_is_enabled(self, node, service): - if self["have_systemd"]: - # Systemd --- -1.8.3.1 - - -From abb6feeb39c027aa0c3f973ef93e3524ea866936 Mon Sep 17 00:00:00 2001 -From: Klaus Wenninger -Date: Mon, 21 Jan 2019 13:40:44 +0100 -Subject: [PATCH 39/69] Fix: crm_mon: remove duplicity of fence-action-state in - xml-output - -and unnecessary conditionals making code less readable that were -probably introduced rearranging the code. - -rhbz#1667191 ---- - tools/crm_mon.c | 16 +++++----------- - 1 file changed, 5 insertions(+), 11 deletions(-) - -diff --git a/tools/crm_mon.c b/tools/crm_mon.c -index aad9b71..bc161aa 100644 ---- a/tools/crm_mon.c -+++ b/tools/crm_mon.c -@@ -3175,7 +3175,7 @@ print_stonith_action(FILE *stream, stonith_history_t *event) - fprintf(stream, " completed=\"%s\"", run_at_s?run_at_s:""); - break; - default: -- fprintf(stream, " state=\"pending\""); -+ break; - } - fprintf(stream, " />\n"); - break; -@@ -3189,8 +3189,7 @@ print_stonith_action(FILE *stream, stonith_history_t *event) - action_s, event->target, - event->delegate ? event->delegate : "", - event->client, event->origin, -- ((!fence_full_history) && (output_format != mon_output_xml))? -- "last-successful":"completed", -+ fence_full_history?"completed":"last-successful", - run_at_s?run_at_s:""); - break; - case st_failed: -@@ -3199,8 +3198,7 @@ print_stonith_action(FILE *stream, stonith_history_t *event) - action_s, event->target, - event->delegate ? event->delegate : "", - event->client, event->origin, -- ((!fence_full_history) && (output_format != mon_output_xml))? -- "last-failed":"completed", -+ fence_full_history?"completed":"last-failed", - run_at_s?run_at_s:""); - break; - default: -@@ -3219,9 +3217,7 @@ print_stonith_action(FILE *stream, stonith_history_t *event) - action_s, event->target, - event->delegate ? event->delegate : "", - event->client, event->origin, -- ((!fence_full_history) && -- (output_format != mon_output_xml))? -- "last-successful":"completed", -+ fence_full_history?"completed":"last-successful", - run_at_s?run_at_s:""); - break; - case st_failed: -@@ -3230,9 +3226,7 @@ print_stonith_action(FILE *stream, stonith_history_t *event) - action_s, event->target, - event->delegate ? event->delegate : "", - event->client, event->origin, -- ((!fence_full_history) && -- (output_format != mon_output_xml))? -- "last-failed":"completed", -+ fence_full_history?"completed":"last-failed", - run_at_s?run_at_s:""); - break; - default: --- -1.8.3.1 - - -From a2e873635db3dfbb696527372dfaad9f58621f48 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Mon, 21 Jan 2019 20:46:56 +0100 -Subject: [PATCH 40/69] Build: 2 *.c: restore GCC 9 -Werror buildability: avoid - NULL to '%s' arg - -Sadly, pacemaker codebase seems to be possibly heavily spoiled with -this undefined behaviour when possibly passing NULL corresponding to -'%s' format specifier argument, so for the time being, fix just what -new GCC 9 started to spot[*] (due to build-time constant NULLs, which -is an immediate proof) since these occurrences boil down to mere -thinkos. Related to that, would be wise to start rolling out -nonnull annotations to preserve more general sanity in this self -explanatory aspect. - -Looking at libqb code (end destination of "crm_log" processing), there's -nothing to implicitly mask NULL with a predestined string explicitly -(like glibc make do with "(null)" in majority of the cases), so unless -merely a blackbox is used for logging (qb_vsnprintf_serialize seems to -deal with such a NULL gracefully), passing NULLs where a character array -is expected is rather dangerous without the prior knowledge of -particular libc (vsnprintf) implementation. - -[*] https://gcc.gnu.org/git/?p=gcc.git;a=blobdiff;f=gcc/gimple-ssa-sprintf.c;h=456a7d400115713a6600b1ce7bb303b6c971550e;hb=7b2ced2fa2c0597ba083461864c9026c3c9d3720;hpb=b07bf3b9730bebfc9953ea2eeee86a1274e39022 ---- - daemons/based/based_callbacks.c | 6 +++--- - tools/test.iso8601.c | 11 ++++------- - 2 files changed, 7 insertions(+), 10 deletions(-) - -diff --git a/daemons/based/based_callbacks.c b/daemons/based/based_callbacks.c -index 723e74c..daef8c7 100644 ---- a/daemons/based/based_callbacks.c -+++ b/daemons/based/based_callbacks.c -@@ -594,8 +594,8 @@ parse_peer_options_v1(int call_type, xmlNode * request, - return TRUE; - } - -- crm_trace("Processing %s request sent by %s", op, originator); - op = crm_element_value(request, F_CIB_OPERATION); -+ crm_trace("Processing %s request sent by %s", op, originator); - if (safe_str_eq(op, "cib_shutdown_req")) { - /* Always process these */ - *local_notify = FALSE; -@@ -650,11 +650,11 @@ parse_peer_options_v1(int call_type, xmlNode * request, - - } else if (safe_str_eq(op, "cib_shutdown_req")) { - if (reply_to != NULL) { -- crm_debug("Processing %s from %s", op, host); -+ crm_debug("Processing %s from %s", op, originator); - *needs_reply = FALSE; - - } else { -- crm_debug("Processing %s reply from %s", op, host); -+ crm_debug("Processing %s reply from %s", op, originator); - } - return TRUE; - -diff --git a/tools/test.iso8601.c b/tools/test.iso8601.c -index 6d70af5..93ca481 100644 ---- a/tools/test.iso8601.c -+++ b/tools/test.iso8601.c -@@ -8,6 +8,7 @@ - #include - #include - #include -+#include /* CRM_ASSERT */ - #include - - char command = 0; -@@ -46,13 +47,9 @@ static struct crm_option long_options[] = { - static void - log_time_period(int log_level, crm_time_period_t * dtp, int flags) - { -- char *end = NULL; -- char *start = NULL; -- -- if(dtp) { -- start = crm_time_as_string(dtp->start, flags); -- end = crm_time_as_string(dtp->end, flags); -- } -+ char *start = crm_time_as_string(dtp->start, flags); -+ char *end = crm_time_as_string(dtp->end, flags); -+ CRM_ASSERT(start != NULL && end != NULL); - - if (log_level < LOG_CRIT) { - printf("Period: %s to %s\n", start, end); --- -1.8.3.1 - - -From cb3312337603e894743789aaeb7bd13bbbae9b56 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 14 Jan 2019 09:12:37 -0600 -Subject: [PATCH 41/69] Low: libstonithd: correct header inclusions - -stonith-ng.h was unnecessarily including libxml/tree.h, and was missing -includes for time.h (for time_t) and stdint.h (for uint32_t). ---- - include/crm/stonith-ng.h | 9 ++++----- - 1 file changed, 4 insertions(+), 5 deletions(-) - -diff --git a/include/crm/stonith-ng.h b/include/crm/stonith-ng.h -index 62a72d9..df99b55 100644 ---- a/include/crm/stonith-ng.h -+++ b/include/crm/stonith-ng.h -@@ -1,5 +1,5 @@ - /* -- * Copyright 2004-2018 Andrew Beekhof -+ * Copyright 2004-2019 Andrew Beekhof - * - * This source code is licensed under the GNU Lesser General Public License - * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -@@ -20,10 +20,9 @@ extern "C" { - - # include - # include --# include -- --/* TO-DO: Work out how to drop this requirement */ --# include -+# include // bool -+# include // uint32_t -+# include // time_t - - # define T_STONITH_NOTIFY_DISCONNECT "st_notify_disconnect" - # define T_STONITH_NOTIFY_FENCE "st_notify_fence" --- -1.8.3.1 - - -From 991f5bffcffb43bbd398f5e20773775ae4052d72 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 15 Jan 2019 14:44:47 -0600 -Subject: [PATCH 42/69] Doc: Pacemaker Administration: mention version - requirement for remote CIB administration - ---- - doc/Pacemaker_Administration/en-US/Ch-Configuring.txt | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/doc/Pacemaker_Administration/en-US/Ch-Configuring.txt b/doc/Pacemaker_Administration/en-US/Ch-Configuring.txt -index 5ca9dfc..286479b 100644 ---- a/doc/Pacemaker_Administration/en-US/Ch-Configuring.txt -+++ b/doc/Pacemaker_Administration/en-US/Ch-Configuring.txt -@@ -434,3 +434,9 @@ to set the +remote-tls-port+ (encrypted) or +remote-clear-port+ - - |========================================================= - -+[IMPORTANT] -+==== -+The Pacemaker version on the administration host must be the same or greater -+than the version(s) on the cluster nodes. Otherwise, it may not have the schema -+files necessary to validate the CIB. -+==== --- -1.8.3.1 - - -From 6457d5bd48e9659b4a6fd42a173ab11ce3bb781b Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 16 Jan 2019 12:28:13 -0600 -Subject: [PATCH 43/69] Build: spec: add hint about sbd compatibility - -The libpe_status API for the pe_working_set_t data type changed incompatibly -in 2.0.1. sbd 1.4.0 supports the new API, so add a Conflicts line for versions -older than that. Of course, the library soname version change is the hard -barrier, but this provides another hint to someone preparing for an upgrade. ---- - pacemaker.spec.in | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/pacemaker.spec.in b/pacemaker.spec.in -index 6b2b268..d11ba1e 100644 ---- a/pacemaker.spec.in -+++ b/pacemaker.spec.in -@@ -296,6 +296,8 @@ Summary: Core Pacemaker libraries - Group: System Environment/Daemons - Requires(pre): shadow-utils - Requires: %{name}-schemas = %{version}-%{release} -+# sbd 1.4.0+ supports the libpe_status API for pe_working_set_t -+Conflicts: sbd < 1.4.0 - - %description libs - Pacemaker is an advanced, scalable High-Availability cluster resource --- -1.8.3.1 - - -From 7a5444d0cf312feb7f92bed0134a32cd6777c3d9 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 18 Jan 2019 15:33:39 -0600 -Subject: [PATCH 44/69] Doc: GNUmakefile: remove obsolete references to CMAN in - comments - ---- - GNUmakefile | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - -diff --git a/GNUmakefile b/GNUmakefile -index 60de623..58ee17e 100644 ---- a/GNUmakefile -+++ b/GNUmakefile -@@ -1,5 +1,5 @@ - # --# Copyright 2008-2018 Andrew Beekhof -+# Copyright 2008-2019 Andrew Beekhof - # - # This source code is licensed under the GNU General Public License version 2 - # or later (GPLv2+) WITHOUT ANY WARRANTY. -@@ -74,7 +74,7 @@ RSYNC_OPTS = -rlptvzxS --progress - # - # --with[out] FOO -> --define "_with[out]_FOO --with[out]-FOO" - # --# $(1) ... WITH string (e.g., --with pre_release --without cman) -+# $(1) ... WITH string (e.g., --with pre_release --without doc) - # $(2) ... options following the initial "rpmbuild" in the command - # $(3) ... final arguments determined with $2 (e.g., pacemaker.spec) - # -@@ -214,7 +214,7 @@ mock-sh-%: - mock --root=$* $(MOCK_OPTIONS) --shell - echo "Done" - --# eg. WITH="--with cman" make rpm -+# eg. make WITH="--with pre_release" rpm - mock-%: - make srpm-$(firstword $(shell echo $(@:mock-%=%) | tr '-' ' ')) - -rm -rf $(RPM_ROOT)/mock --- -1.8.3.1 - - -From 42ff7282a7e6ea65c8ea861c1f52c50b8f3959e1 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 23 Jan 2019 14:53:31 -0600 -Subject: [PATCH 45/69] Test: cts-exec-helper: allow multiple -V arguments - -allows for future use by cts-exec ---- - daemons/execd/cts-exec-helper.c | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) - -diff --git a/daemons/execd/cts-exec-helper.c b/daemons/execd/cts-exec-helper.c -index 76e6ca4..90e5622 100644 ---- a/daemons/execd/cts-exec-helper.c -+++ b/daemons/execd/cts-exec-helper.c -@@ -1,5 +1,5 @@ - /* -- * Copyright 2012-2018 David Vossel -+ * Copyright 2012-2019 David Vossel - * - * This source code is licensed under the GNU Lesser General Public License - * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -@@ -469,7 +469,8 @@ main(int argc, char **argv) - crm_help(flag, CRM_EX_OK); - break; - case 'V': -- options.verbose = 1; -+ ++options.verbose; -+ crm_bump_log_level(argc, argv); - break; - case 'Q': - options.quiet = 1; --- -1.8.3.1 - - -From 18fb7cea3226e13b21a5b147f9bbd60f764f028e Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 22 Jan 2019 13:15:45 -0600 -Subject: [PATCH 46/69] Log: libcrmservice: improve logs when cleaning up an - operation - -Previously, services_action_cleanup() got the sense of -dbus_pending_call_get_completed() wrong when logging a "did not complete" -message. It didn't really matter because the true case should never happen, -but this corrects it anyway. - -This also slightly refactors to use services_set_op_pending() to clear the -pending call, to reduce code duplication, with the only real effect being to -log additional trace messages. ---- - lib/services/services.c | 17 ++++++++++------- - 1 file changed, 10 insertions(+), 7 deletions(-) - -diff --git a/lib/services/services.c b/lib/services/services.c -index debe9ec..8b22d4a 100644 ---- a/lib/services/services.c -+++ b/lib/services/services.c -@@ -1,5 +1,5 @@ - /* -- * Copyright 2010-2018 Andrew Beekhof -+ * Copyright 2010-2019 Andrew Beekhof - * - * This source code is licensed under the GNU Lesser General Public License - * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -@@ -460,7 +460,7 @@ services_set_op_pending(svc_action_t *op, DBusPendingCall *pending) - void - services_action_cleanup(svc_action_t * op) - { -- if(op->opaque == NULL) { -+ if ((op == NULL) || (op->opaque == NULL)) { - return; - } - -@@ -472,13 +472,16 @@ services_action_cleanup(svc_action_t * op) - } - - if(op->opaque->pending) { -- crm_trace("Cleaning up pending dbus call %p %s for %s", op->opaque->pending, op->action, op->rsc); -- if(dbus_pending_call_get_completed(op->opaque->pending)) { -- crm_warn("Pending dbus call %s for %s did not complete", op->action, op->rsc); -+ if (dbus_pending_call_get_completed(op->opaque->pending)) { -+ // This should never be the case -+ crm_warn("Result of %s op %s was unhandled", -+ op->standard, op->id); -+ } else { -+ crm_debug("Will ignore any result of canceled %s op %s", -+ op->standard, op->id); - } - dbus_pending_call_cancel(op->opaque->pending); -- dbus_pending_call_unref(op->opaque->pending); -- op->opaque->pending = NULL; -+ services_set_op_pending(op, NULL); - } - #endif - --- -1.8.3.1 - - -From a70e50988f859b352b72e75a051188e01090cf3b Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 23 Jan 2019 14:15:39 -0600 -Subject: [PATCH 47/69] Log: libcrmservice: null-terminate string *before* - printing it - ---- - lib/services/services_linux.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c -index bd22777..e2685e9 100644 ---- a/lib/services/services_linux.c -+++ b/lib/services/services_linux.c -@@ -1,5 +1,5 @@ - /* -- * Copyright (C) 2010-2016 Andrew Beekhof -+ * Copyright 2010-2019 Andrew Beekhof - * - * This source code is licensed under the GNU Lesser General Public License - * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -@@ -67,8 +67,8 @@ svc_read_output(int fd, svc_action_t * op, bool is_stderr) - do { - rc = read(fd, buf, buf_read_len); - if (rc > 0) { -- crm_trace("Got %d chars: %.80s", rc, buf); - buf[rc] = 0; -+ crm_trace("Got %d chars: %.80s", rc, buf); - data = realloc_safe(data, len + rc + 1); - len += sprintf(data + len, "%s", buf); - --- -1.8.3.1 - - -From ab5e25a2ab840e452d2d4d77694356b594e47247 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 22 Jan 2019 13:16:02 -0600 -Subject: [PATCH 48/69] Fix: libcrmservice: cancel DBus call when cancelling - systemd/upstart actions - -deabcc5a was incomplete. When cancelling a recurring systemd or upstart -operation, we should cancel any pending DBus call immediately (by telling DBus -to ignore any result), rather than wait for it to complete. ---- - lib/services/services.c | 26 +++++++++++++++----------- - 1 file changed, 15 insertions(+), 11 deletions(-) - -diff --git a/lib/services/services.c b/lib/services/services.c -index 8b22d4a..20e824f 100644 ---- a/lib/services/services.c -+++ b/lib/services/services.c -@@ -591,7 +591,7 @@ services_action_cancel(const char *name, const char *action, guint interval_ms) - /* Tell operation_finalize() not to reschedule the operation */ - op->cancel = TRUE; - -- /* Stop tracking it as a recurring operation, and stop its timer */ -+ /* Stop tracking it as a recurring operation, and stop its repeat timer */ - cancel_recurring_action(op); - - /* If the op has a PID, it's an in-flight child process, so kill it. -@@ -610,19 +610,22 @@ services_action_cancel(const char *name, const char *action, guint interval_ms) - goto done; - } - -- /* In-flight systemd and upstart ops don't have a pid. The relevant handlers -- * will call operation_finalize() when the operation completes. -- * @TODO: Can we request early termination, maybe using -- * dbus_pending_call_cancel()? -- */ -+#if SUPPORT_DBUS -+ // In-flight systemd and upstart ops don't have a pid - if (inflight_systemd_or_upstart(op)) { -- crm_info("Will cancel %s op %s when in-flight instance completes", -- op->standard, op->id); -- cancelled = FALSE; -- goto done; -+ inflight_ops = g_list_remove(inflight_ops, op); -+ -+ /* This will cause any result that comes in later to be discarded, so we -+ * don't call the callback and free the operation twice. -+ */ -+ services_action_cleanup(op); - } -+#endif -+ -+ // The rest of this is essentially equivalent to operation_finalize(), -+ // except without calling handle_blocked_ops() - -- /* Otherwise, operation is not in-flight, just report as cancelled */ -+ // Report operation as cancelled - op->status = PCMK_LRM_OP_CANCELLED; - if (op->opaque->callback) { - op->opaque->callback(op); -@@ -631,6 +634,7 @@ services_action_cancel(const char *name, const char *action, guint interval_ms) - blocked_ops = g_list_remove(blocked_ops, op); - services_action_free(op); - cancelled = TRUE; -+ // @TODO Initiate handle_blocked_ops() asynchronously - - done: - free(id); --- -1.8.3.1 - - -From 8001619faefb1bf0db386953b61b491d39a04548 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 22 Jan 2019 16:22:32 -0600 -Subject: [PATCH 49/69] Refactor: scheduler: functionize creating shutdown op - -for readability. Also tweak log message. ---- - daemons/schedulerd/sched_allocate.c | 12 ++---------- - daemons/schedulerd/sched_utils.c | 29 ++++++++++++++++++++++++++++- - daemons/schedulerd/sched_utils.h | 3 ++- - 3 files changed, 32 insertions(+), 12 deletions(-) - -diff --git a/daemons/schedulerd/sched_allocate.c b/daemons/schedulerd/sched_allocate.c -index 7d62730..5b28e30 100644 ---- a/daemons/schedulerd/sched_allocate.c -+++ b/daemons/schedulerd/sched_allocate.c -@@ -1,5 +1,5 @@ - /* -- * Copyright 2004-2018 Andrew Beekhof -+ * Copyright 2004-2019 Andrew Beekhof - * - * This source code is licensed under the GNU General Public License version 2 - * or later (GPLv2+) WITHOUT ANY WARRANTY. -@@ -1593,15 +1593,7 @@ stage6(pe_working_set_t * data_set) - * if we can come up with a good use for this in the future, we will. */ - is_remote_node(node) == FALSE) { - -- action_t *down_op = NULL; -- -- crm_notice("Scheduling Node %s for shutdown", node->details->uname); -- -- down_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_SHUTDOWN, node->details->uname), -- CRM_OP_SHUTDOWN, node, FALSE, TRUE, data_set); -- -- shutdown_constraints(node, down_op, data_set); -- add_hash_param(down_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE); -+ action_t *down_op = sched_shutdown_op(node, data_set); - - if (node->details->is_dc) { - dc_down = down_op; -diff --git a/daemons/schedulerd/sched_utils.c b/daemons/schedulerd/sched_utils.c -index 153dc27..613dcf3 100644 ---- a/daemons/schedulerd/sched_utils.c -+++ b/daemons/schedulerd/sched_utils.c -@@ -1,5 +1,5 @@ - /* -- * Copyright 2004-2018 Andrew Beekhof -+ * Copyright 2004-2019 Andrew Beekhof - * - * This source code is licensed under the GNU General Public License version 2 - * or later (GPLv2+) WITHOUT ANY WARRANTY. -@@ -442,3 +442,30 @@ pe_cancel_op(pe_resource_t *rsc, const char *task, guint interval_ms, - - return cancel_op; - } -+ -+/*! -+ * \internal -+ * \brief Create a shutdown op for a scheduler transition -+ * -+ * \param[in] rsc Resource of action to cancel -+ * \param[in] task Name of action to cancel -+ * \param[in] interval_ms Interval of action to cancel -+ * \param[in] node Node of action to cancel -+ * \param[in] data_set Working set of cluster -+ * -+ * \return Created op -+ */ -+pe_action_t * -+sched_shutdown_op(pe_node_t *node, pe_working_set_t *data_set) -+{ -+ char *shutdown_id = crm_strdup_printf("%s-%s", CRM_OP_SHUTDOWN, -+ node->details->uname); -+ -+ pe_action_t *shutdown_op = custom_action(NULL, shutdown_id, CRM_OP_SHUTDOWN, -+ node, FALSE, TRUE, data_set); -+ -+ crm_notice("Scheduling shutdown of node %s", node->details->uname); -+ shutdown_constraints(node, shutdown_op, data_set); -+ add_hash_param(shutdown_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE); -+ return shutdown_op; -+} -diff --git a/daemons/schedulerd/sched_utils.h b/daemons/schedulerd/sched_utils.h -index b049dd7..6c3cb46 100644 ---- a/daemons/schedulerd/sched_utils.h -+++ b/daemons/schedulerd/sched_utils.h -@@ -1,5 +1,5 @@ - /* -- * Copyright 2004-2018 Andrew Beekhof -+ * Copyright 2004-2019 Andrew Beekhof - * - * This source code is licensed under the GNU Lesser General Public License - * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -@@ -70,6 +70,7 @@ pe_action_t *create_pseudo_resource_op(resource_t * rsc, const char *task, bool - pe_action_t *pe_cancel_op(pe_resource_t *rsc, const char *name, - guint interval_ms, pe_node_t *node, - pe_working_set_t *data_set); -+pe_action_t *sched_shutdown_op(pe_node_t *node, pe_working_set_t *data_set); - - # define LOAD_STOPPED "load_stopped" - --- -1.8.3.1 - - -From 78836c36123ce4d02b27a0ae3a9d74c6f5dc85d4 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 22 Jan 2019 16:29:36 -0600 -Subject: [PATCH 50/69] Fix: scheduler: don't disable waiting for DC fencing - -34339dac intended to disable waiting for shutdowns, but it also applied to -fencing. Waiting is already disable for shutdowns when creating the op, so -the extra line was redundant anyway. ---- - daemons/schedulerd/sched_allocate.c | 2 -- - 1 file changed, 2 deletions(-) - -diff --git a/daemons/schedulerd/sched_allocate.c b/daemons/schedulerd/sched_allocate.c -index 5b28e30..13cbadc 100644 ---- a/daemons/schedulerd/sched_allocate.c -+++ b/daemons/schedulerd/sched_allocate.c -@@ -1623,8 +1623,6 @@ stage6(pe_working_set_t * data_set) - crm_trace("Ordering shutdowns before %s on %s (DC)", - dc_down->task, dc_down->node->details->uname); - -- add_hash_param(dc_down->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE); -- - for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { - action_t *node_stop = (action_t *) gIter->data; - --- -1.8.3.1 - - -From 9d2a7f0abef0566ee88c94688a4ae4653d81d823 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 22 Jan 2019 16:35:12 -0600 -Subject: [PATCH 51/69] Test: cts-scheduler: update regression tests for DC - stonith change - ---- - cts/scheduler/concurrent-fencing.exp | 2 +- - cts/scheduler/migrate-fencing.exp | 2 +- - cts/scheduler/per-op-failcount.exp | 2 +- - cts/scheduler/rec-node-13.exp | 2 +- - cts/scheduler/rec-node-14.exp | 2 +- - cts/scheduler/rec-node-15.exp | 2 +- - cts/scheduler/stonith-0.exp | 2 +- - cts/scheduler/suicide-needed-inquorate.exp | 2 +- - cts/scheduler/systemhealth1.exp | 2 +- - cts/scheduler/systemhealthm1.exp | 2 +- - cts/scheduler/systemhealthn1.exp | 2 +- - cts/scheduler/systemhealtho1.exp | 2 +- - cts/scheduler/systemhealthp1.exp | 2 +- - 13 files changed, 13 insertions(+), 13 deletions(-) - -diff --git a/cts/scheduler/concurrent-fencing.exp b/cts/scheduler/concurrent-fencing.exp -index f479556..23295e3 100644 ---- a/cts/scheduler/concurrent-fencing.exp -+++ b/cts/scheduler/concurrent-fencing.exp -@@ -13,7 +13,7 @@ - - - -- -+ - - - -diff --git a/cts/scheduler/migrate-fencing.exp b/cts/scheduler/migrate-fencing.exp -index 0a5103a..84596e7 100644 ---- a/cts/scheduler/migrate-fencing.exp -+++ b/cts/scheduler/migrate-fencing.exp -@@ -623,7 +623,7 @@ - - - -- -+ - - - -diff --git a/cts/scheduler/per-op-failcount.exp b/cts/scheduler/per-op-failcount.exp -index d86e486..2a4eec5 100644 ---- a/cts/scheduler/per-op-failcount.exp -+++ b/cts/scheduler/per-op-failcount.exp -@@ -64,7 +64,7 @@ - - - -- -+ - - - -diff --git a/cts/scheduler/rec-node-13.exp b/cts/scheduler/rec-node-13.exp -index 8c5593b..782475d 100644 ---- a/cts/scheduler/rec-node-13.exp -+++ b/cts/scheduler/rec-node-13.exp -@@ -44,7 +44,7 @@ - - - -- -+ - - - -diff --git a/cts/scheduler/rec-node-14.exp b/cts/scheduler/rec-node-14.exp -index e95918f..5520755 100644 ---- a/cts/scheduler/rec-node-14.exp -+++ b/cts/scheduler/rec-node-14.exp -@@ -17,7 +17,7 @@ - - - -- -+ - - - -diff --git a/cts/scheduler/rec-node-15.exp b/cts/scheduler/rec-node-15.exp -index 1207b64..c3ee5b7 100644 ---- a/cts/scheduler/rec-node-15.exp -+++ b/cts/scheduler/rec-node-15.exp -@@ -440,7 +440,7 @@ - - - -- -+ - - - -diff --git a/cts/scheduler/stonith-0.exp b/cts/scheduler/stonith-0.exp -index 92853d8..6d286d3 100644 ---- a/cts/scheduler/stonith-0.exp -+++ b/cts/scheduler/stonith-0.exp -@@ -395,7 +395,7 @@ - - - -- -+ - - - -diff --git a/cts/scheduler/suicide-needed-inquorate.exp b/cts/scheduler/suicide-needed-inquorate.exp -index 58619aa..bd0b69c 100644 ---- a/cts/scheduler/suicide-needed-inquorate.exp -+++ b/cts/scheduler/suicide-needed-inquorate.exp -@@ -17,7 +17,7 @@ - - - -- -+ - - - -diff --git a/cts/scheduler/systemhealth1.exp b/cts/scheduler/systemhealth1.exp -index fb8764c..ac8e6ad 100644 ---- a/cts/scheduler/systemhealth1.exp -+++ b/cts/scheduler/systemhealth1.exp -@@ -13,7 +13,7 @@ - - - -- -+ - - - -diff --git a/cts/scheduler/systemhealthm1.exp b/cts/scheduler/systemhealthm1.exp -index fb8764c..ac8e6ad 100644 ---- a/cts/scheduler/systemhealthm1.exp -+++ b/cts/scheduler/systemhealthm1.exp -@@ -13,7 +13,7 @@ - - - -- -+ - - - -diff --git a/cts/scheduler/systemhealthn1.exp b/cts/scheduler/systemhealthn1.exp -index fb8764c..ac8e6ad 100644 ---- a/cts/scheduler/systemhealthn1.exp -+++ b/cts/scheduler/systemhealthn1.exp -@@ -13,7 +13,7 @@ - - - -- -+ - - - -diff --git a/cts/scheduler/systemhealtho1.exp b/cts/scheduler/systemhealtho1.exp -index fb8764c..ac8e6ad 100644 ---- a/cts/scheduler/systemhealtho1.exp -+++ b/cts/scheduler/systemhealtho1.exp -@@ -13,7 +13,7 @@ - - - -- -+ - - - -diff --git a/cts/scheduler/systemhealthp1.exp b/cts/scheduler/systemhealthp1.exp -index fb8764c..ac8e6ad 100644 ---- a/cts/scheduler/systemhealthp1.exp -+++ b/cts/scheduler/systemhealthp1.exp -@@ -13,7 +13,7 @@ - - - -- -+ - - - --- -1.8.3.1 - - -From eb78c0451eda1b25310bc2b4e59480ecf89f63b8 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 22 Jan 2019 17:03:09 -0600 -Subject: [PATCH 52/69] Refactor: scheduler: improve fence action ordering - -Remove redundant code (DC fence actions are not added to the remembered lists, -so there is no need to check them against the DC later), add more comments, -and use g_list_prepend() instead of g_list_append() for efficiency. ---- - daemons/schedulerd/sched_allocate.c | 57 ++++++++++++++++++++----------------- - 1 file changed, 31 insertions(+), 26 deletions(-) - -diff --git a/daemons/schedulerd/sched_allocate.c b/daemons/schedulerd/sched_allocate.c -index 13cbadc..2e13f8a 100644 ---- a/daemons/schedulerd/sched_allocate.c -+++ b/daemons/schedulerd/sched_allocate.c -@@ -1527,7 +1527,6 @@ stage6(pe_working_set_t * data_set) - { - action_t *dc_down = NULL; - action_t *stonith_op = NULL; -- action_t *last_stonith = NULL; - gboolean integrity_lost = FALSE; - gboolean need_stonith = TRUE; - GListPtr gIter; -@@ -1575,16 +1574,24 @@ stage6(pe_working_set_t * data_set) - stonith_constraints(node, stonith_op, data_set); - - if (node->details->is_dc) { -+ // Remember if the DC is being fenced - dc_down = stonith_op; - -- } else if (is_set(data_set->flags, pe_flag_concurrent_fencing) == FALSE) { -- if (last_stonith) { -- order_actions(last_stonith, stonith_op, pe_order_optional); -+ } else { -+ -+ if (is_not_set(data_set->flags, pe_flag_concurrent_fencing) -+ && (stonith_ops != NULL)) { -+ /* Concurrent fencing is disabled, so order each non-DC -+ * fencing in a chain. If there is any DC fencing or -+ * shutdown, it will be ordered after the last action in the -+ * chain later. -+ */ -+ order_actions((pe_action_t *) stonith_ops->data, -+ stonith_op, pe_order_optional); - } -- last_stonith = stonith_op; - -- } else { -- stonith_ops = g_list_append(stonith_ops, stonith_op); -+ // Remember all non-DC fencing actions in a separate list -+ stonith_ops = g_list_prepend(stonith_ops, stonith_op); - } - - } else if (node->details->online && node->details->shutdown && -@@ -1618,11 +1625,6 @@ stage6(pe_working_set_t * data_set) - } - - if (dc_down != NULL) { -- GListPtr gIter = NULL; -- -- crm_trace("Ordering shutdowns before %s on %s (DC)", -- dc_down->task, dc_down->node->details->uname); -- - for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { - action_t *node_stop = (action_t *) gIter->data; - -@@ -1632,28 +1634,31 @@ stage6(pe_working_set_t * data_set) - continue; - } - -- crm_debug("Ordering shutdown on %s before %s on %s", -+ crm_debug("Ordering shutdown on %s before %s on DC %s", - node_stop->node->details->uname, - dc_down->task, dc_down->node->details->uname); - - order_actions(node_stop, dc_down, pe_order_optional); - } - -- if (last_stonith) { -- if (dc_down != last_stonith) { -- order_actions(last_stonith, dc_down, pe_order_optional); -- } -+ // Order any non-DC fencing before any DC fencing or shutdown - -- } else { -- GListPtr gIter2 = NULL; -- -- for (gIter2 = stonith_ops; gIter2 != NULL; gIter2 = gIter2->next) { -- stonith_op = (action_t *) gIter2->data; -- -- if (dc_down != stonith_op) { -- order_actions(stonith_op, dc_down, pe_order_optional); -- } -+ if (is_set(data_set->flags, pe_flag_concurrent_fencing)) { -+ /* With concurrent fencing, order each non-DC fencing action -+ * separately before any DC fencing or shutdown. -+ */ -+ for (gIter = stonith_ops; gIter != NULL; gIter = gIter->next) { -+ order_actions((pe_action_t *) gIter->data, dc_down, -+ pe_order_optional); - } -+ } else if (stonith_ops) { -+ /* Without concurrent fencing, the non-DC fencing actions are -+ * already ordered relative to each other, so we just need to order -+ * the DC fencing after the last action in the chain (which is the -+ * first item in the list). -+ */ -+ order_actions((pe_action_t *) stonith_ops->data, dc_down, -+ pe_order_optional); - } - } - g_list_free(stonith_ops); --- -1.8.3.1 - - -From 6b163036a275597bc738cb158075bf51b90ec440 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 22 Jan 2019 19:06:39 -0600 -Subject: [PATCH 53/69] Fix: scheduler: don't order non-DC shutdowns before DC - fencing - -because it can cause graph loops ---- - daemons/schedulerd/sched_allocate.c | 33 +++++++++++++++++++++------------ - 1 file changed, 21 insertions(+), 12 deletions(-) - -diff --git a/daemons/schedulerd/sched_allocate.c b/daemons/schedulerd/sched_allocate.c -index 2e13f8a..912a38a 100644 ---- a/daemons/schedulerd/sched_allocate.c -+++ b/daemons/schedulerd/sched_allocate.c -@@ -1531,6 +1531,7 @@ stage6(pe_working_set_t * data_set) - gboolean need_stonith = TRUE; - GListPtr gIter; - GListPtr stonith_ops = NULL; -+ GList *shutdown_ops = NULL; - - /* Remote ordering constraints need to happen prior to calculate - * fencing because it is one more place we will mark the node as -@@ -1603,7 +1604,11 @@ stage6(pe_working_set_t * data_set) - action_t *down_op = sched_shutdown_op(node, data_set); - - if (node->details->is_dc) { -+ // Remember if the DC is being shut down - dc_down = down_op; -+ } else { -+ // Remember non-DC shutdowns for later ordering -+ shutdown_ops = g_list_prepend(shutdown_ops, down_op); - } - } - -@@ -1625,20 +1630,23 @@ stage6(pe_working_set_t * data_set) - } - - if (dc_down != NULL) { -- for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { -- action_t *node_stop = (action_t *) gIter->data; -- -- if (safe_str_neq(CRM_OP_SHUTDOWN, node_stop->task)) { -- continue; -- } else if (node_stop->node->details->is_dc) { -- continue; -- } -+ /* Order any non-DC shutdowns before any DC shutdown, to avoid repeated -+ * DC elections. However, we don't want to order non-DC shutdowns before -+ * a DC *fencing*, because even though we don't want a node that's -+ * shutting down to become DC, the DC fencing could be ordered before a -+ * clone stop that's also ordered before the shutdowns, thus leading to -+ * a graph loop. -+ */ -+ if (safe_str_eq(dc_down->task, CRM_OP_SHUTDOWN)) { -+ for (gIter = shutdown_ops; gIter != NULL; gIter = gIter->next) { -+ action_t *node_stop = (action_t *) gIter->data; - -- crm_debug("Ordering shutdown on %s before %s on DC %s", -- node_stop->node->details->uname, -- dc_down->task, dc_down->node->details->uname); -+ crm_debug("Ordering shutdown on %s before %s on DC %s", -+ node_stop->node->details->uname, -+ dc_down->task, dc_down->node->details->uname); - -- order_actions(node_stop, dc_down, pe_order_optional); -+ order_actions(node_stop, dc_down, pe_order_optional); -+ } - } - - // Order any non-DC fencing before any DC fencing or shutdown -@@ -1662,6 +1670,7 @@ stage6(pe_working_set_t * data_set) - } - } - g_list_free(stonith_ops); -+ g_list_free(shutdown_ops); - return TRUE; - } - --- -1.8.3.1 - - -From 493f87ce9d9b122e0fb2c0f83dae2a9b32e648ed Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 22 Jan 2019 19:18:53 -0600 -Subject: [PATCH 54/69] Test: scheduler: add regression test for DC fencing - ordering - ---- - cts/cts-scheduler.in | 1 + - cts/scheduler/dc-fence-ordering.dot | 49 +++ - cts/scheduler/dc-fence-ordering.exp | 259 +++++++++++++++ - cts/scheduler/dc-fence-ordering.scores | 202 ++++++++++++ - cts/scheduler/dc-fence-ordering.summary | 83 +++++ - cts/scheduler/dc-fence-ordering.xml | 551 ++++++++++++++++++++++++++++++++ - 6 files changed, 1145 insertions(+) - create mode 100644 cts/scheduler/dc-fence-ordering.dot - create mode 100644 cts/scheduler/dc-fence-ordering.exp - create mode 100644 cts/scheduler/dc-fence-ordering.scores - create mode 100644 cts/scheduler/dc-fence-ordering.summary - create mode 100644 cts/scheduler/dc-fence-ordering.xml - -diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in -index 8b6b2ac..6bf16da 100644 ---- a/cts/cts-scheduler.in -+++ b/cts/cts-scheduler.in -@@ -884,6 +884,7 @@ do_test stonith-1 "Stonith loop - 2" - do_test stonith-2 "Stonith loop - 3" - do_test stonith-3 "Stonith startup" - do_test stonith-4 "Stonith node state" -+do_test dc-fence-ordering "DC needs fencing while other nodes are shutting down" - do_test bug-1572-1 "Recovery of groups depending on master/slave" - do_test bug-1572-2 "Recovery of groups depending on master/slave when the master is never re-promoted" - do_test bug-1685 "Depends-on-master ordering" -diff --git a/cts/scheduler/dc-fence-ordering.dot b/cts/scheduler/dc-fence-ordering.dot -new file mode 100644 -index 0000000..b700755 ---- /dev/null -+++ b/cts/scheduler/dc-fence-ordering.dot -@@ -0,0 +1,49 @@ -+digraph "g" { -+"do_shutdown rhel7-2" [ style=bold color="green" fontcolor="black"] -+"do_shutdown rhel7-4" [ style=bold color="green" fontcolor="black"] -+"do_shutdown rhel7-5" [ style=bold color="green" fontcolor="black"] -+"group-1_stop_0" -> "group-1_stopped_0" [ style = bold] -+"group-1_stop_0" -> "petulant_stop_0 rhel7-1" [ style = bold] -+"group-1_stop_0" -> "r192.168.122.207_stop_0 rhel7-1" [ style = bold] -+"group-1_stop_0" [ style=bold color="green" fontcolor="orange"] -+"group-1_stopped_0" -> "promotable-1_demote_0" [ style = bold] -+"group-1_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"petulant_stop_0 rhel7-1" -> "group-1_stopped_0" [ style = bold] -+"petulant_stop_0 rhel7-1" -> "r192.168.122.207_stop_0 rhel7-1" [ style = bold] -+"petulant_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] -+"promotable-1_demote_0" -> "promotable-1_demoted_0" [ style = bold] -+"promotable-1_demote_0" -> "stateful-1_demote_0 rhel7-1" [ style = bold] -+"promotable-1_demote_0" [ style=bold color="green" fontcolor="orange"] -+"promotable-1_demoted_0" -> "promotable-1_stop_0" [ style = bold] -+"promotable-1_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"promotable-1_stop_0" -> "promotable-1_stopped_0" [ style = bold] -+"promotable-1_stop_0" -> "stateful-1_stop_0 rhel7-1" [ style = bold] -+"promotable-1_stop_0" -> "stateful-1_stop_0 rhel7-2" [ style = bold] -+"promotable-1_stop_0" -> "stateful-1_stop_0 rhel7-4" [ style = bold] -+"promotable-1_stop_0" -> "stateful-1_stop_0 rhel7-5" [ style = bold] -+"promotable-1_stop_0" [ style=bold color="green" fontcolor="orange"] -+"promotable-1_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"r192.168.122.207_stop_0 rhel7-1" -> "group-1_stopped_0" [ style = bold] -+"r192.168.122.207_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] -+"stateful-1_demote_0 rhel7-1" -> "promotable-1_demoted_0" [ style = bold] -+"stateful-1_demote_0 rhel7-1" -> "stateful-1_stop_0 rhel7-1" [ style = bold] -+"stateful-1_demote_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] -+"stateful-1_stop_0 rhel7-1" -> "promotable-1_stopped_0" [ style = bold] -+"stateful-1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] -+"stateful-1_stop_0 rhel7-2" -> "do_shutdown rhel7-2" [ style = bold] -+"stateful-1_stop_0 rhel7-2" -> "promotable-1_stopped_0" [ style = bold] -+"stateful-1_stop_0 rhel7-2" [ style=bold color="green" fontcolor="black"] -+"stateful-1_stop_0 rhel7-4" -> "do_shutdown rhel7-4" [ style = bold] -+"stateful-1_stop_0 rhel7-4" -> "promotable-1_stopped_0" [ style = bold] -+"stateful-1_stop_0 rhel7-4" [ style=bold color="green" fontcolor="black"] -+"stateful-1_stop_0 rhel7-5" -> "do_shutdown rhel7-5" [ style = bold] -+"stateful-1_stop_0 rhel7-5" -> "promotable-1_stopped_0" [ style = bold] -+"stateful-1_stop_0 rhel7-5" [ style=bold color="green" fontcolor="black"] -+"stonith 'reboot' rhel7-1" -> "group-1_stop_0" [ style = bold] -+"stonith 'reboot' rhel7-1" -> "petulant_stop_0 rhel7-1" [ style = bold] -+"stonith 'reboot' rhel7-1" -> "promotable-1_stop_0" [ style = bold] -+"stonith 'reboot' rhel7-1" -> "r192.168.122.207_stop_0 rhel7-1" [ style = bold] -+"stonith 'reboot' rhel7-1" -> "stateful-1_demote_0 rhel7-1" [ style = bold] -+"stonith 'reboot' rhel7-1" -> "stateful-1_stop_0 rhel7-1" [ style = bold] -+"stonith 'reboot' rhel7-1" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/dc-fence-ordering.exp b/cts/scheduler/dc-fence-ordering.exp -new file mode 100644 -index 0000000..429db5a ---- /dev/null -+++ b/cts/scheduler/dc-fence-ordering.exp -@@ -0,0 +1,259 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/dc-fence-ordering.scores b/cts/scheduler/dc-fence-ordering.scores -new file mode 100644 -index 0000000..92cdb26 ---- /dev/null -+++ b/cts/scheduler/dc-fence-ordering.scores -@@ -0,0 +1,202 @@ -+Allocation scores: -+Using the original execution date of: 2018-11-28 18:37:16Z -+clone_color: Connectivity allocation score on rhel7-1: 0 -+clone_color: Connectivity allocation score on rhel7-2: 0 -+clone_color: Connectivity allocation score on rhel7-3: 0 -+clone_color: Connectivity allocation score on rhel7-4: 0 -+clone_color: Connectivity allocation score on rhel7-5: 0 -+clone_color: ping-1:0 allocation score on rhel7-1: 0 -+clone_color: ping-1:0 allocation score on rhel7-2: 0 -+clone_color: ping-1:0 allocation score on rhel7-3: 0 -+clone_color: ping-1:0 allocation score on rhel7-4: 0 -+clone_color: ping-1:0 allocation score on rhel7-5: 0 -+clone_color: ping-1:1 allocation score on rhel7-1: 0 -+clone_color: ping-1:1 allocation score on rhel7-2: 0 -+clone_color: ping-1:1 allocation score on rhel7-3: 0 -+clone_color: ping-1:1 allocation score on rhel7-4: 0 -+clone_color: ping-1:1 allocation score on rhel7-5: 0 -+clone_color: ping-1:2 allocation score on rhel7-1: 0 -+clone_color: ping-1:2 allocation score on rhel7-2: 0 -+clone_color: ping-1:2 allocation score on rhel7-3: 0 -+clone_color: ping-1:2 allocation score on rhel7-4: 0 -+clone_color: ping-1:2 allocation score on rhel7-5: 0 -+clone_color: ping-1:3 allocation score on rhel7-1: 0 -+clone_color: ping-1:3 allocation score on rhel7-2: 0 -+clone_color: ping-1:3 allocation score on rhel7-3: 0 -+clone_color: ping-1:3 allocation score on rhel7-4: 0 -+clone_color: ping-1:3 allocation score on rhel7-5: 0 -+clone_color: ping-1:4 allocation score on rhel7-1: 0 -+clone_color: ping-1:4 allocation score on rhel7-2: 0 -+clone_color: ping-1:4 allocation score on rhel7-3: 0 -+clone_color: ping-1:4 allocation score on rhel7-4: 0 -+clone_color: ping-1:4 allocation score on rhel7-5: 0 -+clone_color: promotable-1 allocation score on rhel7-1: -INFINITY -+clone_color: promotable-1 allocation score on rhel7-2: -INFINITY -+clone_color: promotable-1 allocation score on rhel7-3: -INFINITY -+clone_color: promotable-1 allocation score on rhel7-4: -INFINITY -+clone_color: promotable-1 allocation score on rhel7-5: -INFINITY -+clone_color: stateful-1:0 allocation score on rhel7-1: -INFINITY -+clone_color: stateful-1:0 allocation score on rhel7-2: -INFINITY -+clone_color: stateful-1:0 allocation score on rhel7-3: -INFINITY -+clone_color: stateful-1:0 allocation score on rhel7-4: -INFINITY -+clone_color: stateful-1:0 allocation score on rhel7-5: -INFINITY -+clone_color: stateful-1:1 allocation score on rhel7-1: -INFINITY -+clone_color: stateful-1:1 allocation score on rhel7-2: -INFINITY -+clone_color: stateful-1:1 allocation score on rhel7-3: -INFINITY -+clone_color: stateful-1:1 allocation score on rhel7-4: -INFINITY -+clone_color: stateful-1:1 allocation score on rhel7-5: -INFINITY -+clone_color: stateful-1:2 allocation score on rhel7-1: -INFINITY -+clone_color: stateful-1:2 allocation score on rhel7-2: -INFINITY -+clone_color: stateful-1:2 allocation score on rhel7-3: -INFINITY -+clone_color: stateful-1:2 allocation score on rhel7-4: -INFINITY -+clone_color: stateful-1:2 allocation score on rhel7-5: -INFINITY -+clone_color: stateful-1:3 allocation score on rhel7-1: -INFINITY -+clone_color: stateful-1:3 allocation score on rhel7-2: -INFINITY -+clone_color: stateful-1:3 allocation score on rhel7-3: -INFINITY -+clone_color: stateful-1:3 allocation score on rhel7-4: -INFINITY -+clone_color: stateful-1:3 allocation score on rhel7-5: -INFINITY -+clone_color: stateful-1:4 allocation score on rhel7-1: -INFINITY -+clone_color: stateful-1:4 allocation score on rhel7-2: -INFINITY -+clone_color: stateful-1:4 allocation score on rhel7-3: -INFINITY -+clone_color: stateful-1:4 allocation score on rhel7-4: -INFINITY -+clone_color: stateful-1:4 allocation score on rhel7-5: -INFINITY -+group_color: group-1 allocation score on rhel7-1: 0 -+group_color: group-1 allocation score on rhel7-2: 0 -+group_color: group-1 allocation score on rhel7-3: 0 -+group_color: group-1 allocation score on rhel7-4: 0 -+group_color: group-1 allocation score on rhel7-5: 0 -+group_color: petulant allocation score on rhel7-1: -INFINITY -+group_color: petulant allocation score on rhel7-2: 0 -+group_color: petulant allocation score on rhel7-3: 0 -+group_color: petulant allocation score on rhel7-4: 0 -+group_color: petulant allocation score on rhel7-5: 0 -+group_color: r192.168.122.207 allocation score on rhel7-1: 0 -+group_color: r192.168.122.207 allocation score on rhel7-2: 0 -+group_color: r192.168.122.207 allocation score on rhel7-3: 0 -+group_color: r192.168.122.207 allocation score on rhel7-4: 0 -+group_color: r192.168.122.207 allocation score on rhel7-5: 0 -+group_color: r192.168.122.208 allocation score on rhel7-1: 0 -+group_color: r192.168.122.208 allocation score on rhel7-2: 0 -+group_color: r192.168.122.208 allocation score on rhel7-3: 0 -+group_color: r192.168.122.208 allocation score on rhel7-4: 0 -+group_color: r192.168.122.208 allocation score on rhel7-5: 0 -+native_color: Fencing allocation score on rhel7-1: 0 -+native_color: Fencing allocation score on rhel7-2: 0 -+native_color: Fencing allocation score on rhel7-3: 0 -+native_color: Fencing allocation score on rhel7-4: 0 -+native_color: Fencing allocation score on rhel7-5: 0 -+native_color: FencingFail allocation score on rhel7-1: 0 -+native_color: FencingFail allocation score on rhel7-2: 0 -+native_color: FencingFail allocation score on rhel7-3: 0 -+native_color: FencingFail allocation score on rhel7-4: 0 -+native_color: FencingFail allocation score on rhel7-5: 0 -+native_color: FencingPass allocation score on rhel7-1: 0 -+native_color: FencingPass allocation score on rhel7-2: 0 -+native_color: FencingPass allocation score on rhel7-3: 0 -+native_color: FencingPass allocation score on rhel7-4: 0 -+native_color: FencingPass allocation score on rhel7-5: 0 -+native_color: lsb-dummy allocation score on rhel7-1: -INFINITY -+native_color: lsb-dummy allocation score on rhel7-2: -INFINITY -+native_color: lsb-dummy allocation score on rhel7-3: -INFINITY -+native_color: lsb-dummy allocation score on rhel7-4: -INFINITY -+native_color: lsb-dummy allocation score on rhel7-5: -INFINITY -+native_color: migrator allocation score on rhel7-1: 0 -+native_color: migrator allocation score on rhel7-2: 0 -+native_color: migrator allocation score on rhel7-3: 0 -+native_color: migrator allocation score on rhel7-4: 0 -+native_color: migrator allocation score on rhel7-5: 0 -+native_color: petulant allocation score on rhel7-1: -INFINITY -+native_color: petulant allocation score on rhel7-2: -INFINITY -+native_color: petulant allocation score on rhel7-3: -INFINITY -+native_color: petulant allocation score on rhel7-4: -INFINITY -+native_color: petulant allocation score on rhel7-5: -INFINITY -+native_color: ping-1:0 allocation score on rhel7-1: -INFINITY -+native_color: ping-1:0 allocation score on rhel7-2: -INFINITY -+native_color: ping-1:0 allocation score on rhel7-3: -INFINITY -+native_color: ping-1:0 allocation score on rhel7-4: -INFINITY -+native_color: ping-1:0 allocation score on rhel7-5: -INFINITY -+native_color: ping-1:1 allocation score on rhel7-1: -INFINITY -+native_color: ping-1:1 allocation score on rhel7-2: -INFINITY -+native_color: ping-1:1 allocation score on rhel7-3: -INFINITY -+native_color: ping-1:1 allocation score on rhel7-4: -INFINITY -+native_color: ping-1:1 allocation score on rhel7-5: -INFINITY -+native_color: ping-1:2 allocation score on rhel7-1: -INFINITY -+native_color: ping-1:2 allocation score on rhel7-2: -INFINITY -+native_color: ping-1:2 allocation score on rhel7-3: -INFINITY -+native_color: ping-1:2 allocation score on rhel7-4: -INFINITY -+native_color: ping-1:2 allocation score on rhel7-5: -INFINITY -+native_color: ping-1:3 allocation score on rhel7-1: -INFINITY -+native_color: ping-1:3 allocation score on rhel7-2: -INFINITY -+native_color: ping-1:3 allocation score on rhel7-3: -INFINITY -+native_color: ping-1:3 allocation score on rhel7-4: -INFINITY -+native_color: ping-1:3 allocation score on rhel7-5: -INFINITY -+native_color: ping-1:4 allocation score on rhel7-1: -INFINITY -+native_color: ping-1:4 allocation score on rhel7-2: -INFINITY -+native_color: ping-1:4 allocation score on rhel7-3: -INFINITY -+native_color: ping-1:4 allocation score on rhel7-4: -INFINITY -+native_color: ping-1:4 allocation score on rhel7-5: -INFINITY -+native_color: r192.168.122.207 allocation score on rhel7-1: -INFINITY -+native_color: r192.168.122.207 allocation score on rhel7-2: -INFINITY -+native_color: r192.168.122.207 allocation score on rhel7-3: -INFINITY -+native_color: r192.168.122.207 allocation score on rhel7-4: -INFINITY -+native_color: r192.168.122.207 allocation score on rhel7-5: -INFINITY -+native_color: r192.168.122.208 allocation score on rhel7-1: -INFINITY -+native_color: r192.168.122.208 allocation score on rhel7-2: -INFINITY -+native_color: r192.168.122.208 allocation score on rhel7-3: -INFINITY -+native_color: r192.168.122.208 allocation score on rhel7-4: -INFINITY -+native_color: r192.168.122.208 allocation score on rhel7-5: -INFINITY -+native_color: rsc_rhel7-1 allocation score on rhel7-1: 100 -+native_color: rsc_rhel7-1 allocation score on rhel7-2: 0 -+native_color: rsc_rhel7-1 allocation score on rhel7-3: 0 -+native_color: rsc_rhel7-1 allocation score on rhel7-4: 0 -+native_color: rsc_rhel7-1 allocation score on rhel7-5: 0 -+native_color: rsc_rhel7-2 allocation score on rhel7-1: 0 -+native_color: rsc_rhel7-2 allocation score on rhel7-2: 100 -+native_color: rsc_rhel7-2 allocation score on rhel7-3: 0 -+native_color: rsc_rhel7-2 allocation score on rhel7-4: 0 -+native_color: rsc_rhel7-2 allocation score on rhel7-5: 0 -+native_color: rsc_rhel7-3 allocation score on rhel7-1: 0 -+native_color: rsc_rhel7-3 allocation score on rhel7-2: 0 -+native_color: rsc_rhel7-3 allocation score on rhel7-3: 100 -+native_color: rsc_rhel7-3 allocation score on rhel7-4: 0 -+native_color: rsc_rhel7-3 allocation score on rhel7-5: 0 -+native_color: rsc_rhel7-4 allocation score on rhel7-1: 0 -+native_color: rsc_rhel7-4 allocation score on rhel7-2: 0 -+native_color: rsc_rhel7-4 allocation score on rhel7-3: 0 -+native_color: rsc_rhel7-4 allocation score on rhel7-4: 100 -+native_color: rsc_rhel7-4 allocation score on rhel7-5: 0 -+native_color: rsc_rhel7-5 allocation score on rhel7-1: 0 -+native_color: rsc_rhel7-5 allocation score on rhel7-2: 0 -+native_color: rsc_rhel7-5 allocation score on rhel7-3: 0 -+native_color: rsc_rhel7-5 allocation score on rhel7-4: 0 -+native_color: rsc_rhel7-5 allocation score on rhel7-5: 100 -+native_color: stateful-1:0 allocation score on rhel7-1: -INFINITY -+native_color: stateful-1:0 allocation score on rhel7-2: -INFINITY -+native_color: stateful-1:0 allocation score on rhel7-3: -INFINITY -+native_color: stateful-1:0 allocation score on rhel7-4: -INFINITY -+native_color: stateful-1:0 allocation score on rhel7-5: -INFINITY -+native_color: stateful-1:1 allocation score on rhel7-1: -INFINITY -+native_color: stateful-1:1 allocation score on rhel7-2: -INFINITY -+native_color: stateful-1:1 allocation score on rhel7-3: -INFINITY -+native_color: stateful-1:1 allocation score on rhel7-4: -INFINITY -+native_color: stateful-1:1 allocation score on rhel7-5: -INFINITY -+native_color: stateful-1:2 allocation score on rhel7-1: -INFINITY -+native_color: stateful-1:2 allocation score on rhel7-2: -INFINITY -+native_color: stateful-1:2 allocation score on rhel7-3: -INFINITY -+native_color: stateful-1:2 allocation score on rhel7-4: -INFINITY -+native_color: stateful-1:2 allocation score on rhel7-5: -INFINITY -+native_color: stateful-1:3 allocation score on rhel7-1: -INFINITY -+native_color: stateful-1:3 allocation score on rhel7-2: -INFINITY -+native_color: stateful-1:3 allocation score on rhel7-3: -INFINITY -+native_color: stateful-1:3 allocation score on rhel7-4: -INFINITY -+native_color: stateful-1:3 allocation score on rhel7-5: -INFINITY -+native_color: stateful-1:4 allocation score on rhel7-1: -INFINITY -+native_color: stateful-1:4 allocation score on rhel7-2: -INFINITY -+native_color: stateful-1:4 allocation score on rhel7-3: -INFINITY -+native_color: stateful-1:4 allocation score on rhel7-4: -INFINITY -+native_color: stateful-1:4 allocation score on rhel7-5: -INFINITY -+stateful-1:0 promotion score on none: 0 -+stateful-1:1 promotion score on none: 0 -+stateful-1:2 promotion score on none: 0 -+stateful-1:3 promotion score on none: 0 -+stateful-1:4 promotion score on none: 0 -diff --git a/cts/scheduler/dc-fence-ordering.summary b/cts/scheduler/dc-fence-ordering.summary -new file mode 100644 -index 0000000..82e5096 ---- /dev/null -+++ b/cts/scheduler/dc-fence-ordering.summary -@@ -0,0 +1,83 @@ -+Using the original execution date of: 2018-11-28 18:37:16Z -+ -+Current cluster status: -+Node rhel7-1 (1): UNCLEAN (online) -+Online: [ rhel7-2 rhel7-4 rhel7-5 ] -+OFFLINE: [ rhel7-3 ] -+ -+ Fencing (stonith:fence_xvm): Stopped -+ FencingPass (stonith:fence_dummy): Stopped -+ FencingFail (stonith:fence_dummy): Stopped -+ rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Stopped -+ rsc_rhel7-2 (ocf::heartbeat:IPaddr2): Stopped -+ rsc_rhel7-3 (ocf::heartbeat:IPaddr2): Stopped -+ rsc_rhel7-4 (ocf::heartbeat:IPaddr2): Stopped -+ rsc_rhel7-5 (ocf::heartbeat:IPaddr2): Stopped -+ migrator (ocf::pacemaker:Dummy): Stopped -+ Clone Set: Connectivity [ping-1] -+ Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] -+ Clone Set: promotable-1 [stateful-1] (promotable) -+ Masters: [ rhel7-1 ] -+ Slaves: [ rhel7-2 rhel7-4 rhel7-5 ] -+ Stopped: [ rhel7-3 ] -+ Resource Group: group-1 -+ r192.168.122.207 (ocf::heartbeat:IPaddr2): Started rhel7-1 -+ petulant (service:pacemaker-cts-dummyd@10): FAILED rhel7-1 -+ r192.168.122.208 (ocf::heartbeat:IPaddr2): Stopped -+ lsb-dummy (lsb:LSBDummy): Stopped -+ -+Transition Summary: -+ * Shutdown rhel7-5 -+ * Shutdown rhel7-4 -+ * Shutdown rhel7-2 -+ * Fence (reboot) rhel7-1 'petulant failed there' -+ * Stop stateful-1:0 ( Slave rhel7-5 ) due to node availability -+ * Stop stateful-1:1 ( Master rhel7-1 ) due to node availability -+ * Stop stateful-1:2 ( Slave rhel7-2 ) due to node availability -+ * Stop stateful-1:3 ( Slave rhel7-4 ) due to node availability -+ * Stop r192.168.122.207 ( rhel7-1 ) due to node availability -+ * Stop petulant ( rhel7-1 ) due to node availability -+ -+Executing cluster transition: -+ * Fencing rhel7-1 (reboot) -+ * Pseudo action: group-1_stop_0 -+ * Pseudo action: petulant_stop_0 -+ * Pseudo action: r192.168.122.207_stop_0 -+ * Pseudo action: group-1_stopped_0 -+ * Pseudo action: promotable-1_demote_0 -+ * Pseudo action: stateful-1_demote_0 -+ * Pseudo action: promotable-1_demoted_0 -+ * Pseudo action: promotable-1_stop_0 -+ * Resource action: stateful-1 stop on rhel7-5 -+ * Pseudo action: stateful-1_stop_0 -+ * Resource action: stateful-1 stop on rhel7-2 -+ * Resource action: stateful-1 stop on rhel7-4 -+ * Pseudo action: promotable-1_stopped_0 -+ * Cluster action: do_shutdown on rhel7-5 -+ * Cluster action: do_shutdown on rhel7-4 -+ * Cluster action: do_shutdown on rhel7-2 -+Using the original execution date of: 2018-11-28 18:37:16Z -+ -+Revised cluster status: -+Online: [ rhel7-2 rhel7-4 rhel7-5 ] -+OFFLINE: [ rhel7-1 rhel7-3 ] -+ -+ Fencing (stonith:fence_xvm): Stopped -+ FencingPass (stonith:fence_dummy): Stopped -+ FencingFail (stonith:fence_dummy): Stopped -+ rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Stopped -+ rsc_rhel7-2 (ocf::heartbeat:IPaddr2): Stopped -+ rsc_rhel7-3 (ocf::heartbeat:IPaddr2): Stopped -+ rsc_rhel7-4 (ocf::heartbeat:IPaddr2): Stopped -+ rsc_rhel7-5 (ocf::heartbeat:IPaddr2): Stopped -+ migrator (ocf::pacemaker:Dummy): Stopped -+ Clone Set: Connectivity [ping-1] -+ Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] -+ Clone Set: promotable-1 [stateful-1] (promotable) -+ Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] -+ Resource Group: group-1 -+ r192.168.122.207 (ocf::heartbeat:IPaddr2): Stopped -+ petulant (service:pacemaker-cts-dummyd@10): Stopped -+ r192.168.122.208 (ocf::heartbeat:IPaddr2): Stopped -+ lsb-dummy (lsb:LSBDummy): Stopped -+ -diff --git a/cts/scheduler/dc-fence-ordering.xml b/cts/scheduler/dc-fence-ordering.xml -new file mode 100644 -index 0000000..3ce3204 ---- /dev/null -+++ b/cts/scheduler/dc-fence-ordering.xml -@@ -0,0 +1,551 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ --- -1.8.3.1 - - -From 220006281e5e7a0d886d71255dd4511dc0d163bc Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Sat, 19 Jan 2019 23:17:11 +0100 -Subject: [PATCH 55/69] Tests: cts-cli: simplify+fix regexp to catch - crm_time_as_string's output - -Previously, the cts-cli would only work with a zero-offset time zone -(suggesting that it was only run in the CI that may be expected to -have it like that). Also drop the atypical enumeration of all digits -vs. range capture. ---- - cts/cts-cli.in | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/cts/cts-cli.in b/cts/cts-cli.in -index 34c094e..68e1459 100755 ---- a/cts/cts-cli.in -+++ b/cts/cts-cli.in -@@ -938,7 +938,7 @@ for t in $tests; do - -e 's/^Entity: line [0-9][0-9]*: //'\ - -e 's/\(validation ([0-9][0-9]* of \)[0-9][0-9]*\().*\)/\1X\2/' \ - -e 's/^Migration will take effect until: .*/Migration will take effect until:/' \ -- -e 's/ end=\"[-: 0123456789]*Z\?\"/ end=\"\"/' \ -+ -e 's/ end=\"[0-9][-+: 0-9]*Z*\"/ end=\"\"/' \ - "$TMPFILE" > "${TMPFILE}.$$" - mv -- "${TMPFILE}.$$" "$TMPFILE" - --- -1.8.3.1 - - -From d6f54953ebf9539146378699f96ddd1f1e330311 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Sun, 20 Jan 2019 11:21:05 +0100 -Subject: [PATCH 57/69] Build: spec: swap a scheduler test to avoid glib - 2.59.0+ incompatibility - -Related to the previous "Build: configure+cts_scheduler: allow skipping -hash-table-broken tests" commit. ---- - pacemaker.spec.in | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/pacemaker.spec.in b/pacemaker.spec.in -index 6b2b268..1394e9b 100644 ---- a/pacemaker.spec.in -+++ b/pacemaker.spec.in -@@ -458,7 +458,7 @@ sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool - make %{_smp_mflags} V=1 all - - %check --{ cts/cts-scheduler --run one-or-more-unrunnable-instances \ -+{ cts/cts-scheduler --run load-stopped-loop \ - && cts/cts-cli \ - && touch .CHECKED - } 2>&1 | sed 's/[fF]ail/faiil/g' # prevent false positives in rpmlint --- -1.8.3.1 - - -From 25e14406c763eb52c80487c9ce884cbeca86bbb0 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 24 Jan 2019 11:48:05 -0600 -Subject: [PATCH 58/69] Log: attrd: log previous writer *before* clearing it - ---- - daemons/attrd/attrd_elections.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c -index a7816a9..025dae5 100644 ---- a/daemons/attrd/attrd_elections.c -+++ b/daemons/attrd/attrd_elections.c -@@ -59,10 +59,10 @@ attrd_handle_election_op(const crm_node_t *peer, xmlNode *xml) - - switch(rc) { - case election_start: -- free(peer_writer); -- peer_writer = NULL; - crm_debug("Unsetting writer (was %s) and starting new election", - peer_writer? peer_writer : "unset"); -+ free(peer_writer); -+ peer_writer = NULL; - election_vote(writer); - break; - --- -1.8.3.1 - - -From 5957a2bf9680320e81333b7e58caee93e054b1db Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 24 Jan 2019 17:48:10 -0600 -Subject: [PATCH 59/69] Low: libcrmcluster: *really* write only one election - storm black box - -464d481 didn't do what it thought it did ---- - lib/cluster/election.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/lib/cluster/election.c b/lib/cluster/election.c -index d904075..13c8377 100644 ---- a/lib/cluster/election.c -+++ b/lib/cluster/election.c -@@ -1,5 +1,5 @@ - /* -- * Copyright 2004-2018 Andrew Beekhof -+ * Copyright 2004-2019 Andrew Beekhof - * - * This source code is licensed under the GNU Lesser General Public License - * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -@@ -644,6 +644,7 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) - * write a blackbox on every Nth occurrence. - */ - crm_write_blackbox(0, NULL); -+ wrote_blackbox = TRUE; - } - } - } --- -1.8.3.1 - - -From 818ff56236ea481ee57b1e6da8067524ae943ebb Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 24 Jan 2019 17:49:55 -0600 -Subject: [PATCH 60/69] Refactor: libcrmcluster: move static variables into - election structure - -Makes more sense, and allows for future methods to poke at them - -This would behave differently from the previous code if a second election -object is created. However, that has never been a targeted use case, and the -new behavior would make more sense anyway. ---- - lib/cluster/election.c | 40 +++++++++++++++++++--------------------- - 1 file changed, 19 insertions(+), 21 deletions(-) - -diff --git a/lib/cluster/election.c b/lib/cluster/election.c -index 13c8377..c0ffb47 100644 ---- a/lib/cluster/election.c -+++ b/lib/cluster/election.c -@@ -28,6 +28,10 @@ struct election_s { - GSourceFunc cb; // Function to call if election is won - GHashTable *voted; // Key = node name, value = how node voted - mainloop_timer_t *timeout; // When to abort if all votes not received -+ int election_wins; // Track wins, for storm detection -+ bool wrote_blackbox; // Write a storm blackbox at most once -+ time_t expires; // When storm detection period ends -+ time_t last_election_loss; // When dampening period ends - }; - - static void election_complete(election_t *e) -@@ -531,11 +535,6 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) - time_t tm_now = time(NULL); - struct vote vote; - -- // @TODO these should be in election_t -- static int election_wins = 0; -- static time_t expires = 0; -- static time_t last_election_loss = 0; -- - CRM_CHECK(message != NULL, return election_error); - if (parse_election_message(e, message, &vote) == FALSE) { - return election_error; -@@ -615,24 +614,23 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) - } - } - -- if (expires < tm_now) { -- election_wins = 0; -- expires = tm_now + STORM_INTERVAL; -+ if (e->expires < tm_now) { -+ e->election_wins = 0; -+ e->expires = tm_now + STORM_INTERVAL; - - } else if (done == FALSE && we_lose == FALSE) { - int peers = 1 + g_hash_table_size(crm_peer_cache); -- static bool wrote_blackbox = FALSE; // @TODO move to election_t - - /* If every node has to vote down every other node, thats N*(N-1) total elections - * Allow some leeway before _really_ complaining - */ -- election_wins++; -- if (election_wins > (peers * peers)) { -+ e->election_wins++; -+ if (e->election_wins > (peers * peers)) { - crm_warn("%s election storm detected: %d wins in %d seconds", -- e->name, election_wins, STORM_INTERVAL); -- election_wins = 0; -- expires = tm_now + STORM_INTERVAL; -- if (wrote_blackbox == FALSE) { -+ e->name, e->election_wins, STORM_INTERVAL); -+ e->election_wins = 0; -+ e->expires = tm_now + STORM_INTERVAL; -+ if (e->wrote_blackbox == FALSE) { - /* It's questionable whether a black box (from every node in the - * cluster) would be truly helpful in diagnosing an election - * storm. It's also highly doubtful a production environment -@@ -644,7 +642,7 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) - * write a blackbox on every Nth occurrence. - */ - crm_write_blackbox(0, NULL); -- wrote_blackbox = TRUE; -+ e->wrote_blackbox = TRUE; - } - } - } -@@ -657,21 +655,21 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) - return e->state; - - } else if (we_lose == FALSE) { -- if (last_election_loss == 0 -- || tm_now - last_election_loss > (time_t) LOSS_DAMPEN) { -+ if ((e->last_election_loss == 0) -+ || ((tm_now - e->last_election_loss) > (time_t) LOSS_DAMPEN)) { - - do_crm_log(log_level, "%s round %d (owner node ID %s) pass: %s from %s (%s)", - e->name, vote.election_id, vote.election_owner, vote.op, - vote.from, reason); - -- last_election_loss = 0; -+ e->last_election_loss = 0; - election_timeout_stop(e); - - /* Start a new election by voting down this, and other, peers */ - e->state = election_start; - return e->state; - } else { -- char *loss_time = ctime(&last_election_loss); -+ char *loss_time = ctime(&e->last_election_loss); - - if (loss_time) { - // Show only HH:MM:SS -@@ -684,7 +682,7 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) - } - } - -- last_election_loss = tm_now; -+ e->last_election_loss = tm_now; - - do_crm_log(log_level, "%s round %d (owner node ID %s) lost: %s from %s (%s)", - e->name, vote.election_id, vote.election_owner, vote.op, --- -1.8.3.1 - - -From 294ec2091501ce023528219ba61f174a80e5867b Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 24 Jan 2019 17:57:16 -0600 -Subject: [PATCH 61/69] Doc: libcrmcluster: comment on election dampening - failure case - ---- - lib/cluster/election.c | 13 +++++++++++++ - 1 file changed, 13 insertions(+) - -diff --git a/lib/cluster/election.c b/lib/cluster/election.c -index c0ffb47..c7f46ba 100644 ---- a/lib/cluster/election.c -+++ b/lib/cluster/election.c -@@ -655,6 +655,19 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) - return e->state; - - } else if (we_lose == FALSE) { -+ /* We track the time of the last election loss to implement an election -+ * dampening period, reducing the likelihood of an election storm. If -+ * this node has lost within the dampening period, don't start a new -+ * election, even if we win against a peer's vote -- the peer we lost to -+ * should win again. -+ * -+ * @TODO This has a problem case: if an election winner immediately -+ * leaves the cluster, and a new election is immediately called, all -+ * nodes could lose, with no new winner elected. The ideal solution -+ * would be to tie the election structure with the peer caches, which -+ * would allow us to clear the dampening when the previous winner -+ * leaves (and would allow other improvements as well). -+ */ - if ((e->last_election_loss == 0) - || ((tm_now - e->last_election_loss) > (time_t) LOSS_DAMPEN)) { - --- -1.8.3.1 - - -From 59ab73a6138c72b8e2fe5613dc6fca3fced51a17 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 24 Jan 2019 18:01:00 -0600 -Subject: [PATCH 62/69] Refactor: libcrmcommon: add method to clear election - dampening - ---- - include/crm/cluster/election.h | 3 ++- - lib/cluster/election.c | 11 +++++++++++ - 2 files changed, 13 insertions(+), 1 deletion(-) - -diff --git a/include/crm/cluster/election.h b/include/crm/cluster/election.h -index ab82ae6..0facc85 100644 ---- a/include/crm/cluster/election.h -+++ b/include/crm/cluster/election.h -@@ -1,5 +1,5 @@ - /* -- * Copyright 2009-2018 Andrew Beekhof -+ * Copyright 2009-2019 Andrew Beekhof - * - * This source code is licensed under the GNU Lesser General Public License - * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -@@ -76,6 +76,7 @@ bool election_check(election_t *e); - void election_remove(election_t *e, const char *uname); - enum election_result election_state(election_t *e); - enum election_result election_count_vote(election_t *e, xmlNode *vote, bool can_win); -+void election_clear_dampening(election_t *e); - - #ifdef __cplusplus - } -diff --git a/lib/cluster/election.c b/lib/cluster/election.c -index c7f46ba..5df5c9c 100644 ---- a/lib/cluster/election.c -+++ b/lib/cluster/election.c -@@ -706,3 +706,14 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) - e->state = election_lost; - return e->state; - } -+ -+/*! -+ * \brief Reset any election dampening currently in effect -+ * -+ * \param[in] e Election object to clear -+ */ -+void -+election_clear_dampening(election_t *e) -+{ -+ e->last_election_loss = 0; -+} --- -1.8.3.1 - - -From 5c80a96423e246b04cd7a05253d7be7e032d8527 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 24 Jan 2019 18:08:07 -0600 -Subject: [PATCH 63/69] Fix: attrd: clear election dampening when the writer - leaves - -Otherwise we can easily get the failure case described in 294ec209, -due to the recent changes in 435b7728. ---- - daemons/attrd/attrd_elections.c | 5 +++++ - 1 file changed, 5 insertions(+) - -diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c -index 025dae5..18a1c6f 100644 ---- a/daemons/attrd/attrd_elections.c -+++ b/daemons/attrd/attrd_elections.c -@@ -137,6 +137,11 @@ attrd_remove_voter(const crm_node_t *peer) - peer_writer = NULL; - crm_notice("Lost attribute writer %s", peer->uname); - -+ /* Clear any election dampening in effect. Otherwise, if the lost writer -+ * had just won, the election could fizzle out with no new writer. -+ */ -+ election_clear_dampening(writer); -+ - /* If the writer received attribute updates during its shutdown, it will - * not have written them to the CIB. Ensure we get a new writer so they - * are written out. This means that every node that sees the writer --- -1.8.3.1 - - -From 40055e8e4afb436762ed8ebb383677e525788fbb Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 25 Jan 2019 11:17:22 -0600 -Subject: [PATCH 64/69] Doc: Pacemaker Explained: clarify section on colocated - resource sets - -Most significantly, reorganize the examples such that resource A -is always placed first, to make them easier to compare. - -Also, change the example PNG graphic to be generated from (new) SVG source. ---- - doc/Makefile.am | 20 +- - doc/Pacemaker_Explained/en-US/Ch-Constraints.txt | 89 +++-- - doc/shared/en-US/images/pcmk-colocated-sets.svg | 436 +++++++++++++++++++++++ - doc/shared/en-US/images/three-sets-complex.png | Bin 82045 -> 0 bytes - 4 files changed, 490 insertions(+), 55 deletions(-) - create mode 100644 doc/shared/en-US/images/pcmk-colocated-sets.svg - delete mode 100644 doc/shared/en-US/images/three-sets-complex.png - -diff --git a/doc/Makefile.am b/doc/Makefile.am -index 1c4a1d3..aa9ff63 100644 ---- a/doc/Makefile.am -+++ b/doc/Makefile.am -@@ -1,21 +1,8 @@ - # --# doc: Pacemaker code -+# Copyright 2004-2019 Andrew Beekhof - # --# Copyright (C) 2008 Andrew Beekhof --# --# This program is free software; you can redistribute it and/or --# modify it under the terms of the GNU General Public License --# as published by the Free Software Foundation; either version 2 --# of the License, or (at your option) any later version. --# --# This program is distributed in the hope that it will be useful, --# but WITHOUT ANY WARRANTY; without even the implied warranty of --# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --# GNU General Public License for more details. --# --# You should have received a copy of the GNU General Public License --# along with this program; if not, write to the Free Software --# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -+# This source code is licensed under the GNU General Public License version 2 -+# or later (GPLv2+) WITHOUT ANY WARRANTY. - # - include $(top_srcdir)/Makefile.common - -@@ -73,7 +60,6 @@ PNGS_ORIGINAL = Pacemaker_Remote/en-US/images/pcmk-ha-cluster-stack.png \ - shared/en-US/images/Partitioning.png \ - shared/en-US/images/Welcome.png \ - shared/en-US/images/resource-set.png \ -- shared/en-US/images/three-sets-complex.png \ - shared/en-US/images/three-sets.png \ - shared/en-US/images/two-sets.png - PNGS_GENERATED = $(SVGS:%.svg=%-small.png) \ -diff --git a/doc/Pacemaker_Explained/en-US/Ch-Constraints.txt b/doc/Pacemaker_Explained/en-US/Ch-Constraints.txt -index 3f76d8a..c28c8e9 100644 ---- a/doc/Pacemaker_Explained/en-US/Ch-Constraints.txt -+++ b/doc/Pacemaker_Explained/en-US/Ch-Constraints.txt -@@ -753,15 +753,15 @@ to be an unordered set using "AND" logic by default, and adding - Another common situation is for an administrator to create a set of - colocated resources. - --One way to do this would be to define a resource group (see -+The simplest way to do this is to define a resource group (see - <>), but that cannot always accurately express the desired --state. -+relationships. For example, maybe the resources do not need to be ordered. - - Another way would be to define each relationship as an individual constraint, --but that causes a constraint explosion as the number of resources and --combinations grow. An example of this approach: -+but that causes a difficult-to-follow constraint explosion as the number of -+resources and combinations grow. - --.Chain of colocated resources -+.Colocation chain as individual constraints, where A is placed first, then B, then C, then D - ====== - [source,XML] - ------- -@@ -773,12 +773,9 @@ combinations grow. An example of this approach: - ------- - ====== - --To make things easier, resource sets (see <>) can be used --within colocation constraints. As with the chained version, a --resource that can't be active prevents any resource that must be --colocated with it from being active. For example, if +B+ is not --able to run, then both +C+ and by inference +D+ must also remain --stopped. Here is an example +resource_set+: -+To express complicated relationships with a simplified syntax -+footnote:[which is not the same as saying easy to follow], -+<> can be used within colocation constraints. - - .Equivalent colocation chain expressed using +resource_set+ - ====== -@@ -797,6 +794,19 @@ stopped. Here is an example +resource_set+: - ------- - ====== - -+[NOTE] -+==== -+Within a +resource_set+, the resources are listed in the order they are -+_placed_, which is the reverse of the order in which they are _colocated_. -+In the above example, resource +A+ is placed before resource +B+, which is -+the same as saying resource +B+ is colocated with resource +A+. -+==== -+ -+As with individual constraints, a resource that can't be active prevents any -+resource that must be colocated with it from being active. In both of the two -+previous examples, if +B+ is unable to run, then both +C+ and by inference +D+ -+must remain stopped. -+ - [IMPORTANT] - ========= - If you use a higher-level tool, pay attention to how it exposes this -@@ -804,38 +814,47 @@ functionality. Depending on the tool, creating a set +A B+ may be equivalent to - +A with B+, or +B with A+. - ========= - --This notation can also be used to tell the cluster that sets of resources must --be colocated relative to each other, where the individual members of each set --may or may not depend on each other being active (controlled by the --+sequential+ property). -+Resource sets can also be used to tell the cluster that entire _sets_ of -+resources must be colocated relative to each other, while the individual -+members within any one set may or may not be colocated relative to each other -+(determined by the set's +sequential+ property). - --In this example, +A+, +B+, and +C+ will each be colocated with +D+. --+D+ must be active, but any of +A+, +B+, or +C+ may be inactive without --affecting any other resources. -+In the following example, resources +B+, +C+, and +D+ will each be colocated -+with +A+ (which will be placed first). +A+ must be able to run in order for any -+of the resources to run, but any of +B+, +C+, or +D+ may be stopped without -+affecting any of the others. - --.Using colocated sets to specify a common peer -+.Using colocated sets to specify a shared dependency - ====== - [source,XML] - ------- - - -- -- -+ - - -- -- - - -+ -+ -+ - - - ------- - ====== - -+[NOTE] -+==== -+Pay close attention to the order in which resources and sets are listed. -+While the members of any one sequential set are placed first to last (i.e., the -+colocation dependency is last with first), multiple sets are placed last to -+first (i.e. the colocation dependency is first with last). -+==== -+ - [IMPORTANT] - ==== --A colocated set with +sequential=false+ makes sense only if there is another --set in the constraint. Otherwise, the constraint has no effect. -+A colocated set with +sequential="false"+ makes sense only if there is -+another set in the constraint. Otherwise, the constraint has no effect. - ==== - - There is no inherent limit to the number and size of the sets used. -@@ -848,15 +867,15 @@ before it must also be active. - If desired, you can restrict the dependency to instances of promotable clone - resources that are in a specific role, using the set's +role+ property. - --.Colocation chain in which the members of the middle set have no interdependencies, and the last listed set (which the cluster places first) is restricted to instances in master status. -+.Colocation in which the members of the middle set have no interdependencies, and the last set listed applies only to instances in the master role - ====== - [source,XML] - ------- - - - -- -- -+ -+ - - - -@@ -864,24 +883,18 @@ resources that are in a specific role, using the set's +role+ property. - - - -- -- -+ -+ - - - - ------- - ====== - --.Visual representation the above example (resources to the left are placed first) --image::images/three-sets-complex.png["Colocation chain",width="16cm",height="9cm",align="center"] -+.Visual representation of the above example (resources are placed from left to right) -+image::images/pcmk-colocated-sets.png["Colocation chain",width="960px",height="431px",align="center"] - - [NOTE] - ==== --Pay close attention to the order in which resources and sets are listed. --While the colocation dependency for members of any one set is last-to-first, --the colocation dependency for multiple sets is first-to-last. In the above --example, +B+ is colocated with +A+, but +colocated-set-1+ is --colocated with +colocated-set-2+. -- - Unlike ordered sets, colocated sets do not use the +require-all+ option. - ==== -diff --git a/doc/shared/en-US/images/pcmk-colocated-sets.svg b/doc/shared/en-US/images/pcmk-colocated-sets.svg -new file mode 100644 -index 0000000..9e53fc4 ---- /dev/null -+++ b/doc/shared/en-US/images/pcmk-colocated-sets.svg -@@ -0,0 +1,436 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ image/svg+xml -+ -+ -+ -+ -+ -+ -+ -+ -+ A -+ A -+ -+ -+ -+ B -+ B -+ -+ -+ -+ C -+ C -+ -+ -+ -+ D -+ D -+ -+ -+ -+ E -+ E -+ -+ -+ -+ F -+ F -+ -+ -+ -+ G -+ G -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/doc/shared/en-US/images/three-sets-complex.png b/doc/shared/en-US/images/three-sets-complex.png -deleted file mode 100644 -index 4318f6c62f0e72d155a84f9b847705ee50d5dc78..0000000000000000000000000000000000000000 -GIT binary patch -literal 0 -HcmV?d00001 - -literal 82045 -zcmZ_02{@GD*FXNu7>s>S5@S%dEK$fdgG4DyDiv8OTiK%Q#*i&#i>%p_Eo9BU3<_By -zBKs1ujh!)!8UN`U`u^Vc{a@F(T=P5^&wZc!oX`23&wcLag`SQk1MLY~000;+YhAn! -z01yfQ0DXs2k*~lU9rMY*XdJcfI0FD3+n*m0knr|60Pq2qFRI>f17U03BkYFnjc+eH -zd+;tRF|BO9c{mS;pVfv=U8fOX&X7Jk@SaxdmAa^gsE#q$Tg1t1+MOVX6gW02I`nOX -znP?`hp)fzU5cI2Bvo#pR>fynvwossbx%r_YyDA(BS{|+2rH1Kr+dPu>K!^ -zNYp~Mp^0v{`h@=1KLxSOsgiHlAft$K>Gv$BZyO>H%zUhI{kN!WVfzjTB>u2*~s||K&*$xfwhJut{o!BWd>^yi@p$)haFn!jWGVC~ux^a8U^)V!1Pe-;< -zA;96rt#Y@3mxrUpo*^5?C8qq*!T<6VXPFbk7*cxg$`yY{>&(nBEDFvUfC?i+o#GIZ -zC5CS2llG40GptQs$EBIv9*w`NVb%^&puxJ|me`a_>||bL>F-R+>S~|=fu!9Sh3wzW -zKL$qps;-QelZ0*kwwQh5^lua=K}%eP&UrVZj;4+FQ(*07sP@pC>$0oJMNL1r{fS-{ -zIw{5eV?Gn9o9KtYeo)*NP$qV0Ct5TGs#d%@vf($@?hbU)O#aY;AEXL_XlE*u=vLVm -zL^fw#(!WE~5KQUZ7MP^iH$NlgLK-gwxDK%|o4YD3Y_l63( -zn8D6>T!d0ft?XbQzhgfR!E$_(NEwYr{dE!xg -zj1~|DR}J2vpO#%vuT+`{GQamV%k(gEeO1{aq|Lf7-b_ARoy+J(cY6bPw>GG-tjtnA -zn;A_Q@*5Odc{Px=61tn{!%3psLY4Gg&0j-9#RNTtDa`PE%#A1!oca?z=(L94!ySh` -zUD@@Y#t{gau~Jvwdu1!~MV7N!9f3hX>KF7rIiFJf$ehZLrbT}U^>}y`&s5rcqr)$y -ziJ!1}KH)pIe%8Zo=hw3gC7dwK@dN#Pii? -zm|b1=FUz^@U)$Hug&7xw8$aFn&NqW}^Z%THG96|zt@*0!{!7m9k>e2dM$?$LSKpYk -zM;~KozuH+K17H${WDBAbulO3nBg81?!Wy>(4i`d73S2X#=}Tg6RNTji>Zu+@h#k15 -z@3-uTd@2Opt()kuFS1wg@nfJG&<>Ecwpa+Clg=95)oz3QV4e}wtbmu6*K|Nd{e8lr -z=Df1Q%rtn4G?>i@{mv!dr<^Bz8$~%;qI-8lI)dI|Ho^!WFRomuyuX5|W^IS20>`Y!hVx|~dPu7ke<3-}b -zo>HIpUrlhD?3DY_5+YDOM;Y;rCi|ca%|I3&K$;xkWv_I&4xlU*0cOrNH=EV`2ubA! -zWPtUlZF~Ab2jloc>fAb!R62sNh^=HU*Ng|0)U@Qf6(dH~=+aKn#1n6PJp2?zM87~v -z3;D~apGb{)NrW{02316}Ps!dHe*b6TAvX%s=#fr_kk1GxZW$qMiBso1hE=vQN@K>* -zgLrpLz)3L-T5vQQ-=Tj!hocEjHhZeza7O7P*P`IZ7>Qx`TDPett^-(x!s24qHgsjn -zW1KF@i(iES3g-=Z9WuYq11Ro#fmRrN#;(g|+WRH6q>!(C8WLf57YDjdin$e{N -z!o?S+c`2Zhl&*&0iI)^o!9jEmduw+ZQ}Eg^T`U!rPDO}8CNiY~+4giE3l57$@xzhv -z7rKMzmyM@466?q-YL$KRV0hh;fJ1JfJ2Jb=3W5YhbV1P46Efv@arN~IcHxE{Cps@_ -z?TpX?4u6biD|x6(}2HIZ@|CgrtXr5k&@V$}&uj^9YKMjN_>fT!7G>tDUQ2wx! -zxF1M|H)fnLg{}@lBJl8L0i}~}V!T(-Kh^~oG{H5}LFvrHge8Rin(X(zWs}bogiTs{x;_9>1ixb(C7Qe%weXUk7WdD(bT~ -z9LZmN=ddMMIJcKwZ&WLmPDUA#Wpc5fZHb6Ac-rnwFhw3CX -zv`4u(f9C*ctyrA*b_bT*^p8b@?A@YF%2~{yq~M2n)dKXSU$R{lO5!8Xt|leTIMD2D -z(l`=$>0zseRV)t&4tmVBq!zZ9to9-GYLdl*5iTYC}IA+7vsm -zVImwuaGqYH4#Qr)FY`q>|1jkP(A8Tf)N00W!AGXhV4w|-u)0#l0%=-$AKixZjC%?~G|T3Prk`ya%>f@D6qM4P#GB -z>vZ6lYm{m1$SNgRTa+5Wewc_-eaM7yv+Ih;q3Os3WB#TFU~D0j|oz~fyA-yg2F5tt}J176)+z8oB2 -zv(}komHg@%Xu#?y^; -zUHL_j@}@UChPI>PvtJwVjnP_0ySA%KBQePM2#s%&UE;=zR>^~vFycd|vAfjFxR{rH -zfV$OG*w*8}&{TiAU7qQ4UlKCT?7y>vEI97arF=$qOw5uY^(u4|K9X+NtvuoQT?bUh`KOdr7l+hjFof`_45}aq~*xnD8s0Yix}**{tR{-ARv}@HmBB -zD;w|CqlqxS4A*<8QzY!nIh)0bUb5O}Q~w#{uLium+dz-SL@4E0)5MJZGz;dnT9#$1 -zJR)Zw$WD;W8fkjKI?9lMn(+7yb-gdD4IFA%D~AN{r}i~h*&4+lhQd^Piy-((b^gG$ -z)rWlR)rn(zAa;HGFIRoFv`?vCIX)R_yFD9jK{3`~ZL`Kfxrw0JiW8hnq}-(k6;d>P -zAP?oMa$MP3>rwsqKkGsd7A>$C<smuxhX#4!CX&0w9MM(2^o*cd3yzN~iO~hMhmU8h)#5 -zwW6M$eUA~)KBQKu6lB_&l01Gu#z -zYTc4J!#Go$!Js;6eWz28oieVc8km;Yw;x75{{-3c-BJhTyRSHNsu@EfwZA?zb0=|8 -z*O^!(akwugGvVpdO^?@T{ -z{kb{e^Le^vy|Sl?Jiz^Fit>M1%Y*Fad9(18!})5U{Z3Qf6a_n3VA8+bBybk!vbTc^ -z#;+sW=*a}QajZo4&Z0*B@l3R -zK%US4Iu{be;gG+d@>6rELAqDaSJuQ!y*rfR)?LPlkIc(a%wf8e*KY@{GDiJ|I%XnS -zyEo#9Q7LW!TtwGQG3|aq0CoE@+Y2&$DLi}u$P;Y49Dg6^8~TSqu)x0@RIA7U3IRlEj|VMGLA)Lsk4J)a?k3UV{f7Ykd4s -zOY4otMTx9q^ZnF)sdniQ7x)BH<3U{X7Y?DA6GXegeon;gIv2XUXCLP=8uN4S -zG4N4PYJVlcFTy>2L~WJ#omYq(?c^)<#t1YqOB*$=AsO-#Z`F#JPGoFObd~&Q8x_l$ -zR>;z|uK(c9irKB4h|5r7Q(yTit=+0S`e*f8&)cTVO$eOEIfByB&EUYIjX0*m#sB}e=XCV -zK^gLj4RMybvO$TW_qb2n%Z`}R6m$M{eAfO4b6hR>~n2v -zDfU|}XQ;v-*7NhA*3+#Ki<<1CwF}`@>4wSXY)htw<%WS(dP4H4SF0u~luXMXZ!C2w -zuGf?j3_1jnE?h@khOB9#sr(Ki!S*%af3?m#kU;~$c8&2eh`dRpI&IGWOAPXuc8y0= -z+sWy@NWg@2{L4Bx(F6G!-5%HT9gbMgY0etp+!8=E84QPno{yi7?vMqSwVZ7Gi1wlB!(>9C -zPF^Jn=MR1Q|1prS)hT4dYnPAo6}8>PiF9A=IKYpJ26>mV7F;UNk#S7dx!kke@!`Vh -z`rqzOuBQ+Snw~ZTxSDS{kqNbK4?mBgEE~gg-C)L*EQyr-rjh$O(%p#ga7qT{1)qML -zYa7f?HMYC1u+v_=AKM>=I}`drn$H1or}Y5(eV<}63S)C;! -z8tBx62+H+W^}^KZ_K|^h1-=f|TvR(T_Cs*Drd}&j#Ot;5XnyY-32zAEoI+TlBv0`Q -zsvWL)R=N1&b~te(xM0+%INZl#47J4MFgRsVP!?FV!=={rC9hPsNaONePA -z4Oi**OOaxh#)5Xhai1O4(QiCDDIU{1ImvS0;%@&mcRzre34k?1dICs(`s~hsaKUU5 -zBK+`o33!6E5!7_qB(!|SEp1s%G@6RkrSWqUJp6p3nH;l0IXDrZ(e5)`p-n_3CrB?g -zB=Y0Rp8mrRh=ui6t%zH4gq6D4JmQ4hNsW*mr91WmO<&F{o|k#lJN1mP>um63FW)t( -zaDOUF4x8fryG;C}D$PVttoc}f4BfQ#Sx@0M;v|wWf2;LtxdatzJ%chwWW$HZ`WArTdQ?oJiwr23%1US0* -zky}9hM~AL{RCEBVwuJls%iZhyuSFZTSW%3D_s$lc@SdvlWuP!B0fPfj2czdBr|?wY -z!@Y(?KA<7R@AS9|;;#Lm9hpkyUowINlb?iR{e<_JuU#1AO;!t(=#^&hH)_^6yKHV; -zzoTz7{X`W%Y?Xzb-QT5DR61+ho_me(b=;$m)HnMvF<*RWXq~O3&%$T8&N4JcOqO@N -z9O1JRLfoz2@grJ|Z@Ul_VQxX*?HK|%CQ>suc78Y?X+gic7(oVOqb^WrhOL6FP -zYo9DqI^!m6%M9^6t*V-DmGH!~$X=fF!GpzE+6kpT<0tSV<)!;{s^WvhfWs1mthfaD -z3`kYNmMeu&iUF(3Pn1?Ol_0kQIb|Jct_H31znTw`E_elMA-Q55nJTdkRBzSckTTs>9Xf7($-m!-w4 -zR4xkfiTYV7wq?w{F-(VHxaUz5YC3kmgM=4z?)L|qtwoA~YSu`1J`b%1GlX9oT>Wg$ -z9j|o1N&d8B#{e~Aub8-;Vti>4>rm&}zaX{|Sw2Gx*~p+Iivr$(v=Amv+eMe{6jNob -ztsPFD8pRzfLuvdhz5Tz;a1q%CghCF79t8wk5d~Y1-_7m-udd&JUi~Zd0_B5&nFhHT -zXlpd}s~|A2{GroLl>K$#(u?$0KNx?j4}5=7MWkL)3Ve;-C#4U&>@?TDhu(I3nBVs$ -zPQ?_KNZ;V|aMzzfm!~9h{((KrHL>L^0fAV%8~w!Ese`nh7#1jHhCXG=QxwAJ -z=~?^xis`N$Vz`5JbK|>dGl3QK>*$a2eKV51^%}f;-wP$V1al%?&qZ1@4v%`07(O;WxIakYKxha}?0lYCVH1;{X -z>*VcEh}t?c)9MQj4)V$mVE8h*ffj_%7Y-WI@v{oci#W}wYTr*gW6jsfe0qs3CLj~t -z4y5n`ZF$f*jRjJmLL0=KrH?oi82;=L@)QZ{9nh^Ih*KJ?{qB5S{dDyLr<#brn|}e@ -z@5z@?=v?r|SAy0QfwnO3I-kUP`D4i&P~~;adnP{i1A$C6L`TR}F37nYcH8;O?Dqn3 -z&D*1~lxCNac3wjZ0&Egygh2G#2k(2A`<*fxBE)D=(>^-}xdqB)7UNbJZeH4e-yvNR -z$T5XFDLvt{N)mG4%XybHhdObH-B+(DTP9X~nPSe$nB`B?GCq9!T4>qvNR(9l!|zp9 -z2n*H!AsUz%nrXG)3wPmdYEoOQBH3!E10!fD0g}PAg%`9KdP#xlefQmFPJSoEPWtg1 -zbAdbIcvj!}qRp87Dbt!60#W>KdwARzDFyQ;J#d9us){t1C)GvIpAM7Zm+rK?VjbM^ -zA@%MKvWB*u2=Uu66R|q}32&S?TaPrz@We$rd{o6|FR2p;+lr>HT3Yh=`O-zn;NBx? -zbVz1XuaQZBEZt$LpDG5Tjj3b>UQC+W2&Qp@mm=D~z(=alZ8^JX&{g^O*WP^?c62+@ -zTEr4q!JpSn{~q#cyCQi<`5yac>Ct3m*^q82Lwn^{%HX_7HLo*FjuvQ12FxpT-15r& -z;xmV{8=ewBsGpCSyPyk1R&{B0;z9{ol%wN~vD99&AeG^uOs2};53*-I3J!WgH#Fh3 -z!}Tvn)K={AA>j+xiI=(1E+tDh$=&usD77wO&MGRj`mggsYewJNj??=Jaq5JZMTK%9t#w~F$BAKApJcCu26`^ -zJ$yI1dB|iyEVnhuHhm^P%>t@>G{bw?3-Fg@jY061T(SY3Gxse4CpF;!`Jtz2+0{qUJ3H;(C=Y-Qz_T2BCSqw`f`jj*7&wVU>!UfRCkBZAJ4oR -z1xbPIzEk-DBW_guNLK6Kjt90_40$V9`DD*LJ>}>5?qu0Bu_dcS^X)cWOaI`^CMjKE -z%MloD0RQXRGpq|oTS)(J0wo(!0|`xPU3o1P4Lr^O^U-u&^D*LiCz{zkWvZ2bn>FII -zVO~~{*sqb$A5yEDX}rufa;+&&@AtmTcF9eevY_vn*i}r_QPB*xfB0fVysJ#J8tm{B -zcAj5&t7SNNSM#^CgeY{cng{X*Qi!(?9C#* -z?!Ft`o+gNNVl|!aeZFVOP&*|b*t&2`qmwy -zNWXj>jCct}jW5*AxP(iK;YcyEztue_AQS%u?g_mxzKuw4PJ&<&EL -zzh`TkhN3>zsN?1bm_;G6BWiXfi`W6nr+%47LahsUe`A{cSG1ZvAQv6B9aT^P@hWc6A(6=V)Ka)gy|rF~g3fgK?BJw7Fl;pD -zsOw8AWy{b^F6~BAn|Cm24(fcabpMHc?>&d8dbq#oU|Jl1mdSc|ox)T$qKDr>ao=Sd -z_N=Ju2`_DoJg1VRwR2yn)W58YTURJL>P#7cfS~c`rfh6bZrY$ejb)b*HDH@v55P=6 -zfnU_LHp;=sEonH$qtf?oNb&7=YUY*|$Bae$OmXY-wJvVYfboK6c;jSOxdq8sM5^}_ -zw_oq_ZJ7LGyk`@wu=%s6Z9VcV!+9l*ocqGhr+5PyNPS)clf#&1mE0qr`knRy+EiT> -zX5#JDht=?zS>cTCf`c+K7@(O*7CTzX?&nB}8;wb@I2WIX^pDqu)kH!XjkXFWoAqg1 -z>`sB2;$qe8ribsN$~Y_@Y&3S&%7*{K28|Oe6Hju#=%Ow&b_-I&+-CxXKTS)q3;s-M -z!o@x@87uG5(b$A}FGY%hkL6d^nB~8reEjjuE($V`DkZ`j5xjFpfPTym9 -z@DZsgJ+zelom2Adb35HfSk5R)CiRR9@rLz(m5JndfU-R*%I1nU4_MeAb<#w^cEoz8 -zOwkrPrg5TXRL6X&)P1Dn_jVEp8;kI0n%RlrN#ZSmd30%CHZwK)6a)JxzRI4XycUn_ -zfMA*fQPWE;!+Y97mzKV;O*LR;Oe-HerVGz2avV>1x$%*Hbq4zQ@%PvpVcV5vXXi+> -zHQkqXizvV&@|;pia+BY~56y`&fH;mvABFGSbAa;dfRdC902a>54?>5yMN?0nD9(X> -zO|h-m!1tK?%Sa3+cERjkR2kvkaBJhfOOC9mb)c|>G4m$Wtiz5KmC<7fOq0FiAX4y! -zOA)HH{-!V$X_)ElONeevKY5r5CCxoG`QlY-7%M7#xR`uT9=cdE*)?~`ikP(SoC&sz -zkXYP2)?A4(WG!~~58N2h5ZXLi?ucmoufz1h-wpFMZhe)sb8k-R!cAzS#AepLUAnY- -zJ=%-OYStR!UsD#^DF@>?^jPfG`7R}kPwSd89xrAhE}wr~q346cmlCJyw?T;~p1{X{ -z?#g*}BG|zr2tG?n|Mf(#0L~e@*`tGN}%z%YPPL?!is{u&#brAhz0IN?8;#M -z-jA75L%@>4Z9p%7xpBL&dZ%D?*|&6jv6TQjSr`p^)*@-kNm6f+VxI-i3jPwCeK}jy -zdiqdyXFT`Ei#O*DXa9P!tTvy|)Ya_{d_+>*>L5r$D@adaI(JK5-i|IWTgZR5L#LgZ -z`EK5@B;}Lf@}QV&annz5XyO$|a<{dQg`e#kG8DQ3xMFVxWM5@u>Z -ziMbsd_^9a?jJJHZHM{=Bw%uO3@>uJ;o3^_3OAccERof}VD;DV~w^=97dyKXSod|RW -zrKQXU+LvBAk`Gg_@eu%BMZH6fo7p#u#!;3W1<1)# -zTSBeZOyp>i!_d@BW@mCjpOD5qvxCa8cm>du+<5%kL^ -zBWVmFR|PeYg910^+g{9F`~b6?FDFJGLzsKaYTCNHQ98&kBp$b%^``}~e@fP7{|Gy% -zm*l(2`Axn%%QT7eKeGD384&c%j_TybYr1znZk>oxL+)Zr3b4`*(APku4(GyipdHWb -zsw^GA5t-2sR1Q74+AD^5bmcr}J?VGNi%#M~6THUMv*NxFVOGTwv3t* -zVUiZYZ>7WR`^ytiWyN~0FShYNi6uR_1zhc5ye&uI@AOBjQH3yxg4Q?Kx&HP2=#d4U -z)Qj+-1K@fp1T+-CT{a;%3PI7tOg3hxUPQDSs3ISNe#CceR#T}>?ZgmNU7~+7j2)Jy -zVGP@el)f4RYjke^xmCV!jhic}?7XS-em$|?^phzmAMuG(S9dVOxdl62r0!zv(fD!X -zPH91>a#5};XZ~agovm#Dug9PP+roKMK4juzdX((15OidbO>SR{ahKGuw49aiq#s4% -zpX#mlYjp<@fCA`-FGZj>^_UyDgm0+id^WvSj*_x;+YwX#pNbE=gfn-eRjo<+gDBD+ -z#qqDC-(IXb@FH1acBL$)d{&Io@D@iRZJ-_v;HUA$LdP6T|7_n?eYUe`4CA7K^Q7ydkMZ* -zh;J@=oEANKVe-N|9`UOT0S1RbeV)b^N}fxlHiCo39FlNKdAqDSSm5pa_*@bk6& -ztMRyGSU@#XL9r1u|LVT(S9K4ac{pC^dxHL?_MLkXt|>otPpL!ro@;mY?Loszd8scP -zzk-gln2>?oeO3x{*wyrA!#vqn!#LT}A1?tS+BSZy1(0X3XI2H*MPTResZZ?mo_?;+ -zgc=c?p}4`F9e!F=^QVDZ%97-U08Gg~JzU<;dmxy**a2#RxqU$lFjpF&{kH*aN4Ece -zB@|^EWQBiP#Mkm}J;;Ro`I1MoOy`#tD4^Zq;}w$m<_^EEc?LIJ?hcopxagjo84;na -zdSVT}6Uc1(eo&3%CdQN|-_|4<4YuhALl$?lb#qK}joK;qn&%S4D0-X#)P2WRN|NAa -zwKO?tr>31Z6g0b}3dQLKU(C^{oRhR-=WCEDLv)2(FidKS)i7mWh?e3-k@T9roz$V_ -zdP|+mnJhGveDl!y7gPTT|AGG{{0C^$Cc&eR(SoiT*5fdgme4$jGjiT+by_QaPw4#Q -zArA|fssL!Y`D>xZ2Vm8ZYkr=QmQdX34Nc;;6x_<`E|?6o#_s#E4p!?OdeFFW;f&4q -zGE25Lx;c=sKKhaRoNES{sKt>K>`wJvNAO}d-!PMXT6eHht(OD!)gMz&3e77n@)<%> -zSCn)`-k=Wlk&qB4vWch9O&==aypVpHEapo@vUHCtH!&^__kQ*x=GLRS$Mppdw$Ft! -z&)s|aCBd81Tpbz&M)#Z4 -zc{l1}R?8wzeAZA+O9%Xv^{&1=WT;R*GZRhUFvQTv-_Mh#q-D->7r(K0H371hKf?qX -zE_r8CdcJ=u9pOk|^}UOw>9+S_T~+pm?zRT0*qRdbQBRc@eWWxyzts&|56Pt%mBqi9 -zyq&440esU$h55ZXl!lRzeim|fD-YwLYM(0ir^aAc$e};Z%PV`*XNG}+J0~a`o4DFw -zPxphlqKRGY>D9rY!bQ!iRb=ms6z$=Ypux_tKId2NnF!9=Ol(d0XGxzSJH#SW21MSyGP}NwX*PATU9$gH47StYr#ctK@(n6J-dX?C2rK%i -z!=5qfBqTw{z58*6yoG3do!yCJKQADEzsDiIYw?xS@z@z% -zJ1x7xM}$bGnP?eiz1kqk$F4>D6AHiHnktY!U$Sdh9A2F -zvJ|)I*m`(@n+-Zy$}BaR3D8;qVPC#zcrK>V3$67flqn9?cDt`9e@rTmD!G}A;iXdph<=e30H*YwqJ-0uOR=alfd+~Vo -zBD#VGLA|}bxG@0+LgTX%YC1hCWC-rsidv110gmp1f~cPs(`3R-Gb30J!p9z&9Jh6v -zQk6Dq;^dCflF2uwfXfTiRd0jQ+@yx)bwEm<-1m9U!5;tm)!D>^gpdrbI4&tUIjg9s -zDA}`TS>}OTS3r7)JH}+PX>IxcwjW@O=EMv?4&)ZpJ!0Zjf~K*xQ&!vn+R!%RGot`P_ty6#o -zFm{nl&>%OUJEfD&3NAH8fkbi1gnJ78&{L|c-S^%ysmcvP+cj=|&XDqHUn!wKKF+wk -zsc}M-CL&M<+&xzt2V2TCCzuOm030If4Xjrijv{LxpDFnqFhL$MSB9et7B3y6?LgPZ7lKXHrzVf(B5b&{V&6 -zt$mRl6!!x3+l<&qC+Q!`aMEFJ!Q<6?h)Tnu9xOzjNB -z;0y~o;)(@kE#92q>u@_v{^~4~`Oo&OLS8?w;6~oyT997(@PGl}u*Y@GVgIF6rcDb+ -zo1bzFjMz%hMcATlMPBi(U#4$(T>`EUr08g1pVWh5!sEFIZs`60E{d%d5CVAhGd9j* -zv9=vhv|ih_CHVjzwFRTD_}a%V=X>stDl+E1n5}Yy*4PaB+&#@aIp*hP6c$s#iave4 -z0`W!FRwo4btxM~7XpVGfh?tjYaUmit*PRRJ-{QyY!OHRq5xw&i}1WL)@TbKghe(mytWy5a?2Bm3#?!2YE0u1wq!sW0Ffs&rSB-b7in*AOmRkS -zUT{E+lb&03asa`W0|Dul+=01|HU -zToO32yh>j&%zL)jmnRx9yZo_|AINj;MxJ$^Ze?ldk@hk-d-m*&{$VRH67rVpS|_Lu -zT)mJ6%q+TgSrr-6^D;;xXR%Q}5e#_HQ~Sx;jl -zqnFP8n*}nhM+8HVOpmMu556K($OXWdZ9Zw#PI^S@(a5~j;Ym8BFpa$rM8B)P@P -z5vf~K^*6T{UH~)m7m>VdY(yX)(D<8^Y^SaxGde|lCZtg^yDyg?) -z(bd;e9n)XEXVy~lf_{Tbt7Eo(*il92fMvI}l_wd)-0bGT&YCB!PmRQ{UM(LEKK>9@ -zru1oa@s*2EqM9-x1q-K$f~{qMICX!hM|8abO}~)+O?`LlE1xp%De~R`mL*Z;Ukz9R -zHaXn$M?z#zqm$-mXJ^&3{T*rGC6CmCaY))6X5?gG=;jTmStSrA_x9spc&+DmOD_c_ -z9FvyjU0LQGz8}?J$KM5wQ$C6lG!uHl?T7MLOAYE~GAJ^T=rX)$a3NxyLBi*b?jwyy -zLjJDXn)M?bAwXfnb>G5CT<3ms(A%$Sg>WHZ3JfE?bLRA`M-$7IdW_L>QUjZ74bQ1Z -zUyB+aKprFgpOUE~=a%N*Y+7~&7|*Ns+_rz%Vzt=Z++1}`BQ4tR5wtcyIyFjdA|Oft -zH|1k77^Yi|T%1>vl3=G`PEp4PEHz5B`SYCS+*Jl_)SD&QfyfW+yDUtjNLGCBYd`kD -z;yIuGy)jHSwZ$dHTFk^mnynmx#arE{dv?yXGpviAl1^&=Jv3FCFD~NwPmSN%<%tcP -zyoD(g9iiNE*soDS|2<*|`6SmwISf5?np11e|FvqM$u}dy!ArcD@JJpnh8&=K!nM$$?M6QW!7pkYHey -zUy>*72AS$Tzl2>aM|_Fvg3giW!itt1DvE571_`^Q>Qy|Qzb?!p+m`Nj2wLHgo?hg4 -z8K2`?x*UX}tcAm4d*ps8$ktC2PjQ#@7OMrkWo62JH|-1tU*6>pYUnGzWR7EW*W}aq -zPGh0>@5fD%Wt5bDbhvB6l)sLJ?mboi*u0Ft92W#IrpebwO`T3*rPbMQ7y49o>uWv^7aVNSMPsOL(xf(#&P}(jI$Ze7J1yv$hq- -zcc}N~D9C53V0(dMSAh50DSCN>(*qI2PY!p>%1w@kT;s}tpcyCursJ`pR{s4fdVfMz`s8*f&VrcH^B7Q -zSt1nfr=bTI@|a@ahug_GY2RnAo<7#X%Q(+U?MHuiz|l}8Zjjf7iEl}YYqB0LE@iWU -zzQiA&hacMq>8+IC>!3$-%TTP}0lyVl6)K4L-h3$_TOapGjjMx>yYc)sCx+W|r1gn& -zIJd_WBdc2Wv3RCQsJ4xMUG+%U-Xbrn-p3Y=5B#u^jxOU72#bZT -zayQf6LWUk*2C-sn$;Tbg4MLEtEbqj9(DAYR&fNCU-%8|skp$B{X1K&wfHwjAoW*KW -z89&_irYMXB9&Q)bdKV3vJ>k|B1u;$8@wmFyPZ<(hVcG!S^sFxMr5Dr_1S~%p*pu8& -zR}u!|VD|4rH-p7S;2oYLtE&;-iy0iAT|dv;KH~QljR*k-j9CG0_iRY;dbd~U*^(w1 -zbjqIB)W&le85s)Un;Hjs5wk3$H?nM?PCT80pv98JGNKK0kVlgrf^Jvl{*}3Egh8WAC2$C1SYB64Zi~< -zMFU#88UP2)O-g{MzY6$B(LqX7^+;H3UTAnYQ{hvNL)u#%g(NJmENui)sc>Mw^?$f| -z7|7Qke75zw3%PJKOvP(}>__F|=hto}4m)n7ssU}F(ic}iL(nD^IA!N03t`G|u`HyP -zlDHhY*E;pFk;yd;3RPr1mByRXW~5?I+?fK1UtM-8cvjpC>G{;IUz~#$Y64e162nCw -z$zFHZy^hYT-3ZMsk_0#aX8Z*Nni=v+ -zSZi;kUI%DFqLgM`l6ix(l7y?iV$Z5c7)qzEaBlZAT+j?OfjMS+S6XHF=KGoF$aqNP -zW`&Fj(``KYB%*=*IS`c2prY%f&CsyKIV_Ll$H9(=2_)Ug6m5{xDe|GA%aM9#WLCkh -zS7`K}Qgy52y_VX{kq8r^%gSgHxa`7cgljAx?#yYc#!THwuV@q -zLRZYr#;j`%gJSo!oA85PdGdK6fZ=bOmM_J%3HJidtcb8EK#us-^*Zfzh_B8vA)F|_ -zy#%jzDz`UW{;@=26Q+0<_;6~O$pXN!3dcg-0RL63*q+949}#9h#SwLv9|8L>TTZyF -zU2HN@8g$y%s~k&dMELKGTtfu-46)5r_F -z-Rn+MFOu~tKc58l5K07l%gq44xHJ>pKgjNe7LH7V6)1N;Tx+OH4BdS7z~l$W)!uNF -zrBWMZKUQP2y*&DIhOwliL{>qeO8ai66uRbDC1DyAUB91tt4pwW|x5= -z$nse2<8u2^T60DR_ci66T5I9f-b=8#Z%ZSiBG-7STc*bc}k~_H<7{r -zm>{b)CsJIL$O2i?1!v2GXR1qw$tsK754+0FO8M7R7>(QUv1vhL85qU7GA^H#oSyy`eHhZfJhBS)7eI^i%qZo5;hOeBkO4DDCV3XRP? -zhUz~FABu>c(PYbdZhp==*jga64A7Y}&J_q5Y2UmFEsVCGe3f;N120EwwTJrTFkWZp -zx;*mYno!Y?hqku1^)5Gwe_zuuFiWnr2WEwCkN$bN#h>nRuwqi -z3dZ%yAu;ckxGI;WgwC$6;nvn*Rk)OrQjK9kfy)7CBQKk6qqK8u-yLdrh3CP}&dzC2 -zbfL>+Bc%wqEKt8v8XPC`WO<}gPs02`L#|=L36;rz(+T}Eouyag4fX%@33&fu`@BkS -z%!y;FEa>tIX?`gv_|si+96kEA&66kJ?MEuk2H(FKc;E|IVRDt@!d@==_Z1xjG+?|F -zpaO?JFRx%@V^h@6Uyw&aQlXgBetPue#cnN(MIY|*V^5LmVQfZ8M%mxFkQEwJMofbJ -zaj)}c^^CjsB>Pt(=eO5G!vArNLO2M!9@aThn8U5I -z9Vk~es`9ej-24h3IFZj0J8)LfX*$@PUX3f@?2*Z;lgK8kCer74ADXPfeZ$i0LaIqY -z?Bhy8TlPhMDFAHs?QPN5ub11cvglK9a~wkJ@Fg<7*s16G{VfVzLNK+S+gK8jaFz{7 -ztRf$|Q`9ioiU2IrCUQB_)6<*1#iR_7hoY8?*dJpr#_L!AO?tchtJ7cEn(}}+vf#1M -z4)WGFc(=rdxg%1zA$iwU^PcEm`ye67Z^$F=x}vx9_o>O;5fDphSkJ_u?QSri8 -ze$wE02|&#(0ZS;xvH6PIQ3px&t3Q*)`e$GGJ6R%2>rYEvUEN`F?MP_PzQJGE2`O|1 -zEORg&ot<2k1&4A6#{SRSV(b+K9{uf{!307Ij}eAGJr0Zq;hp}Z#;qANLyky;4lCv- -zrIZmRm1!kL1$?whiGNlp@mt`~-(zOooFFXO9rKOT@8!HPt{|I0_p4ZtQOR?LSd9lkEsFQMB9=X5`gO&I7 -z^~J$cCFzxd*wRG*nn&T!#<{>T%D--jy#yxXN)Z*eT~mT>US|x8@90&aCJ!*W%eLv6h`~3diYu7ujGmrB;kE<+|Y{h~~ -z#%DXtO1mb8bo&-TS|dR?p6zV_7&C;^$4bVpzoq`Oisy|o9Mb*Ty%`>E3F5>BE-js@ -zggQ{UYL5B>MOID%oIaJm!!v6Nyr{uh1CE@9e;45-fZuc+!}ouE*Zy}C5AlDMHw9h! -zgJtm+-4OSj8my9OLmSnWaBi&NKDkA^k%`HSqnTd7zo$K+#R=8PaQ7?28=RYc#ia*c6DKD(K|nH|qO>0dZ58%qt0$SiynM-gCwk^TP*LqoR#!WiZSv1>@x|Ac -zFcRwR1svvrI9wlOu_f;M)MCyX(l1=@Wa;cI*d!97OZ{Isr$y(6xuq!T=f7}cXcOs* -zd}+31}Z}mSYXyM%!76=Q4xg#U2K>y#LZ~7N}K2&V}>ws{Bp@kaAUi>t< -z*hgdM9!eHdN%j!B7BMD)$WWO7@q4hEC&c%8|9|>X2yc7^A@6f-`hR?d($aE!q^QBY -zz>c-|=N>J}GFzM6WR|jhM@L=)kYzFE&i~G%hVI}B$1Og32I>EvHsgL-T^*Bte!_Fy -zQR(F#F7OhSu@d*bErYc5^k@jA4l@#;baxv_|04;hI5n_L&0>=Lr?ZLbHZBFmbA7^v -zv8=CZVs+CSXgD{17%vO~9jrw4_xBql{%^jV@aFBb=oNGRi*fhg>1t*bk(G5Q(#>9Q -zqSX*4xcJ=C)%9#UH8h<77mhZG;e_np{mtIY%kShAi~TE_QV&AX@g_?xAh;kNuT2pj -zch-ohq$NC>w6wG`&-DRSUDx^gyBY7h)XvAwUBp0<$ZUjaKUU7 -zYYsja0UIf+oL3jl<PuHgSpkGREcv`Ubn -z!0CROD_JHkUdW+e^7vX^T^%@U=>2bX!*cyb2M+rT@@{-0tNo1!Od)Bpgy4a7v~i|8 -zM?4f~p6{hlD$%+fZ;tCxc(ERZq!;G^`tazcs7q?htJ^sY8$)8XZ -zJo-tx9ZY~}!#f#mX}WjDD{~q=IR5nCtKGS2)%T%$efIC+mGShU-$+|nbZ}rrU%L2x -z+_esogf1`R$|FjmArGZ$T^Dz`Q@{Kt232y!nVh4wJfnY~g-=`Oj#P8{5Fwp8#fwt~ -zCK{u=o}8pH5C~$CQZD*Rid_-x$9eu+=Re+>1Gn=U$Zw5*tNnbWOWHgO_J8}=I-(gV -zUFB{D;iS~@E*vehDVg{t85kI(3`1W2cS`xj2~tF4{=4fi-W*{PimV?RfI6bG!Or14 -zb?Ze+0eGt->;EkJUEEHc!oME;yYnMeyhXM3e{5rcFv*dsc9aUPF5X9#_VsIpTAr0* -zi76S}m#X96eST^g#VL>9*Qh7|N(8YI9?PvKdabzdibK%}0pZ|K`tkS)kqX|0v*lv! -z@bHT_=H@kylO-LC>Hl+Owl}_r*+k=Cs*fSY(92$YjGA0ivJmcuMPvw${S{6YL#}YX -z7!EpITe@xdpX`C;!;LriEAcb&=A>nnL}f7MrlF$fQ7+eX%EX* -z+mc@Vjkordetv@-QJ3?rr7j~MbW>ujkiXSEoHvtR_0q#Q&Xe3jV~~oKpVQQ%-P(z;!;&` -z!(mqoA3ZL-mvZ6hYaTa8lBlG(<{@qW>T15t!vorBC@FBdzT1#FzqC-@6LR^-X~h<> -z)1Rf1l~Ww+!8FkuE0Z3knlo#(ML!@i!ojog}~b7fh#=2W8bju#L;dwqguBVgdfIJ8fmi!Bw&5BVLYxK?9vXs -zI0RGdsYO{A^N+fa~BAJw-#NrI>T!j4T`We$g>}`Y|FN@WDn=w0bd2l{v@p*t@9|yPF -z4BibOa%wH^irY8K6CB0fg{&Uc;i}Ea?b5=;U{~5M1lOB^a#G;@)s)r^iC_ehzxIMT -z19aX1sAty2jn^~Lhi;}4zL(t5mglR1_AE=1xf@7d!6Z0 -z(hb5hloWTTUdAb4P`e1*bKKFSS#@EJ!!qvxJyuN(edf#KUSTJ|+I0MR9acFb1>>}n -z&N3I=#oEe3L-rq_|GaVrbcy}S$k^xOaJg6SAZ~iHa6iPI2$!aN#tiRD@f^Ap7C=); -zd|hOGEPdtbW4i&FXL>@tz4IU=^V+%lWySy{hcdEC|G0mEM|KUconub9w}hiq%1+U1h-!>Vx)Qle!MR-l(5f&sz$ -z55*$K)wr0Elv7&60DZ0n?M8MB5&_x41*r+jZEDwZQi)$CNoKkBlk -zx(9*kqT(JC9wc@qre(KW{@MRH>>_c`{Vi(HIcufi*L$<=AZ%o|1&v=q$UN%%)>Div(l9h&eQ4q|5>EOI@QQ6WDlC#~&7 -zP7L(M@e!WOf*SmW)>Mgnc1t>eMlRFG9b9mjX)UbdYri1n^woI1pk$=ej~A7z6Y+x5 -zcQVUOdfj3suiP#+>_-P;&P0c+My)C3rH3oHtx?Lnj963z6n?f8R^~#j@I9ScwyM^r -zILQ%CY7bVtvTQQ@Ww49o42kV7V`pYb#(B*!V9}LnB4(*Q;J)*MFzzDPLZKWR_?NKd -ztJ^+7IEJeR>X9{tFF`YWgu4v_bymeb3skUdDBi%%YHzBul~pUQ@ai;imq3P5?EX+J -zUMUHTQW|nu|3tb}!QW+1&0zV>!+6Nkq*nK58P+1TlI#~njlX;FvKls{!hk?4uS0^7 -z-X_>kmEl_CPOl6*Wa@Xp*JauW?=$P@i_u%rE;HX5iLq_8LM!X-T@Hp+4K}v8VV!_( -z$Y^4<9-bcUQ2!dCu1j9rWh?Jy-)64G$st#ZXxI!$*@_AA&gJx8YRo7GthoNNdY2Xd -zGFGpw1&f#FFVOl{|1y($jdf@IrHCRLrOSsdEGpW; -z)%RCD2hDd9XB7Mh8cuTFnfo$zI%>Ea){pk$Ug*-`sPdx(YhC26a!ty7BAswi-s2any*pHN#t@teymCOBHN~< -z7cnY#s7d3Up-LJ%SUr0BWl54?+Xja{K)utV``i8j-E^z3Z1e;M@`iU-aGjQ7A-wTY -zGmk0{i4NSOW@M!#6-SpQF>`04FvItM{)L++l^U%646KR`s)bB)Cw;MqUK-p>^*Y2( -z^-d7VY;>;>9TX@a;@B!ji*(nAJviJ58mLyKO4S=qT1w1)kLH3}NCu?5fgF+FKUmY& -zbXs#Es6GX9ofM9+x_!ybE_cfkp?@mvoTX&0#fLe3`{ZI+V^mR5+9|tU`faIj%%3Q` -zN|W$AoF-eqAH&!TY~E`iI3fh`Q1$lwwoA{8Pgu@-pfC6FW*EnkM%ol4k_t!rbtJdv -z5r#VoMS<_P?FHE*zR@ow@RKbSzT`9{)$4l8^#04VF#&7z_hgrud)cg~Th;OM0THie -zT^_paEhyl^#6ki&4aJ_C=rX#EaWp7(8s?rl$3`rw*i -zzR`vr7?bk(U48q#eqVuc=O~*Lk=1D(V6D{?JIkeBNNb*H6RGI9YbIKIw)llD9}-wciQh>wQDNhn;s3Cbu;DdN*Fhi;wN4%;lp_ -zn~OGWmaypE4KN?+6A~Ix_USn4Wgx+Qfxm!X5$Oc%fNcaku5=1FU0)t8Uj33w&w#>> -zOxe3osj5Ja2u%k=Qpd;7D`g`upaIaAw6cRM`3#%~YZ3jg5_q2?^l4kR%QTyq0 -z(ecG(LQ(RNe$7#A0`cH&(9dr~A!XTGFKIY|qlIYtMJi3~GsBfgmf~V>>u3( -zSf1^#Nsn;-44Asm5uT=QD0>^(sqX*y)_0nvj$1oc##OXD;xYs>gCels`)5Z^DVLo^ -z`?7WWwUe4{)fp_1{=U8`9B9|;IAOjwduSgV#P;M0ja-r>YXCiS7Yc9jiG2|RyTO)i -zFO_ITZIZ57CvIykR3gi|gse -zI4;v5(N@)4)RJlV@f^**)?Go!o{tJf$J+Q;f5Up(JHuW`bUa0KxVSp>-X*kxuZAw` -z9Qc>NYY9IfRo_~U>qFN)YIv@zm(SmR`&juQFq)KWepK!8lpb-;vl}i`d&>FXcYWKD -zLIP6(12s>$LYN5vWgkOaEgyM(o4fpEt5)FFP4OY5vj`#@JE%-iKxj;i4Z^JA1%BC{ -zj#LU#c;bNG!Z-cgZO38!h9BRWI-9hkFB^yM;+p&#lcw(VHYV{mQ?>R#9o*Tm^BF*o -zdtRJCoMI8{*O^qzH4;HjIBidz(^!3ilrZb6;hOk46h_9zWHv2xNXZM{U`5(&;*V%Z -z71v_>K+gxI#RbNejATjS%?J^;R$LPue>6mA#VVae8XkLqpcEZM65Q=6LdP9&y<3hZ -z^lA^6F2)Ef>s}V9%8NtUJ#K2@uZUZ0K~;Iba{Ksc$QVQWq1Z -z&~AM$Z(^D(#>r*4$6*K;laVjn4B}VXxAPhnI_ti6kcKsDs79s%xb8H8QMrvOrNOV0w@FM!L -z+&H8^^cvHe)_B>Hygu41^+FeP2)3CwSKC!fKN?j_>zg1|8pbth;417@1qB6u-|&N7 -zM0ECZpW3?8EiE!SvHQE|PVy<#?p;nz_zCPn+~Gk@9xEwH_*a=qieoLEVg1Msn~UmZ -zrW%W8fK9LCb9}sIAhxxTg_?yqZKeNXQ1C`eJIO(-DoZ<)OIi|v&%tl%SumDQ`mR|@ -zY@2cLt7yzCtFiiRxsbsHmP++r?SR9h&x4$wmJTARMGaWUjThA6be=1aCbOE2{_69XrMr5PvLWoG=n%px$8|yc-b`=3~qRZ -zwaVyzeE+|_0GDqomA4u1SNf72d9sao!n0<5-+y!j$f3-RGMgN1l#1v+i42goJf-At -z!uTKu5)Ce3sk}Q7bU}#G?Uq$MY&hO2{Igj1DE3yU+fu&@7ued8NA{i-JCm#6ZOYqk -zY3QTA!~^M{$j*_*rk(6@9^?s?N+h!mRG10629I$#yhQ)hRofn9r<#?g5}Y;?^#0tY -z_@z}F=p3;wH7CYGH-sVLjvb6u?^$@>DhjP-vKsVQv1x@`1h~z2D&q59WA>5muSR

fg*)2e>A*TF0Re%JxYh=xzI!b(_+am)sdDS#5U_x2huMy5 -z(9Vy}ZXgo@wB#x2sHeQGC<+!RlK1QSr38EHdtZejR}*F$vk<-55_;>TpE(7A@%13vg++s@D85ZaFj -z{SJn1DbAr=i5XLh|Dm?Uk-N_93Ki%Q-ghSL#rjT~FB*|v(ZAHg*iwdH_pe!>3$7vh -z$yv)1IL|w#2^xM3L*6sole~x-jPdJErC?=u*Un$>r_S%b@)BpQ(Q+lYm%VjnN#Ou4 -zv8ylykC*rPtS?i{7RT-@x0in?z&DbPsr`tyl$5+%N%$62#7~sN6w)e2Ai%-&arsQmcz!PIySh!jowvu5_dhObJ#ES% -z<5Xqr=YCpA@DjRVwW}!^7bKr`_X_i6Za({cz_=^1&1jxp18Rm?hAH8Af+w$+PvR;w -znLg3pvZ}vwGojK02y-$?X+=JuMc|lp*}5PF_zQXV!BgO=sI_{f2;%eWWZQnlRTlCl -zE05_oNcCNUVZp($jQg%J^G^B6psyjFcCGnxSOp{GA!@%OCkV?7rlRUpq6=m?jKVcSCmVrM}E8s_ZFGwW!IL2@c|BN7Apqd;9i& -z%5BTc5hYQXPmNFg23`owR~?E>@CR%W`LXqsDxdv(@Wu0u8L&<);)8KeyY-M~3ikyL -zcJ(4~Z|~x85#e59Y+9LFH2~lhkZ>gpJDu~A%%AXb?=_8*%CD@9uql#`s+#8WBQn?d -zkxF!{bw47Kv7cfdpzPoW>%S?6fsVkh2=T>=>WTQkUF`QpZ`4agO-y(>i_cTZ>3=q+c_7{j-m# -zM@M&)c&N9e&9c@x;UFqY`{K~5?PDStAqPkB2VLyx -zw|!X9AAp5LJ+^vQ@0gp18d+BoQLE`Ule``*SHo6OS7$HnGWRJ2mjwCFNj98wnjhC{ -z%1;T2t)Gn$%Ir;+&0`)$89tmN=a$GK`@C_`^r!=LUkqar_aa3LctY6X*CECQm^7dq -zu_D8lx;xiZ))zZsfRr~gEVeQTuBJGfC30gwcsi(R+0Gw7JHQ)T^kkQ>6nLxs)1+rW -zfQmu}KS#8iBjCgOE)!&{WD~ek=wIyKX76_&L;%ypOCh$bAKt-Ter2$fgKw(YR3t8S -zq*catvD_-C!w|lkF56e%kZMEHelv64M{K4esjH!JD674%mmAyzg69Oj!@M$9L%@%v -z5|P`7r;$5vAZOAkk%?lQ*<3WJ+-B9RKDJvZUK}z|xKAr`es-bsBzCanxFM+6x@IO$ -zXP+WW+gH-URG&X|engD9WENN6MxiIDEvUbLeGfYtEhz+U1Z8p0G+1c3$0aN?W*=mI -zCqYB8Z@O_Hfc!wrBCt%j1y@#kx`C+cs5En-fn2E+7milvm;^~uUxJ$E`m4r8&K<;| -zxcB98;)eXkR)iC@q$hb0qERsWE6#g4x=2y@#Lme25oIzy6Lmux`oo844e!sxH!d9} -ztTj~~o^Pi1b?U*;^+K7hQUgH?M>a=Yyft+w&3PjjaEB8(LJuV6yV=e^uxajzl9t*u(5vVem3JZCtH?UC9t98;*Y4 -zU&MzN=)F0nKgpO94)7WsXg69@t^v8U;JM&z>H1Z@^M#VeAU;m>-Snnt+xMSmB(DY4 -zi)!Ls5H2|qU;-jXXx!_9mQ3&ouPaRhjDn@%quz!3MzxnD3asmqBqdM%T@cgJ!F^ -zS5KNpAH=aM+|vlbjeh)j3Os!6oD^O*M+FI*pzVS!n=UUMXP#>E4FvB;)A=|K@1@2j -z6~4+0@ShVl4U6;kL1ikro10!BLqjF+o5?*~RiH!NkBy{mW&7aUqu069cDc~HLGD+6 -z{ku;0Jk3{4n$ZzQb=^fP{GcTzvzxR=%Z@-cBhye?I)#W;-bbKGfWU^kgg31IBq#i; -zidQ$aXm9Rm_<`vcmP-uF@*??l5W4Tj`bo`Pt0T=Sl|n7`n(K2UZN>K5>J*e65*R3v -z!bqn$g+tP^8y!xDVeZXKOP!{qzk={@7bsn-;@@%^qJOnSl5>PdqjOHX%JbG;-~n;E^(5zA&tzJg@G_jv)8fVy=0@9=gsNB$i%vkZIaMM1YlQ&{ -z0POR(j3@3KPG=Qzj|YOc1^HL|WEZ$a84mZmJVQm8DZ=dFCn4PQabZm-=*;7QxHQGu -zi~fM2k>yPN7ZG+o3bocm^pnb$vVRJ`p&7%t!Tv&z?I|Wk{O0BdQTyvH`n$Y88?0N> -zInFwWsFP*Ntd@==druax_p21l!`(6#K5%k0-HW%yqla*+f^Y>0AP_Kwr?W3@EI|N5 -zKA{d?iv#Q~MxXUccyj}0XFt%Nsv)Sr1iUZ52Ut3 -zBiKz3@|1iye}kRce(xIt(9X+ynn+JO>_-tDD;Xbo7_%Bn@E)vnmQ8Ef#ux{C$GsK;0%n1Vn^Bk@>k{>~e{sQ`G -zo}SZu2>kTGYHARkYEbW9^auS2@zf8HQwMqL1WNJJ)pmCu)JnfU?=oodcwm^fS^BSH -zl;ZgNx>H!JNHaB-QCQjCJo@6nMCqYoR5LjrN%3jV6=3&#hozK|wW!O`TRg@GTN^84 -z7NBJ1%aWx(<1eIvWM!l=ryi9Q6B1nQCK8VEHI;Rrk1ZbrptoR+@2Z(e5M0i&h`;^D|90PSZyMZEodB=zlJ;}>yoDU3s>TF|EqhTb?I(S!6g6j -zUSt(7UQJ#PG#A0` -z{0pz=1Aks%f4Uezi}lwt<-Chs=Ic|sH=wGo<<9`3fxRA@sE;IO`GKqa7Ewi~9~}`N -zNg~VNS_kS)xmR1HBE>dk+&}}KS#_IR$JpVAxGSA1!&&Dw>GpGV#bj+)1Ui2p)OxrUSlwg}Uak>@QGcb+{HBei5MAjh -zrBaa}7qTZ6)HVb|62tt;>EF5WI8!k5Gm{K=qoBL#WRj8M5YVT^Y1NH5>ulc~3(7q= -zJH~#UU3pL@9Q4bkjCnC(P&QMY%9P?e^RqO^c=y9`xZ236Z*tMATjfeqrMI8Ch?p|1 -zTi(GH#K`2pKqFhZe?|( -z{D{d6F~k;Z3b3T~>nK8+gS`(xZvzr-#seqCwoRY|(oMm;>p?<|C~a4b^{fDw2a|N? -zs5U%SLEXmR^#ht|RTkJ)3msR#c9xMX^V53FTHXHF9+CkEDx~m3^$T|l6h8~q;>W>V -zWB-J7CMJoi_oyN!eh|b&!Yw@X5Rgg?*)r8oJd)_&Dsxz>(>YDMPd@5;n07JXLM`~f -z%){w0Vn9glq-m(znw|U`=SwCrCN450O%F1Lf$*1}QQ+zO%k~SZ0?0y5Jz#z*NB&+` -z;0zn>@}qC=Fl%Bf4O5;wAP*C+H$3Kq>n&{&k!oDGWaCNf!;xe!zAdV|zDdmajoM}- -zK(~bXB)#kru}S%|_*x7K{(6C!1j?OBut64mEvonK2*T&&`xWIHiy1QFm?c?qPj*AWh(bvU>jgQTI=e^5m@P#ZcW+vlWMlmzefK!=^IE%if -z+4m+(Z#ax2k-p^Xq9c)fD$jD?E=n8)+J$gfKz`T^R~7v6sT-I?Wg+vb+{6j3d~c>u$sEu+={zk~f2eeO -z#@`N4;*{YT#!{7fy6-3F@|`pH3~dtok#0NojkVkxskmh87Snj1qJQCSYDPyw*mOQ# -zsuO&q9-M@oZ@%z#=i@|aP3;66C_;f|Sh+^FwJcqlrY*BQjpjr7=0x-B^GP~y75TZ& -zdb0%gsH-ZJrP&zsQo93Mz-PY-Frss+jeD+kq|WKsm{FS5;e5mHc5W6HwVj(NJPHM5 -zO@X$f%0`RKC_;V&$vG3Q52*8utnZnB8|LA-_x%NLzF;#lm64QFb@!vj*FLVE`+t(# -zZ7wX8)ZHH>!8^!xS@J4h9U&JYg@Tr|lF9A`aJ3xqLWDoj+t9xkygTSD#fH|-)=^w9 -z#GO-Cy0N>$B-Ix7u!JfYrD>sIXy$EDzg^LLAI^I@M{#@7-J2buyE^zl802PX>aI#q -zs=UM580R~sh>D(v{l=I=@!7cp#5=`Dq6OTA9UlFSM|eYat(cSYSq-;D-f3AXTL^#L -zs(B-mUE^phwH1@1|98`^DB%5L!M?IH=cd2DJ}>N3@G;yL0|lJ)j+S*JX`}JcqZC!{VS7;wbYr^n^`@td -zU{FQ?%2d0^W4&13DuH43N4=@nbVJ2f-&%V)8gu<*8*r=lJ5HC -zT^9JyG)EG9)ad1yn^12Mbgg~>aM<_#bS=YGr*^^Lag?~W11m8u)bnm8PMi`gAI96u -zSH+2YOHll{UzyKHptwiZjVkbd*=*OBYLIGnX7ToUfwO_BAAJt|{;rpMIuxA>61M;{ -z)9pLn5=MrH>;&+POmY}KVm)GvT(Y^PGUXO(`KY&xLA<7`V7E=9%eWW{C$-azE3t3^VNP2^Ue@-9Li7BJsjmDR_6)|zUgRbN{kQYv1L^_mMa!|#ut2*YvK=SpQ1sjB{ -zk@q-UQQOqQBppVNp{T?xJ9zsvo{+$fJdNaq6@3vEwz-by?R36UmRoL*oQD0q-{2hJ -z{#9P$;Iwx^4cG%XZjid&E#po_eE?lh#EWkb_a4|fFe?m1%=%t_m2@0Y2c8cfHJvsJ -zU!7%NgY==ZPT54Yu`8Jr%OGV{$g18^Nr{ZRqZ2mAi+Rtw#x(~Xv -z1rx*CBq?XE=1!eCnEuMm>+8G9v^g(K`gi$Q+tsGRNQtEk>vGdj2iBlE%egi;nTa%ov_C4 -z%pX&p$8~;sNUb8<#>Ucc*P3;J5elg7K0X>Tz>Qu2Ks~etQ=tVJB1`w#(W3U$cdwa;ch#K3Hif!S8u_AuCRyLP!jfM7TAMSy2y_c -zRMsrgT{OPncQOMUsTQfzafg9BuCq0Qfh*h`QMw#Kr|?B619@%WObofm(|M^jc3x*N -z(wA69uB86MAQV3+_fKQW^G&ouQ#*85_QhECqDQ~=<$zGJUuaYeRq?$0%+-qrp3x(v -z?`OpKHnvZG1!$$}u=6joyai=?h_+Dtip~7F%dSa6wKWeEyz}5nbL-Pq!8e<~JFaBv -za?p6MqJ;53X#@)($lZlw$OalB^4lOAxzRuyJ-EzChz2g4+YwNkhg!5o>JY|ZTI_&P -z6OWWVS<)nR=Y~JvN*W%F5-(WhcfT?e4nBz=%mTYydxza)rr~EU(FBx%p+-i5bUgP4=ZbgqwOdeGqKh@EF}=*d(@Zb}$2T -ztwC;HbkO?*?Ei)gsm{0M*Uhmf$`M}Xe^wNW`e{>z)Ai(Ayag3f*}o&vksU&E=I}MY -zW#>s1S%m1#MW9suxw?@1J?(Fk5!?)fmbE1DCf{<5N~TN9@L!6>h$kRGlA)B=!=5i~ -zx`+!uGp=&8daTrvgt0x4D3=73>$j?}g1#=ch=&$0d$NBnJcw~5WDI*(g5z&n)_hM0 -zOvS8MKz?yJwQ|cPy7d8`p}w*;q=+t+rb733s^WKVhm&k1P#^|k$)F4#q$W69f$cX!T6m89=}}LgK~r1keCm -zFLdxRhGEeGvvZ3;G1LzkD_!{eO|6vbr1OJ{rqXuG76Pg|1&rwim&L^RVWY^SKW_nK -z*Gv>k$Z+Z?hT8~}=H-2Oc(?VQK#GCJhSy;Es5_U^#Cey$z2c6K{cE+FUnGU)vz`kC -z1}?Vb*@)k;Y+ngKBL7>mM=T|k(PC^S&Fwr8E|yTFr94Q=$0x>7Jbpd=0i^dmHXVeK -zase#YA1+6EUARV;%lZN(Na0exmfZfzIxk5sP&QDBb}a|+0!JP -z{4C&j{bc3*k)zl=w#%q~E1~;nV-=*_0{*7Zl-sGa;4PrI=v?h8M^OZhpdku=fA(vY -zpWI}bZrbhvM^9=;Zo)XXY~bY;zg!6U>p)z5#w30!ScH+eECsLfqc(q*>P+H-F4|k7 -z_DeHCY>do0Es!lD^|q5A#Z;39EVg>=?Mukb2HNsZLH69K$MLaf$C|D9d1dl-%2;yR -z9);UV4^g!-Y_vV133O*U!e4Ym5>q&d)_9jZ*(>DqSGqtU-(r?TLzGlsQ6*<=qof}% -z$GzKKr4A|G0-9gWqM6&V=dluF_z29X4v=A>)bA)1SI~BH67aK@&1cqZHRhRjxzsro -z&>5I)-~qhydBM@`ImmMx0P}37EO=tL>%C3E>`pYStlmcVJGbIIA^XrKM=vVj89-<< -ztsQ&o**Zy{mm8I82om;OYi=4CT5wXlomRvPbwbk5H5Dx -zX_x(IUyHi(8K`aCaYjM+z;+GnEgye1b!|QOWZSFrG3Ncd;609gWi`21^S4%bz16o# -z&gEOg>`s+=jpC+Y=1=S4sI6$)NJaIc7$p?=Hr?Z>k1G}US`DQk+~%|iz#vcKMgaaC -zi7rewIQS~C&MogJ;-~pl1Vj?wCZjm$;MkP9DUt=Hk6es^TzoKG%K6}NcOq&mW&flN -z#`vx;RjSz^BbXHrvTdHP&^fJoR8hCPEPvnd%(cX=7({e8WQ7}vCryYM5W1&Sd*_yLgaJE*hq@XtOy5h-&3y1l -zHwz0^00G!Rx6`5OpzSvlIm=Ce_`i8ZO`ZG%f+}Hr4FRsXzoOZ+m%oeH^A+N*$@k)?u!!O-{j4PK|}W?$FB3Ohu4?YI-sd0oDO7Q<5Yn#+<{&a*f$;Zj(?=Unr>VOpWi -z^g10ymL67GQY;92q6976Xm?f2T7&g+Vcb2u)kW{*W1 -z%sbIpNU2u~n~!*%5dF8tg@4(kn<5WPJlxSN<(kW8!>iGCjdzx7A$evp7kzMBi#J>F -zZ{MhM2Eth)O{;n3q40B#%Ow&B?yEG3&P#xOmsIzPU6OGi9jkU&{F@^u_qfxmC#Uo5 -zwRFuBs_k_m%rfnXDXqhQ_`7$y2Nlj?V=F(GW{K4ahphwY?uOzYOaF6gn)a==ThiE3 -zW&l0&&W~R6mSxn;lf@mcK>}N}#P>J5JCpVHILY>|UjTnQa!2B7Ab&H?l<&qJ5oEV3 -zu%>CsM9vG!%OAbxCvQ^zrZ -zE%mZg7-C`MuZqG9^%E#Fo{+QJ+qpRm4LLqcT=x$VAQF_edcheN^gY85oY{U7T?TWDL< -zQ4CXhdT2e3`+if=lRJtv7w-ctB8wg)RXX*7&LZqCx)%#Fm+iqA>d}nF%eABYJ%#9} -z|Lq0fRd{5XneEk1Q_{N<^i6K&g+&-8dw*s(BeLK_2QasPa{-pUp2X89lJk9Tk&G2T -zojZOM}tbK9O7%3>8<8#w0-DDhzZ4;YJ_5Mj9OD#fS*B#{>eCj)Tv3QMr -z<`J+7ZlU^IdMNPGO`EW-Y3ONtjYw3kOc0@|vp~cPzN`qyORX4GjNt~KY~)Vl=3`VG -zw`Z5Kd=LW~CUzx;Z}gYgZz%WJI{m@S!Df@;#mB9S*P@3$S= -zsDGNhyA`VN=lYPn@^#J7BQ+pD4L_+js`KOp8jX49?>4yXb)B{EHCaN4(gflPyI-`0 -zyA9r_OXIO9sZ9I4Fmi{Q5wYo%u=s#fOx%{DSf}mqqRX|!?%gSXF$VukjV)NbL^(g# -z@AE>VRnAv+=eS_>;2^aUD^i7GmI6X*0Yir8lL_1r#_Sgigal^Qoqd1QTj#1g3(nI_ -z(#2OI)aq2C>_y8qQcRUeuWYj~ST_sOEguF&paq`PX?@>CGjDZ{G+(ECj?fJlvDF3d -zcbcK-CS7z5gM$NR8eJ{E0>FH~$3C84j!qQvW;Tj8qQcjU5LxA1jpxioMT};3j~Y)- -zOSFCWCfGx5_(!_Zp52irvMfV?9QOQp9Wztv6I(wpqQ;QrG6U0UkQqa;V8Y0)h^JMz -zH*4;kNNODzA2Pn?ln$*$D|lWR`L|k6a)?Lz06auuIp(-QW4Ytsg`i0a_c^A^lno~# -z5)vNr?N~`}zv_a-!5}HBFWT#FUnxxwA0Od9Z;QhMaed;ZbHr|3B~84$|6m2@quE&t -zhgqljIJ!i->J9HZ5=$mp#F*CoD6cI-k`ZcORxRnBsZw;SmmacNQOFoGE#{a1?(Ond -zcQG$BH*?T$&}UW^6RFP+lr(sQo8o#cJ+3tUmgfS84>kf$mBFXRr*`1T=d~>K>s5(6 -zEUEPhf5uOjs_4CS`^iTMo0}rKh5qi`KZnn!Z>6$T3VJOK`JK;EQWq@3H=Dc=1MQ@~ -zQ(NVgsUnh53|*hAhhnzBQZP4ndcQ^&S5`(>G(`?p3*eLm1o#RFk-yyQmsB5#7BxnrL1-O(-lXl;@!rwvGl@2)QZrlyK;N%2b-I&T67CxzG66C>1AR%P^^;^aI -zlQo(H8Ug^b(mC>O>V6~qk7M_$nBV|PgWoGw$a4Zs_zD&~h -zMxItTT(=OE&?!hKyS2tLFGUAcN(-bow -zTSB&-DW~Z-+$`P_77_?dd9H3^z7?ctf`g&Hh3`9q&wOfG%`=l(&Dg8)G7aV7EAA()+)+xS^30hrQB6C -z`7lSUld6dg_Tz5gWMx58o>{) -zI(1jWLn2Sub*=}q7t)#n591`7_Gkr(w#T2e1PKz^1J!>mk(&64yR$RuUoVMI7Pww5 -zWgQhYgH`>H9JWR-EUs5@Qey^HdtG^L?R{@N7Y|OO&Qfs_V>t<=yR;h(qU+ho|J{O% -zN#1Uq2N6nqp~;%qYoduRgGNn|24qVHcUvcdl`*q%|Eds$CL6%*iFC`9}2Z;aq}Dp#y&F`|B|XZ -z+8~%Zhg8HB=L;xVr1E=(n*qB~>Llsvw8lZi9pk7&<{B;C@b-dK^LvY8>tz)+dHpTa -zo)~z^%v?eXqEiy)^qcpm8{kxu`#y4E$4A#>)$C$TJUB09E_2xEL)}NZTbx8 -zgzWQFbQ`EBASGp2`NT1+cJFzHE8PAuoNexXs`EFqvu|eQ;b~)$aeO|yE2$rhU``TO -zP)*@yuAq(%loqb4dl{gs1-!~?)E#4*Iz&iEydkDYH&4I{SiX>cI1T#ea_?>*7S|K>{&>=*AtP7ctj> -zDFGj)T{;a%xtt0)R?4wJXsSrZxciE@Ry*tCByLx8-|pq+(``YA!xno1s7v44pRGl=YJA^?X|;0A -zwM!Gl(RI7xeR}w{PvT6uSd(3m)UR#o>9+==<>Sx7Uzo4iKZ|LTrJCv{(>_lauz9KU -zScZKJHF2OEi$hO(coDT)RK>AXr;U4oD7HO6^#{`a`R_{hw$M2hYvNRMaT7Wt!tm{i -zBVlYuv;BExyv*0~N?Iy$g=bdAEifJYz`nkH8j#wgxZT>3MGJsm=U| -zTX&m0dIoe+)W&`jY{ugHy5(LwidvE)zeN&1U- -zc9Jel++3i2DdU -zc-q(jIg^Y^PR-0(IQ`+sEnZ5dWgiOGvM3#GqEF`B(1_S{8xwm2D(;cv!7%q&d%<#) -z&dQ&ZHwTsygGQ8|nR7l}J}SIpw}{%8Py*#uKC)n0IGe?CVkl;RMh7rFl-kk{Ee)7p -zm@ha2Hp95*-S?h4yv=^iRLvyvgAJb#yZUpY(B`Lf!Rl7c#!T^6QeEeSlTPoX>+{}; -zO&aCa1vf<4i-GIoBlV8g+hjWeX@DxjM8U?yKPzt-Ux@_a!o2!4LTo#OU(;w0~^HT0QmW -zTG!`~E|(my%yH~k8C$kWPvUqoCz+&zz@d}RJ{r&Cjb)Vs<*vQ<+H}b!mo(Va2C4Ja -zaZSb{4}h~TP<{UMpP#<;r7ta7gS*WR^Uiu`Dg)g8KkM>5COXdJ1owSmDw9fQaCY8f -zfsX}4mlXwZqoQLmQe&lFJDS+9u}-hlO&*jnKCbP$80Ci#YWzEpuKCW7(oenf%I{GE%^hNNA02liwb?Ys+-u}@w>JjvObz`yU6_ztE8tFdit#4%y_niCgqblwqliR9oaZ7TL}9H -z+h=t7s`UQP+z@@6GM@IR_3>b3T(4pT!io7}AKcA5!i&YZO>U?C`Ig6QBa6xsi-une2O%ewTsHPR2M3O(mtM3n -zSo5)O+>;LM+mk;1wI9X(mCwI0#?6WSyW^Gg5q6-nv!%#g_?o9um*ru$@^rH0v2SNU -zv{FB2ApZUP^)yocX!I?9vXLS$^5v%iK;}+QIuvN05(GKOoa4<5-DL;y!W{`uQq4H6 -zdqz%-gn#tB)6yDk&ET}r|NP}!wFn@lh%p#4Oq7BD3Lw1K0*qr#Umq5@?kdj%g6{8R=w3z4xHDPPJZa0nnO(l|Mx -z1+9<#q^ot&eA?~NA+2|?XVpKghZLP}3Wp}kVj}PS -zhb^>9w$REop^M0ZKD6B*`b9eFux$K0AK07D(awO2&e;-YgJGld3gt-H?b==7UBJ}e -z+3kheC)XP$c`VkiWG{p&{1~51l6c@-B`e=c9={=N&{=f%?K+sg`rZ4|h!)5{`js2f -z-+AHV)7htPOqV|8QRy$g|3FH-j($)NptZv{?DG)#T+dw0gL5y4(XK);dYQ3TeDtiS -zG;$WXpUph7Z!y5kqXvf`+OG+S>Y;)lo+>S9kxfo_kpP}~FbB3a5F17)H)*wtv?<`5 -z238@+5M;PdipEA4_}nbYxslOGfo>b{tMp4igMCXMw-lI@|VBPq;$IpuZw_ -z9>7>T+|ZEnT?hGtHVEP(aC`UejrRC-a+*}P*Wvk;Pt(5UuSp-g2%Hf;@uZSr!ajtH -zr~I)KK(ZQNvJgz)B$2#jFr%MoA3-cNhU^wz=Qr$f&Bn=i9IbyjZ;ij>jt1XI*bqIQ@F8u@0i84$rq;6#AyN -zc?xbqP&Q0Y8)e&87L0UA*h1LsJjml!fda1=A2Jhl&x5>C>7zXdbKL!pa6q_r?{Z-H -zfxTLw)VLhygz-vcu7}^6UKG~ZMVX{?^;^@|GDz7D&$m2+i{-(1wmO6{ou|A(&M&mNw0a)Q_@v8d?C)enA8FlhhXY(fjBxK -zpXH?@H6WTcgX*Vd+%U*v-(o;UtH(g@4;?N|h)$g8MZ9ifkpRNwPMh;V#Be1)X)zZ9 -zK(0y8%~$gD@W4UPGWSWF*NmlgE1A%z^yTkv*Cbv>8Y?qW6l^bACWXR+Kx`{Qsgzo4 -zgEc;X1fz5&z$1qy(_j5a?{RC}@PX&I -z+;U4=zI=JwxN)OSt>_Y8K21Lj{nD=+=-09PR(b?mh+U8IbpSW62OK-GSFX~*ZfE4(cHg~2XMFKc -zPp@1~=pi1T5g%D0T@d=96hPQzQ5Se+C8SWkZ_=GtC^DQ6!hb4w;+u;mma-PJ-M*0I -zaS%VnqLw`5`wLE6m7cOKFSP!zuid4^CO!SvIEbu${I70FZ+Z2WbonKZNn1CpPCNIp -zP{1ebb?7|j)0G1-B>{~LAZ~-|r?#o&>3HEASyVxci_^(t-(o+il>fOoM^1)P8A379^0Nb~3Pj}vVXTtd5RCXKSZvQaIXU*SQ -z=A$W@Rapj0+k&6Oqcp_eP#${vA78G}J?Z%eFAD%HFDc5eINei3CLlRkBR%)=r=@2-{;V|l@o(srI*qy7n#*El80AH$9a&p0GpnVn$wGIL -z5t#{J`l{$EZku$42oEygiEpk@d%3K|Y=A=>Wbu?!PDwj&$k#J#GBl?4`*-S{nSOHTxCBy=r^7)S*?I)|H<7c%IV@pbJ -zzWgkGa&RQwe*fO|*=z5W&d_W`dMdVL>$N7%_+ot^B!Hi&TmckeE~!cD=}E?|oE+H| -z481f%cwou8)a>>JC67gZYIz9u5)Z%iV|jQ*IXLiZw>+Hg*j~Q;r8y>Z#XtMjz3HF* -z%tUazjC#$YM!$&b0UGSR) -z{ISSm-(mpu;Xud^rII^Z5q2lke#ssLCBA)M#q}Cn70}M#zOUe^LPV!(YmI3#uj -z$arB@QQI9y$Mhbgm<;7Rfd?MYZjJFUzHNZl+jNIbKH*8z -z56mF3s*Y<0jFOkM7!c)$e1yPd;DD>-DP*w7VqK>D+|lH#$zPS1YoX#CIvRW@pP5Fr -zT<+VqFWqs+9btrHoY@$MPmg=t;{w1s$bK%RqIbpbI&Gna4;;{w8R;bqgh6dxXzdw< -zFFOJrdg!6hNeA_PY%V5dw0l?#@K83KWuyjJuheb9=w_K49z{6DIBnC#g2t7Ve&FNV -z--@0rd9J@}uNDJRdh(-B6|9VWZJi85NMk3j)M?<-KL7$ou4A0a$|dwqL>1#pnL_lH -z9NDQ``5iYMJozZI(WkzbzZ4eEH*MOK9(+(=P}2B_9^#_!p@%x%gD@>VS1(4h`vZRa -zAqhIjZdmN1u)B>^@#(X6bvXLI5>P*mukChr1i$gQr>56Fdu!Tva6G;1<3EfuSw=WJ -zNkG`{EJzwY$|xi`nqA|%LJ3uM2aYpa#6&=QcOqfMZ6isB)TU_7jp&j20mt-_I~z!SD3-)!Nj|D3Wj -z;QmJr#w0OL`&U6c&FA1hJ|l>Nh>5=PFV`yK%4L{6IPC_|WD18#lIzDbl3}xQG#xm! -zEFF-d!9_W2f21wd4rEhMhjHr@4lML=g;Xv=Ng_eM0ypmW`D_x`l~5uLiLT`bwU -zmyb+lFUxS0){_eh@gVu+o$5V%_QVAaOT}S`W1Q7b=xSb?4#)-r=t6XL -zduT^}(Vlm`>s@Kxx^?NwE3aIL%{ykCk$K!3p0qE0>W9nI_aDgPO5Nw1!s(Q9O~w-s0vS;iJD6gR2*#;kO@>z;k$v -z8y34L+X`roMy&;Y{(0RCFHHOQ>C0Uj6UTIZ$*2|;InY#!M=*NnNQ&vKO)Jvbx}J8< -z+O$@$2j6_(zVvTCadY~Xb}7XdztmVW=5)|ud6!vkZvwBMFb@^o%$3Isi;PF}Dn -z^uhjPN9CvqtJ=!^ni -zqERGqJ*+vu-Ntom(p@|BDPKO7I>D9C7+@@>JeNs+YT}+MPo}{`-JQ!6SPHs_Gd-4~ -zs~?iEI2nD9z9wRvOuHT0+pfrZCENX>gREyz2F3lqKXb8zGxEQF^RD#XKf5K}aNGTQ -z7`sRBYU`C==_V#=V;mOl-ASTA{>!>23a9>1Lih@t1iR=zzC?!_%CTO|`}JH)!40E4 -zW_if+7(8gnqh;e27w3Zm|LofBu~3x{2UMTZj`KLrLW@;b-LyOH+;b>xTDv^G=CUWI -zZ+`zT;@kkv)*0o~-^*9TxNDu;^hX6O{_|<4+a|CNJ^B^{oInG7ts0xuA#G{^TONuO -z_wo^+S5q7pnRxQ9%;kBzpz>ltLxy66Wa1BoH)EnoDmLR3y!`<^iI6*ElR7`2X``pb -zVgTEP*b$H?@#^b=hZzz;)BMC?NqYZdREN2ZOzQAKQ7CVFgeDa*_yk^q@jv$VtDeAH -zH1lWUR1bm!FU%L-r{|qj4zuFoX``=+p -zQ0T+d?7pp4#Bo0Fg~6?zCvk9>AIt -z=bhK+DfE?hLRfdrMjsCoZQ%Ri!-vzYx8B;PlV~4l|LNOn(s{2(LjV9E07*naR6#9J -z>ux9v@Sq*z6aniL1sO|-t1q(Td;Lm(G}-&$N7C*CJzdOV03}W8T}sAHPE)3guzW?6 -zaKIB8z5*w~0QX`C{f+*JF5=ZP^wBbn>mDa9ml@H8_sbY~^UXK+$s%p_>q@l0t#f?8 -z@LGFxjIDc)o@;5OqF;x?E{c9%I)5R3HccAsl0Ns1+tVQqXOjzCh+VyL7-^sD5VyhlrLmUwl@!* -z$0tV9XY?hz*FN{G^pa;hI{oYaZ&})-g@6ft)goz!OMJ0MpAL`lE_U=%k*=zK##vk* -zA5V@vGG5p7NZVr2qHi%EFl+s+7-f~QXFPyEZHgaFM#8y)xoss93bP0x7u04V8&N%Y -z=bs9`Ft1Q6SEamnKB(;&a$}kZow<2+x^0gpO3PS!(1e(G7L{h4DdKb2?@V8~{sDa; -z;Gk9$VVFK!H=*qu8cY!a+;9K=XQmfD^{iOk8Q=S0nuzTn`s|wyBTVrP=3HW5`!i^d -z!M=|YkFEU~cM2#pSD6t~#VbbpDu`MZ^N-)nrHvRlem9O)M=ub}t9(u)-z?@}RyQ(J -znDr0d`-Sw#gWDy0hqaom0gSI-!~`ERHuGIxk*gxjdJl>3=q}^aIg)WMI&xk(DzcWn -z05yD6i42wU3JOurvGmW34W%qy`K;xVMF1Zlh(h%M1BL1^_c-FLfOE{?IHwJhKd3!5 -zew>{!XlKBnHrH?qDZFbJU!2Jk;-OrajDFtV+0j3tOiXAZa%9#YibszCMZVm#11QIk -zR*H;Ql_FNdq86bUU!}<6q~-Vm4-w=qd2#`UpMQg=Mr)ym+j_{a1<>CH_0>WBa8TSZ -z(q&M4YCNrb&O7hCJT5XePI1+63o$%sF}kgbhQTgEHK1ogcZ5dBilNpGzw)2|PkQ75 -z+d_J{$5+6xg~G;BPV^L>?DX~8y8Wu>KPJ8IZ#`b0cH5Lb^7bpz8-MfD>3etYi(LU@ -zveQTPsxak;4y*J~DJIGaofWw0mdt@D5x86@(UDm2pAJ6Pnc*WGb&sTf`nfgf6wNK) -zdFlQ|`;cUz$YagynxA887PYU@9ruG9iQ3r^a3f}KuSBU!vE -zz+%9v70c33|M=t5hrf7(77uiWjusF`4=bNOQ_q|<`T%lO30qx%BY(&Y!gB)U5nQI4 -zM|u$*DHbAGf!5(wS}w>_6s6~;1;RESz&-LuLzE*rf%2--=oFM$!Zb3sKD~8fOv-^Gd^EZE+aaD&(R4yVHS3cBcK0JeUsbelYFV -zb@nN%$Qp-09DI03I=ov4*X@5qiv#*(p@e~H-SOZbbSn76@T0ZA#y?N^GA-f7ARd7? -z@r%i!U^qOG$=0@Q+u{>#gxyZlVOZpI5P#vLxne(f6f&wsHpx|d4o8bq69@EZ%l+E+ -zCfOW6plhEl!f{X9?yP8dF9~_%^xVStx#(Ym@Q?C~P=&lkUsgobzk&~?ol8MJEtfoT -zJm;Kq;;P@vgjeC -zQm1<)bBtGu%q~Kv>qQ`R(dnn39t+awBDZ-M?4rg7SHEoA{Y~GDZYrIDEu@JT=Z{Qc -z59w2F*hDzn=63hQgD3%dm0go|SK=^RbhKS6tX0$7;A- -zmdVHz_i+Y>WRX=y4%*56FOss{m*BDVuQvV^bu{@Fl1o}XEthN8YKKP9YVcDy(#|^T -ztn}dze>e`-;}rUkg-UkY_rMIRvs_PwAG>w?7(q8+*$E7tZMRZnW -zWQ?#96YpyjLI=g6I`?I($Fh~{bnx+}w0sTM#w;EoZVl`42r^QayG>Tp%vf(&BhFyX4()Nc1M%4gtM -z_>1fX{~P(}v4W`r?;b7Fwzy@hH-s!MLmt;|PAk`*68;K><65&>^14y^Hfo!a4kuxe -zigPhkTz$FpPCiYEx%VxXF4;zo?`t1fc*;|rlCHh>T8Tmy3VdI6-eC?KcW|u3HO?5N -zH{N(-qnk*p`)-|gp@uQQ;eYMwHnt7$+&6n;0&v9PY>@aqB8!X=W?+uAmhN!=nZToe -zV%$?%S^6khjvLoyKjM42!Z1)8*i)z5dCn=E*sY;)d?Ib%xi8}w=@H%hG-IZUG|nVC -zJQMsQ?=}gG8jtxxe>FAr)sJ+5PnMFNj&@w6lcxuM%nXa@q48o|Jsbfm;0%zXD%@X}F4TeHKgD3Dm -zGXva^g<2*;ehRNJVDo1Lmuc~EZA|o+>)Zfs59s49GEhnk`45SXJb592a(Z -z@@YEGy>DJmRq`{R`OI|9HP@sUyx;{>Wm$0XwVrDu?|#|#$>+E>%tE=Zc}=*!dImT< -zctVXwvlu(v|01Hf!2sn(bZO$^w(rMjjVfsJ&K$l0X-dv9)-O>9iUge%Iwe1ui0lly -z(V(o)o^aOch`#y0y=nge9UQ1X&PE{04vk$ToWn+;-{a)hwh-rjp*uChi1U+H -z=z++xl{$c}$Mp%f+ji_p7o4>ropbtzc)fQ*4@f5UUiS!Re2r^| -zfovs8?%7DiE-PYTRaDb5({Fb37Mf+f2foya-edkjcrn}U3r-%zpWZyO$G69wu_`_O -zth~_wPe1rK=_}f8en_v1^QsDS$1y!Xj!$3eD*CJtK -zcUtzRho>jk^}DSl3w%Nv!fh-;$(MiS)!i!bYg*zG}5bUuG?>1?L%qTNBzMI@x=Veii>uU -zYjHcbv2Ob?$Y;%;CTy9h`iG)=F8U1pjMoEJ0c64{e6<4HBe)5w?%~TsBN6k17iN+1z -zVbG}W>%K|aRQIGUM(6g{1~|rGTxxJO3^5^cAI-)hljxEr{hi}_dF!;}fe)Radg_7R -z_1EK<>&H@ZIBrolvV4~nVifgH?DWFSXkvuq(%E`DXqKcz@r@I|(_+ET2e<|BV!!D1=L(NZk2S0!Ml2c%Hx^L%! -z^jA0BqshDu0_4)1azd{J@}z%U3j>Gt>dRhxcE)u`{vmzk>wkRlI{~|H&C2wX&$%GZ -zy@)d}I0J)+(^Z}A`m@XlyvUsWejF=*^MhymHWNbyN|? -z-vp*qmu+H4T%n88n>@knLNgNOnzYh>)4yK)a2WjIMa>)M&!sK?U$?iW!7#~ZaMuG* -z;cwtv_-hWCrkA$@Nh)&O>4tRHfcFp6z?12Muc1zOU03TG(x=`8;d} -zxb=YjFklOz8RCLH7(z~nU@?}-R~azDNu|H(($nKaZ<0U#wL8M#&SNb0f^}3jDSGvG -z1~~LOk*=1*I^5}=Om+9y6Ul@5EW76UFxu^n{So@6pHf**gof$aLSf@D|B!veU%V2D -zj=^r1o{_FO?~Dy;9S4KR-Ks;};=Co%F9Pb`exKxo#oIV>7;KAY8M2BwhZb -zjX|VOfAhXHu1P#!o?~p}^_6AXX~IJ|9u&tM6u-=mk;iZSa7ViJo}B^z>dP+~8(%MF@H6E`fb#-$E}-yqzHci$w8Yv1SCh5s9Ump-Y> -zhQY8cOh$hnAWu2o`X}>}z8&$fMmomhm37y-TUMl(KO;Xq`TkwI)3rD4(739v2}oT1 -zZEuWql^SDuPcLR(yKKi38#K(xQFoI^WjbFI*W?S#GAy*$h2z&@H0DxYIEN;k?y>Yh -zjm70^{o|GqhZ_dFJ$h;Al+s@dl7l&-4qtW%Sl{&ezaT2z1U&70^>a=wB_=(|-Jv}C -z=M@6k{BdSjXppJ(NN2Y<))Pk-uo6c_52U>_Ax_ -z=T~ginH4&>g7Yi%Dob;&1?O9=+mim*)wc&7FTU`s^w@1%V^_g=`OKgMGRCKnT|zB+ -z25=bUv2QWJMh%R0c)dQYuywUMnHmt2QdaAfo*UMaBj4MnbLL!mHKa#?9xQonWE#$X -z4R1n4nkPpp9pmFeU%oHB^^d-l_Uu2D&Odu|`jcOINqXreThq$KXG(O1!Qushq)$RD -zPp^1ivPeI>S7?MmP70_P;7$+C6NZXn!q0GLZd{h$`cr2o-tT26 -z-f#ZT?-hgTpq`d64(9{F;$K@|u8$;n*cm*;VcOx%c=*xW1@6*VMDirh_-`xtz{!Uug39Q%}!}+~2r) -zSGs41z6PKt`b>;@EtiAlSFJxyhgRy#9h&s7&;gd&*Bq03EflO=w>5qCI=w!iv+8iK -zzWm7=o3u+MUNzvgb>@>2EXKQLT+C8t*&7CVJmLUBlqSrqug7fG=REf0tVZ4)hQ`wg -z8JolBo=7?SN@|?ra%xVQ2|Rc~uo|9i8-ZD&4>GV;tm&Og1{no9?XYTJ{$5kG73+M -zEHQ#razx)@03j~-Bl(4gJx)4>G-V}kDaLv1S4GcQzQFf*;DK!}A+}WI(|RfBDLPoG -z`yL-x$G7iY#vng{hxCUwu@La?cfULR^$rs+w*IAE4trs8$L?(0 -zas=(DR_Z3`ymIcR?ssl~?%Tu`0~9|J(E*0)$$3MR{U4PSku^51nGosH)j_MuXF>R$J91D{wC-j?Ym@WQ> -z&uyFo_nc3O!nnMF0cV<2m_fucHTXT5c^Vft?yyILs4)*5jbE6y_sMl5OXeSF+mWSOs -zSKu#icpzQ&$KK4CO4Y#nvQjxu -z6VY$nwLDE~QZu2?`hi{^|ak|Hw~8P|mk*d05*z?oMC0_Flao -z!lxkird_-Im_%jjJE!puBub^(i2!|aTf)>TlGTs5+j=DaB2fW`AcAVaXXB}?kgCq! -zY9odYN`=n~_TE*g`4nX@fuFj~gojB!AwNB#;k@`um{^?o8#m>LRPr>uj_g@J2m%^@ -z@((U7xmY$cFbF5A09B{?Q+%Z+=hb^z7d)gK2+|HV!pip@U7e -zAmG;o8k&bZa7Ul8+#^p#MU@oiz|Xk|WvE_gYIsU-F7afk8Q_^l(6rM%rh$FaW-@%$hZaH3#|pwL8*pee}C&_pY7sS};mwL?^e8 -zVY_Qiz%C`MoQ#5PF%T(wq|{oWs2K8-x9BKi){|>7WQ1VN -zKp6Shtst5IYJ{UbL?|sLLPW@gQCRsANuJUXAz(w#NSq!XMojSTh46FvA#|)Dq6Bop -z=(w2jho8ME{oW_8OXr=wHl2IcDe1IRHl$7K*QA|$#?$s)`_oPL?oRhTxHnz}n9yoX -z95y(q#g6aoP8YxFL&01=_vW&}Ghg~G4R+j|wqG=1i39m?`H&_iU-Hl2nxD40V+DR> -zE{^-f>S&u~$L%&9hDAOz*9r%wkZ*^d^L>2ZMYMCZ@j3d4iCEj^rt_a_U+r3UiPw=1zPikb5wqke`src -zSsyJn1AJUtAOG85-jhD?nVVt<-S`2%<-I2k*d3FB!&8%G`qBvp@?wDFN>7kaNSAa+ -zFW%|)$<-O)L7-#&aldnaZQr+x9h9$zakve4qVJ6t_0-`(_e)v#m=EyDbRx`UJv85T -zktbf!yRnA~uAv*Uf^ajA>Z5-ic;sN(@vyF42h!jCXlMG&H||Lf?%ExT0EhNHqJ>Gl -z->qFqJjq?gSHk$5TkH-%#}pb-=M;8X-i?%HE@#X7C?nRR@j&>_E^7E^9uu(;tVAN16W|tqJmZgbkz$Av!4ee(eCqeEsuSR0cN0m8abv;#uq#| -zkmo)fxE};j$DTm(XpEmI19>A62tqSyrYowGF^dj`Q0G!vG|XvL(p2z01(P##xY{_K -zF?I)tX^I{Z&KM=P-M=^e=-%6Ny~M!?;f#yz9WYd|!B{B{7#{`=iwk<+gRcpU@hKDD -zhb)Sv0EeIxo-O{R`Y5Iip9L)^(f)WBQ|*I-Th~kk58y?u?9C_q#qR31Iy`Lh*=9q? -zhgd`B{q#2tJ0eSq@C2Ulorm-W9XJ?pT`^SYuD+Xni`8I4A0xw -zu-HXS$FIpPd_Vn-htemmzDEyDA5QxpzAqix^I%M-#`t`)bV=Tst5<*pM~{v?^*;{Azj3$2)!7P;d}kUXw0Qu)!9X+YWCZ*$o;U$V~jf$y88}J -zq$_^?hFDBGBw6J3fRUK!Yo4VYCd(y{Z0(O-;+l&Y-aMym*aFDY1nHBb>DB-C+v%V_ -zIe2jQ{Th#C+jDjRpS+hZ%^Y0<6!KV@kr@_woN_V1#*G>Il+zEUFYe~O2|6cTG31OQuKl*#>Kbv;`3+C8lDnR$?TVfhx;5se(7 -zSdgoU!&<$@fF4s^;v`cRmx^n|lmJ(jiMKq3Qvd)U07*naRGIV-N+6$UcD(RCx$Ko2 -zqDCEHrk|xQpU}KVZ|<^loc;69KR^BYum5`b&Ue0({=JUv)%Wjv-}};)S6-Q(``qUSj^OFb@b*te8(i1P@ZHYZ_c%G<3+K@(Q5h{aa|2w! -z+Cr&ia&>qu_3L6RtdkDoNnBnGAU*4aez_eMgJ-eHV$fTC -zukNRH9PK5o(>+jSaXIt0t%qzIVKdIB-Hz#k?DdAQtP@aA;K)NeKBqh`{V_4IUpj_w -zxeG=Mjy!C{j=>q;_fe(_$F2&yU-mmYl$NhvuY2tb;K83E*>mPF4-L(Tl<+y21m2J9SNU>iv@xS -zU+lWI+HyA1b+%?^x!ajOG=xRw;2px=!T>w1kUzx_WTF5pHwh@J469x{3jtm+U22W?D -zjABa&N`xH-N@Z7oEFh(8n#LjnlQ@`4s@g*R7W~=ZpKH#vannEs2I4~_uHZrVq>ZGf -z#$o{PWHOmViRL``OU(ss3JpvT2EAdH&k$_&5B`ubR=1zWn7cr^_z8Eb=)GKN~7N -z_&UvD%A70b`s`D#!}xW$uJdhgds}nc#;3lo!wtX!4$vNGg73?E2!A0O;D`55YMkpC -z;F;$=Ifh-Kdo&h~hT*+hPvSUOsTfSWBEVmzlq{ii2h%K+UddOwNx!aFLh?nC~K)1;O<(ZDaNZq^CdsbR -z*HKD|B$W(}?x^%m4>0QN>>|@_=KeyEM+OsEja*HPNS5^AcXVWZ%nO%mY>Pv2HAf5~ -zQBLyq8ce9O69cmhtk;_Ceocd>5Ocz7a`PXdFr0gf*{$&7Cr#=3qd{+sE{=#|6)G}Zo&5vDIgY{=pUAEx=grP_7BrY -z!gNgiYby9oKFyGs`Op!54sz?)uTLNR;0M#|UiZ56)vtavop;`Oec5W*I!q?>r(c)$ -zfgjUNjjO}>b$F_RjT62v@;ZJhxWy1p8_`4dLuX4L&DX*t{1O)epW1*8egwg1W`NTU -zq&M_aF|K*y!K(lm+$>;X@bKhB1dy(w&dgEkuD}JEpq!kPL)@tFh*vszM!GYF9?bE8 -zMLq&J;E0fq0hcES_DSm&&Y^+(ZP2Xx>2yy;j71WJ-#C1(VX=!cURf5i7641M3b-OGzLj$FwtV#@MyDI8O -zM+#P*1Wa`GYMSl5rL&7n;hvxQ4tvBWUBe`g!Sdj``wU_ryPHL+3GDF=u`rciJJIVZ -zG3IH`C0X@&N=vhZ5|Zpx@)U9=(O!l;>aW?MQs$R=k&Bd4e-tr1^4PZ+0B&t2+Q4DL -z#j5*VJLzBkS -zRUSo;uv0m{&@OF -zl-rW8+4QeL@SS`@m1A>#ycAz9z4X%bj(5Bxz5L}bj|G8EoAQaa)-8i-X_sMLH7M^) -z@lETI*4#$PsR?<{cE)*Pm#I -z@QC=5=K8%uX{p-n!zhpDO}{*5{`m`X$fVfXJk57r+GEFLzS*U&c-u^r#N3fZavuCm -zXb}L(m(+xuhU}KHvF;CspY1#teCARfk5~+F1yo{%V*eSB+MoX7rWI-b!O?VB?+=bC -z4+r?sSf09jvRqGU3N12CbQ3fMgfOr~a4&d>oa5!0_NDm9(}EA;M^?&EdHRE}JW&r~ -z6Fs3^w351{e@YeMM}9?R!v$P29G{hFm_?b?3m|oX7dR0|na7Dg@bMRAXQPT=7Bx8S -z@)?iPL58d>N$FUzxTAput`^iBv-p1dgdUpds%ZjkhFLzd6iL3oTewb+g%Abr3B~GT -z$N+K%|Bhe2I{7^L{h%$U!AbMALg(jx?&s2Tp7We^S(+o{lX-)l%SlP-!HC|W9?@snio)pWEb{6Pm~eOjQ(~Be>pshN&V$X| -z=sd##Lpg>gjPfFav}2ri03UzRzdE}eD&rQKJTj?|c`G|gs>yqnJ;|8FBxDR(sbmOQ -zLw?Y&^*l67pzM66o!Yfh%Ogd^Tr*@wH$>=XSmcp*^f?;tt8L({TQ!!Rezs0rRGvJ6 -z#)m@CsrYJOLp&NyB60Wx6z5>XmirpMNuL3HF1Npa;G?6;2S0F)3!{xNI|el2CztWh -zj8RNM&+y>egy#@{%8aI0?GL(ATfzAmI;Ua%YHfp(@#Td8ctW}_+PajS+Z{A2M!4xS -z%<>t10KNu)k;A!{t8ut$5yx9}Mp@T~DqbTzNHa -znLe4cI=%w1f`hm8e&h;0p(AXcPjoHU`@H$xNgc2o3(^^uGTko1)Wta3w8c0d2mbB$ -zW2i(O)K};i_3EDZ-om-`s>TJ6m*$Va!_~ZV++z0Kez)6xr+XF`*K0lxquowl#i1?K -zeWT9?p0a~DP^KAwu`LuS0VDpZOdRD?L2$bawK1?#DK5O6hug@HUy>vJDDcKV8d2b(J*HduGO{`Y9Li?r -zepma1+n4j(cb`9+PhRT?!`ZLn+u^#*I=|zIGmPUp-D64}-8Y}fwGiy0sqAj_2bb;P -z&?ow#|A+2kJdvK53Z67zxI&h6-`PbX#z}UIMIMhuo**^y7jA*cV{cX|?9D_E8f^oI -zN$Dx;M$?neJfwvH85KNXmrvvIYvN^s^g3}~8KGO}lx$e!1qZ{2&mhmh75HGdxQrYg -z1RwPlMGApYS{?*Cn+;^f} -zy=6dD-`55@gn-f_(xHfSOSk+a1nH2F5d|rUp>qaAK&4b-q)}P~l+GChWsnZ(7+@Ib -z8gj^c0R6xB-cO2h&R%=1XFn_UK5JW8^G$llxB0FHQBSnGyRgPvqlf*UU6ndA_D{Dy -z%sgJPABa2MkIRSQ^iH%6D#3@6@0rc2Lp7@hna1{F!Gh39#6s#lj*n5jYE~^3?n;RS -zYyto2y#A&IT@X6XqE2glC_}H{*v)_|JnRN#!uYC#hLmPh;or``?frTr71yWHV#^S& -zoBVN-&0k>hsUpy`hiWqH*m9Ax**wth5s5-wjIz~BdmA$*sQz(+L;aWX@vw$XfL=y4 -zn<=mmet+g$)DnrPw?96gjE>-W4A0tgP3Mr3M>_v}!ZlxpV(*V%o@i{4>*0q@IrY7j -z*DKJ@#&CbU8jB@r_L?PFyzM3WL?wVl$=wtYZLq7xsitMit)lp#t|j+maeCRSdG+9< -z$s}TNLt6NwPSfWE#{<>z4q=%I#>9-3h*?!UJ?Vs@=l7`owJb}?4a)eSItJp>@KS#& -z`s8RSEUV{`@?CX5ExpqjX5^cq{upPT%y8@S?P;5FSz*5z1!D&P^b-{9 -z2(`!n+hiEXPz=1qo?`j{LLgz1f*JQ8+v^5;Yh3JcS)vbyh?SI2x0KP-8e_&KlX-_ZZ-5Rw*1SG5ydSP`$@g)-xTO{kV;+CeNmxul7)lC|fY-ZDHN#qS3)$EHaKD<62 -zR=wd8wvtyc -z@eRoZqdbEU+CLij6!({}RqgWo{! -z#Gf#cND={(%} -zWB02PTa=CKgfkjH?&5xAh(F&tVKv3RL3? -z+yAwn-AN$Y*rl29h9_6bV`wqqutQ`blRlO{ZtCY{Tc+pyi@)@k@AKpZJzBZlWRoN3 -zW=c#u`2zya6#k9BE~Rj1X*I7cimx|F$6mByf0OiT#XbMwJCUSwh5qq;mD^BfW}3A* -zBgKMJ4mR-V!kp*YLBip4p>prVRj>Y6K>wnSe#v1gI?Ej(c+w+j^P&sLj~|`M#-Dv$ -zEhzWj3^uY3|8kniK8;q4TKUL*Ry@OdN>1m%(9^ZtF$~E_>2AwgBi|DKGi?~Nv%+wq -zm-D?m!L}kvd(6cj=y1Q-?OEnUT}t%3q}5)(rrR=deD(7{lHrJ#=qhk36W5v5$M-}Z -zNMleVVr~xOlI5Y9%l>Ni49OA_TMRE0gy;MhtU54ds?R0qwA4OMFEm(_ZR8haK-c^Z -zcG)nCL!>*CHj4KMWf9r2VxpmthjLZ*zNJ=J%$DZdyK=B=Gr65XKieAL+we;ql>EvS -zBkdy5?3H2^n)vy2@r(?p>-`S6$oe?bVI2)(v95xu1Mnu2?0v7R}RDsgZFKnN1Pv554v?x?-S5M6Pg^;XL89bP8oL*0$waku!Sa{}y4=q2g;kd0RAdU5P!*;uiz2hPuXzn~6YP!n^6&C+FlkOIDF -zu6jQD{sgalI^Pu^Jb!5y*CC=OU{(@sMm>c~s?K3RY=nPo0orX>A|aSouZ7-^+r4Vk -zxq)l7Jn5d@vIKh2w5>n&xp>Qg#}1SkP6{QP^h9Bf?7p?H-QWF6lOrN(;76CSKof-b -z%^9r-450-;he78zoi2>#GJYG6x7`k8N-w#LeEnDo-uo5xxmqcid@_vfwI`bkQdi#M -zLq#{67Qw>k=&B>9PHR7RovpgExpRJO?pzv5)L# -zf=>U7eoe=$$HelBV(fsMB`2?2`r;u;%zMudWhLz&JCKPI@r`{4zVc-xcbt1e -zPNdYT9245cPCkFS-+#RLJ4WE3RU64L$u0n|3zEWxzPgtD1Wt4CA{`XlqT%+HOr~uR -zuc_Qu7b!=wSrS{+GPiE^N33D?g$8hW(CEFbQ4ztYbcH|PYGAP)3s$ZEF`m7=Jxqb( -zQL!BmrVN7P|2nzs8zew4RY0~(UR@6py6Af3-ew#xs4b>&2vH{!L1tA`BDOve?3qIb -z+N@%=1|G_`lM3$5XgG2+{CAb=bItt$#bg&O1#JWR-p9+2Uq^WoZg@+9DmsT>A=uq{ -zA9?L+>YTXrKHiqmBO`KIpHR8-K`A|`yWvdh`U$%CZ{gmY{E)Re^bWG+R~FO1_HbU~ -zPml8HQO&FJqLv?!p4m^!#apr>P&TOvbNlo}*ZS3}hevjipwiniU06Xo^c@Mxnn;la -z_f|zOxs#+FxU&{2m3VD^PnRqz~jhlqsFjo(c!) -z)qiqvD$E_x+dN|X^YEHJ0^9gP_CHV{z?tCm(M#}`-t;PtRRi~@hI_1q^JzDw$MMh>pm)L1%9+J<_zmYMFZ?@i6QJ2uTc@L8>> -z?dR?K);=xNda2GwlP5ZAiY7*7K*t^3MV|laQ<8eG6ZLPCP?>FU-Avlodl08H4zwIS -z)@r6bMjiiD@-Xhde_AH^jM)e0FJ4Dq;aYEzp|#bxHMHl=sAz!JgQNW^&4#g}O0f+K -z*&2#V1~coUS!Bvg|wBbOmvT9wEH}nz7!7IJEOW5iIr#i||Y)%u8LI2@Fy; -z;f~p;O{LCFt&Yo`BptbdD)JiD-;kLXe{#(;<%n^v#{O$L2l@P^o2^X>SJbW5|C?tq -z^h@a%)aOmxdDpu2p=mDs7o)T{ay22LB-k5m)h@`P({B@?(XAFzYwOEIkLHDf%KW~j;*-D)DEgdu(8L|+kSJX%d)(FR8cL~_L2JA=f9d4o^Y349p7X?(awUtvC3*b<^6Ldelp^9{Y01TH$SbmeyH^D-@|Uv=}xjn1dt -z=9PzOpFQ80{H#+bs_Cg;gWJ2UHKKh>8+}3&uNxe{pI~_4`{lv=RLrIWI)B}XVuPyU -zHy~<1Bx72c3$DrBL7)Xj_;e`u% -zNo#2;t7law$-WlhvnPd`heB+qP4hVI&FTGX;qY~@=mVKbWa8RK%r$m6+VQv{yIHAJ -zrmNq(!VoH(o|*$(^nkl19SmvGyXs5@{RxpqQTjC|EmvC)IDcT-n2b$3)*4P>V8bm4)C~j(Oh$ -z-qF~i2fhuMv9IG0D}kOgSvsGI;xx9bpQ!Cy2CdpwlX|n9c|N3 -z-du~aK6*7?8*w5w{AVp6qQHK#S+gn(WJH8hj>l|65tgU;++y5r(K0sG_xh8*lz=Ba -z=zRmk68v{($g5@M0zU6n727Z0441I$;#9}6@y1W)jXUkG*_{$3lpe1_J*{TqcImn7 -zpi=AvSox1LrABP_?YMp`e0XVbLTagrT8KSyMJcI6bg`I9OBQEUOqfEFVtFtGd{k)9MM(gann0PW#575@lUBkAwRFd8HL5cW7Ji -zp{ES~8`G-zxp=)YHX*VlN#$*oyO~YDPZteJ6s@uAktTUD&dcs6T6UJ#PowBIpo`kW -z6Kf1VA>bdu@8V*@-X3FmZK{86?&G%GUdIrOnNLCFwbpg8NcX&i^?HOUBG?z9u=VLh -z-3Q)p{!cgz7OmX62@9B(VQsbXT6=K9R(so~u6{_+=)z^?cet6EW`6O3}X;Z%d6*@}xjakMyb!Jbfu>z!9@ -zWc&N0T2(r-O5SNsLgcYMY*a;D@pDhBMFzh^)0$hh3>rf@5>b!=Dno3=^oQ#=2JGAK -zbUn1Onq-*eR4?vSVL*G#lcce@bFh$dONMEblD0K&wz4=jcv{L`mmJrakAifgoWcAs -zW!;%Dg}vxyuFdSr4zRY>TaDC-ac7Y5Vr5MjUz9OURl1%YhuL(GMHNO99EY{Ycggqm -zZ@%x?%uOjpe6yIU=P2S)9$soPZCj>vN_ssqu}O4KN!NM@EX60OuxxCb-Mh>RpRWb( -z`LLVfO(dr~xb*>Rl6x`%6{@%NuN1(7@ng8y_!}y)@^31SZJ8RbXL%UnybXd5ai;j? -z$tR)WggQ=Sno>@T)jWj$onA+CaL+XEGakvHD&aD2LwSJNy%@Zq&PdVe4b|#E)}>*@ -z+i|S`7$1kULZoL4*HC`fuZKW6d>u;^59Bu2T!O3PZ)9awQywy|bx2X4WZ!E#yx;g9 -z4a{kR0Rm1SZM|`{U$X^qSoDiUwe)f2OOZIB{Oot^Qis;s!X2=YZeM=#kZRW+yCe8* -z&9#T#A}m}ieXD&`@`mzHlI6woaH~+>YbP_Gf+fGqbrv4YxDD7*7tGjXNHy*@YHQOY -zUBLH|;{(4cx=A#%L#)Yr!8CPgxm$&Qpm-UHXNT<=+?(EQC#~E(J4M_t-mLv{ -zZ3(Q%@xp!lk|o4&6>)9a?dD-6sE75diM4k3nJBWtvpWeldx=RNO&bB4rzm}Pv^sRTn-HtYc8c!DM>T;c( -zt%`A10ajnGE9lw%Az7+DP|4NE*4wZvY4$^NDyhWT+PX~~CK*)Uu$1u8YFKMNxuT)e -zMi|Z9Z2~U|Y=5eDFpMa(K53>dg-d7;x|b|JtN&V33a`{j?_ZM=tiQqaA|zIfiLgQ@ -z3qknl5>ap|LDqm)mLdnR2a==6u{f1}=(9FXje)(2Q`I_@We3Ny7aqP0jY|J0aPp&j -zlzVkA_PL($F@JC~WS!c(0h*Ok -zu^nj)c2a6?=Fz(jFYKW-gE>DmNG)D+QT?_QHhnYZ!M+=!%H#wWSDe(r@TZlT(92-= -zPu)ab?kpsbXZmT5VyujktkNFzKLZa|f@6*W`ClnFvWBxT~*>PzY# -z3G@o*`~4hj_^cd6>w|Y8GHZO7R9okPt8dWOnkm(jU2-O=Dig>&Vo1`)*K%0#DF+l; -z&>tJC>B(#xtJz@V(RLHA;Ai&x{x*b4N*0SxqcNUJaayrG3`(5^oMa29gFwTPc=9Vt -z-k~DFi`rpZOOB+!GhizWB5raFmBX4nIW6wiX!s9zrO8vyV9DvJ#z24MM<#+4rjW3Z -z%loXq@2^GGxb1C1M#>ba2CJG`<01nUTnAo1CCyn=SPh+iQk;O6mU}Nnrdc&U-!s%c -zjASypiIm_drTC#r^9Gss-+f=bLZa;StLB4;3J=RNmmty2F7{>W^qb-wIqxT9KeK$p -z$v*&Ja%lOiM*@Pw!Tu7+emSJ8eu9*;vJv!X$O%8nF~K{Z6luibLh%;2-X$$rqV}{c -zfTd7o;)HhNTi<=WtWuF!6Dsc(YW{@HH~-shzHeuxH=}RCJA-DPSlmFH&F>H6TXFrt -zRPTR?zP9oBEmjVf37)|K=bO!qGil}oI08TV;X0z_IPht6fn00j%Z>5?L!6$50wLql -zyL2ZjU$2#DQ~J1kl$*~HagHQe+v#ThGB)$HF9;t-&CtxN-|-u_lUF!o -zxom`TdU!vGk?O82wcf7TClbYk85%Yb>^PQw9%sS_@6R;xR#sCBnQ}=Oc&+NHgy7s5 -zWO2sYBVgH9NaM_-aS*2?w>G}mVU(;_8YGiOzg50YymYb)kx -zofmnyR_`C%2t5v{1%g?=yl=q^m~J%$K56cZ5mz<3=04`@Wl|wnn@$^l)us=nVE_a( -z43ct9RCBDIQpBR*`5LZ9floCahD{C7PR|1NOr<0|W?}z1E|cZ>4!z%f7oWb~ohVr> -z#HeWKpLiL}kc5;V&gs-KDKsnjE_x@;=M|NidE$w%V*7$5NLJv_{b6PT55omE>ks46 -zW&~0l!w7q!;Y(i$RCuUZ2yVGNFb;AiW*h>K`^d1&&Nq?m*Er-JP01)lMu@BubHI~G -ztDY9@|Dc^-oZEjimH|uXnCH-XG#|wN{&{H(Lj8%1Tj?T3HnZxQkW%5Wmsh7%ofjXH -z)qRXnbZYVPikFgMN)NC?-H_mUgu}s8m-km$<3u##CVWg{(m93j;ps(=xkqxz+tSLH -z9NmKEU`Jk!M@4&2@|q4kH)Wgk+p-(v;H4Bf2Ap@_6a^e}D$1j=JJTfQ2@5KITgRuu -zE0v&xdr-#{KKed980KAm)N9Ft)j%{@3#S$9ofIEw&r}27&V`sR{g{c@0DrzbF+=MR -zr(OA3^vr$Du!ZJl=k7LQQ>F3ED)qyt^spe|S%7YA0~=QU3~%043!xyK?@kv8P;A;` -zs!!x*BfKfPmi8NI?-oM{u+9qWHG){w%+0MKZfe803PjqW#I&~L6vO;y3qGV&x{hG* -z04DxTOYytu_#UZ_^+ -zoM&`7ijVWGl)#iAaIkZuRbZ}ee$}k<{VMqf(80wA!7LTDn>BEaELbP+N({!+FAdS4 -z_+JMqJ>*0gPflc!g9GT%QB-jaGuCqwCr=F_vM!CY67f|Y^P*U!dl7K&v7;lO@{NK( -zI#%L5Zqc=%96x|kL{fJB0{V_v?7)o=ZU_`$$abROo4TDssObTT#VDQ$dK#3Sk~cgnW6aI_PxhmGw_2TvC%A;-6^+6BfDD_q4Y$Cqid_;Cv>|{0}He= -z^VjoO9x6Uc+jM-|i-#$%3!w -z+n=Vffc@d8cf|UCiY6@wpN8W%dfXDDIt+KEF^i|B&vUJ6YfQQ`3zHc6wpVXsVO9_2 -zf02N+H{}=7t1}N5?hIow1{8C?BdDj~rIBThxMNY&yra@5#z1WqFk|nvXI;Cq-Gozdn#|LVF_M!9iHIdhDz*E4wmd&D9qH}%tvOTi7o?q -z1IIUeQm*;@la5Kk1Mrf8)%#1g9xu|}1s@QL>GC57tcJX+%GS*oeYYFa4vwxw -zWgKI2I=1b`)ukAfFyG?hGuV6Uc^@-UP0}`Cc3ZMuWLIP_r -z&z`8?&?8X{hGa|4L(Zfep4`1ZPbHhFc3?Qqm}$Sl!EMrww?HnlbJIt%=w(TkJgWC8 -zwrULU4ti$D`IVa3Yw9g9ye^Zkt6|-M;G;Z|ajCNJ`+fduB`_+d79~`4{Aw)U$#<%o>g$lIkq9(KmFhio{J3{~#d&%C7HTi|v*`qH^1CBe -zOG3G%Idv@#dko;ezfG=s9f2-ygoiFp$I9&$MC>*UWH_&I@QiyDqrIydolBjMmor)r -zy0P)?^LO9|EV$~j^`8Hr>Q^1W_5Q&1o9(G?2w8ECcQi6&gz;(dW1iZ}d;WznMjUKa -zFk|Gnr#gDwQ~T5kvyU)7c42ezGamoJ_J~l<=S%`KKmv?A5@Gx_q`nXR`j~AzFK(=` -z8~tZ_c=f9u=*svyT{wo-)h_iJGRMkNFJ@-%7VXGl4Q{hwa|{i4NjZM+QBofffqLJ( -z3K`t1MmkHFKm6ONx!_WAax2@!tfP8UGhli--mSSW7$kp}X!fp6WRV6c#((hz~Y?2k>hx~EJ -z#m|qkaTO3;bxF7jOu!P^zSDx49h2I~)FC{^=w`2=_!Rg{y2keep3h?**c@7p9%x{h -z81nsCiLIpw`0fPRJa!BG2(va2aav}<_JP)4=(^oBoBxk26o4!`*-QU# -zS+xI8mVNk4Y?@eBcrV&OmxX>^2J3h%EO9@;p3fx#8RNXQjnNoyEATCF(i`6k!t8#1vNW}G3VdoolxfAwtI -zCM6u~48v8SFmXcPwF$mV=z$}`2O^+!gEtIBMW_*@avFp=iFxCjhO2f*E-|m$#8xHvGR2PI>b3iYAAgAm<#vK?_lfE_`u_wo?;WGKq}+j!ia=B9P%kz8taopG!m -zRYtB{&!~dpG6f0i*6)txnQ;oSkWJm{l}tCEC||j&gxO@#@@uU>_m*Y&P?&F0B-_4MY{*nVgfueN -z#QV)n-`7ha(o(=!igJ9?3daz(ov+ug{C*K?VUKA~6u2kh;7$_)BIX2z9^VQPd#vne -z$N>C6K&JCRg*;mkdQ>AQC^*;b0N~W=Y%hc%^S7Y{a@Z3A%`TPl=9Uy{A#q=pZ_^ -zCasCS=*wr%(sG7~O_Qw<*8zj8Oy8mVke)8&*0^KZ7Yz~;-0#!iNPor}eSD4X?F}6i -z@`PjIv<2ucpAM#x`Kie$cP=t3CqXm;DH;6pjk8xg7AFLnKa&tNW}b}NXC@x*t?QY- -zwW5(xC9a5wxIUB+suKaSPL@@bhkra*)(c`$64vkOcj0HVPg5uC=;)B;ws?vR+;c7A -za%Ywm{R5=qk}rT7b>q#DurrMjkg?u68&L~hcQGPAA%lpuAh6@nAztl<7Pc^8BMpt$ -zon9Qs4;y}30!CVQXDY+`BRluco>iG9qiJGvC-xzHbfNznp+#n;*O@L=yiGxTD@h!L -z(8v@Lo0^%ClG|G&f@&RFRL^t_BBlxhwT850>9CyJO_V+qXcNYjueB8(TKM%H{sZmE -z$cSbbJm(ov!|s?f>N&9&!}(*e5`e5I|MfoKGwbjqsMLorgBfK$%HtELwZT*X#w8_y -z=(5lu9VC?X`ribv0}n;sOYu3E_%T1>Nd@rGhbt_}ahC~#@9H~W+LwV7P>=%CIRl&y -zp_M;Xyp+qiq)o=e6`u|OAChbbz|eT1E$HRtiF#iV<&S)7K!=b*J?Gi;w}J>GAAp`V -z+0f-Hp1q1p)VWvizm|-ZoAOWBMJG7S-2|jWJi{6 -z>R(+Dz-(03&CgWP`wmY=+k3!pV4IhAojlVZ@~~+vU*q5;_MUEyzn2 -zFE67wU_t#6ZGO7E1WY9g+1QoQ@_P|NtmhC>nYN>AV%#X(1Ffo^zvl;W1FjA!niW3c -zP9xJ!yzLYKzS(>AMm13I@PoAQhF3yK{Jj) -z#p(U1Vf^vymOGH);O2hCi%UzJO+gP=6L>sxqOV-oo8`>jE^&R}bFK0+2wSyX8hQCb -zgu-2g)#RMq#^$!)L_H&!{%Zz}aA3gfGM)v}bIEn?sgQqll=MM}iqcDob6h3pBH5_B -ze(*AD=t|RHD8^m|#?Wwd>t*IdVNn>SgEgYa2B0{<%Vf*=*vB;JvKl=8(j)LVivOV(X9) -zWluh$5TzRhOae0Ja|mfNxDCL9cb)BkTJpI2Q2zxlW}wigFx!v(;w;~kHaBfOD% -z$jk*6A!k^uS(81WJRg+nvcvbP>Zt$$^!U)>xan{8XSc5sil{vH_O1-t_)cox0_^a0 -z|$nIO$;Qh?`}X1 -z2?5_P0N|y2#X?ZOH0vKy{(qX#(a8!%)(1DsLoV4WR07NTHG(xOoCplm%|4mw{qwPy -zGZjG4VcQ(%;J$r(IuUK6vJM3p>vdoHSoG?7q+w|dSlp=NMd$@%{JIHD$aYmvo#@;I -z^)wF1%cYL!l#`gSCSJmn%NsM197&0ZiDfl44=WyXX40_Udvl=-nb!3ftx`#)8XzJBhBZb80^G9IW@IIkL>!Ag7`8XrvFjFK%?YpU~q6F -zu<8nX)8BNS#c#kq*>}KTXXVWcf0%ZQkda8Yf67+y99E)Y1P}Ix=4Y;7zupd106cU3 -zx~?^;PXCJMxmQj&w{3o(DdwMI;?)7gXpLt_lm1mqFVtpw6N9QPnEny|%kSppHqjh! -z!nqRbMhj~oH=&c&B{}0^07*@ZO=4&66d=r$Ge8a2!^Wp~+FvQ%sHv?r7q)JA2;^;5 -zLg^P0s!ZRa1-$P2-P=z?&q1kp<^TZSoM8uSfn?CIAKn!Qwb5{fHQ{g`z{#N%(x0He -z^e7Ju^_7VA{rQj6+cY&ncyVAn#rnW;%}Udk(JDZ=;jFnp_oBRf;-*X3e-|j%WhE%_ -z;&FwGbDm2(B0vBH$z~p?`U#%QAj67hFDXE7$YY;Ai>v?et96FooOPSl3;fd2op~tZ -z4$ -z)@-ltA8qw?XmA84gbjRo>j}7x&QNjwbCE)t$N;OUtK7Ko8@keGJQUz`1Vl~guADsX -z+eJl1*f46Pws($=ue9O9=}iXWxBfoq2RzwYe&f=)(R9VmpA0wx{GX8y#+EYp>C^0n -zGe*Cxx;hWk6;^F2-}ar^3|QQ^fpG6Jp{DJH6@btRSfonw)m2qKa;j-r2Fo1;DYRL^ -zi%gsr0)+p<3l{**X;q5de*eO)lje|&+Z~))>wTD55rQt9-@&#cR7~ZZMo_-se$R6; -z{low%t)n?|FBZW|ItQfa*Rrzr=Qwd6I06(b`M<%6&juS(?5VjJ -z>FNB<<+20<}>qwXP -zeUq00Ths>g15Fkg9XaT$2RPM*zDffrubj1M%JnNpKqn>3ia#yDX<3UYKTHgK -zkbW%u0`IJRz}#bhuVqPG5Irl|vlpCd|IPvg9c`kpz?XOTN~wY456f*0fm8hWVzAEP -z3mGzFYE>hH9?I*)$bz%8!E!Jz~dyDSti*iyj!+-iZVb -z+BFCY3;$7ize)1f)*&i@iwof*IL8X-S5m@&r6Zu-d&1!0;F&A5(#tY=v!{>!@1Dga -z#JX=yBCgs-vY$H6c?846O)o7xwIlmzW@vpoEMawplF8YY=3flLDSDRWv=MTguM-s@ -zq7^4z2;T1M>hi$0Cw}_5NYx%=IDPmA(NeFq12GPEK^Hev8=h!M4Pt^y%8OADQtr6% -zIx5nip#gu*ClBz+S30)-zbsKt(|gCOQJiz{PpVrI5JlR-ug3Gl8IFwCm@dLqn`a=< -ze9O-~xHykuoa;|SVtIKvQ8+9OjW~HzN%stt&B+MjQ;dcLz5{}vo=ZfMLqN`vGpUvRj3cI_DJlOc22b%9OOHEsup`H<2AJH-_^kB{GKS -zjXZC5NNbTEnbWf~zVGhVum8)!AmUaaY!}U-3LNZN!20)CaCxdV4VJP!Lw_N(u -z^ttr>RBUS8d`ESKqB*Z`X3urSL*GU*ymIqk?_*>x=@<7p8eCyR<}8>R)3gVE80FG< -zr1E^Yr#p|JDIx^Y9}#N$Ip*X!0ZMD_&z-!poX4R!Bs@IaW9v_sxUw?RQmp?k%XG|BOh(LcC*y}7DwgU1)DZR>r|#mr^cXO|V7#Z3^ueCa7VpRAKb))?f^{~he{d-4$ko+c4 -zDQg11U=x<9a`SWfupnexFx_R&(v5n^S` -z%~4IIXr;rAD+HT8*oA|^6u?l-Ksat~Ap8zzO}=G1EobS9Q_(Ib$0qgB$I+s$-uaf% -zbxR`jRWDCsaih5h&MHSfc4fBv_Rp3oj#B5P+Vvip3n&K$7h74SJqQblCY(BGC1AN^ -z5f%~yitQ+g+xG|jDbD!)byo#nVL{seS^xIUX#wc7SQbJso`+xO_Jy)R#Oib#3?vB$ -zSP@ulnqpwpSsF5d2nH(dd%@_ex0Imw;)I%u)!PFwfm0w*b)@Na2A!f-!*M%-+F1O- -zE)$9W4LGh;JHTPZ(jebTlJtiiG78S>KMGGORhW6UJrz6;r&J*c{Ujk(Q_pLiHK4Ua)3}4>QaIqA2Z^D7 -zAw%=wm5TL1x`gW$Q4&Uys~1$d@!||HKK`?Y{O5}|cdpwonzbaCO0XJpKcbIhQjKY2xC?ERnywigO%`W>z)lXfa9dytA%-5o -z6^CnIRhuM${=t<{vHqJe;<14LzM*_$!KaMq=r1*MiqeJ#*sYeeUP9KId?-*z{_72B -zMnXd}o*;o3h)6A@Y-!S`n7fym;5pgb_v5aoJsJ{NSRkxc!7Az=f!gSBxSX9;nec{^?Nw`85)mAqPYT0KOCn*a6jCPrc83~vTG%4 -z8@7V%sMSBHPyD5=M{+z%AN^=9f(r5yc*S=Fyk46vM8wYYH>Z4t+ulNQC -zG4sVhnw%*LzGMyw=z>|>j?z_CFvIqjPs@Qr-VIPh37{V0xoR<${vF50OF)51J!AeA7trEsvnTrdIL#%r-Q -zSr3n*Xs5|aCXkZ3MFjn4{||3~z`W4?9n;0?6J-hwde3eSlmV@`?FfRknjoKNfOblJ -z+S=NH2BSfl%3vYbdObOIcujC#C&Y?VG~x6p{`Ygx@!}#Zzhe2Wb(qPb*`!aqy#99X -z%SfZN2~k8unlxQK2g;m3;609rTQNl^UWI4tGSZrbEc+!TvFFEy)A4UfP2k2#)9!Th -zqZbA$By>!s+gj>Om(D6i9)LRzz45r$bR2T@H<}|`xt}$~oiJ8b2o2v(5fRys?fEk9N?^Q&fS0}raM3{GnN=&e)IyFNS>M`l -z9)WAW8k|IN_*s->SZaoa*k5@SLrE}P3Kjv%t1JoSH=4v#{x7Dx5$fVg2r-(GkGvce -zPkEtgP^ePq=O4q`PKad=xET|uK`A7|Ynd+0(66#-Ix=p!19nJi-;r)_L$)wPG6J+v -z_cH0Ov!x)LkJ_#Z2N``g`()JOmWxcK6xDRzYA+QBdW60kpT{9a3tB@8tRJ*7(s)!^ -z(y1(8u>wyCo(`I2Cb2O58F=`q1jrukPJz-o9t$eH!AbNYjq(c9^#9QN`ttrk+9gpE -zoz9r(zoErD;^$`7sw(Y`Qe6~Z#Z;V$7hYFW5TK!e0*0h}5+en?NkVY#g$>7ixmmtf -zkmHP_p^$eE@DOvKX!KKmzU3u1X2M%0dNKlA?@br5FzzAReG%A(BYsNTvv%sm^(Vgt -z=+92Oc@tM8h2Sc&#+l2B9RDvTElRS|{fhOkEU(EPYQcZs=C)3a2Cd!=l_EZxYK*~E -zBfSv{kBXxRoghtu(th`{BFZ6MS6?Y3myyM4Dc7Uc<>*Vp;Vjw>8*OFbApTzmuU -z8x4$Ch+!w)8gFJnG?RP2*syLK5WIf_w&ukupN95yP=nJuBpHOkh%|DNDsu8CG@3#G -z=yX>nByMYsiS5!~Po_pF&qVl~;;X7kE2xWXe_V`fdeLcbRE33bb_c{ngEIl~joKLZ -z<{=Xv$yiSP@dX?6TIHFmr(N^=`K!`nx>W1kL&ZZY1he>WuiqT0sD+-}LofcSbHn9GjW_ -zEMh}a5I@r|;mYn!kTTxz+UnJnQy++HRF2o7}B{2#&5z7=<}|M@!t)N|$S -z&7ucj%Kdfd>2v=Y=KJo_M2{t+L59rub_6e_UCAtjk(eL_*^tt*f+B)v5OMPuCGjAAl|=@seh3o{FZ#y -zpORp((06SBILXMDS>^u5sGU>qM-MNInM%Avw3sB$*q@2yPLj`&rL5Xb84|8w8`aJq -zC2lmL^GS`7uU`JG(*S`0BazP^HU7Veng*fie~G`Ke1*^#E|*LPCQgUf1rvNTUf$sB -z2TnnEkrZHaXo`Zoy`P0t%&h{2QVI|Y!DrblX5zZ$8+FMbjhcoS&>9j2W{QAtQWDNq -zBzFGjF1_-qkW%QEcujKp-uJhc{x^4ZARiHZbLHPA1CY{Q-ideO`#ff$*5o$HhaFZ@ -zl4y%>FG_gGL^8WfP{^vymE;^vC{895kR>y0Wr!!m_M`2-K)Ks}D>~+#iTY40j(hZ% -zVvzhhDPlBHms%+ZzR|THR8`Wt6qB7s{^yB(fSmWS*?+d@`#}U`(YN*KO$AC!?u+dh -zId@U@`po|dz;hsjtN7RYyF8z%FjC9q`ev2N3#Al^TmrVF1BSuv9+R`u?bsrMN7XAm -zR-d;%B<}5dvVK>$5*LoSwb&&V_3>d!kv+JgoMxqE3Id)rq4A~-TkJ)cL%u5dp)>-{M(ISflg`__R+;a8ufTE!MM%ByDY2t6 -zSvZ&Ep-jgzs+S!;pQT}OhW_6QFt4n=p^Z1+zJXRFnlXdLINuWo|E9q&U4!bO22Wig -zVlH;40KX0tghE;_ibH1!zv&n5MlGhqCu2&rCkK%dSj)^V`wU}`N|mhkmtJg!?K`&` -zB#4Y*)u6{QMQp5pitu_C__Y3MM?&T&WmijLP1@WP(}1qnUvE8SvTW_5q7mDA_kXPC -z1Awb)i<9^wv;FWYg#LlRXCR+$1p>;Ub~MblHc&{#nMhN$Se(V+Kbhgg(1U1Ym~#z+ -z{cU)Ch43>=_W5E*@4OJqXIfR4U-pX@X&q&Meu+Q&ZIe{0dDXZo3AZKP8A2B$^b26vb~{;2A`y28a+L_%AKGee*H5kzs0rJ`2&8tfj)8(uVqn>?JaK -zCJtNQtM5!re7?~nL73xZ#%+~(r2^buZ`FlOCNxoXGx|(0eJeqlUA3Tr#8 -z+*_7CKwv`pn5F9BMsR|L1$ltYVNN6gfD3GNrF>Qu6CnHidQI{yqaEknt!Vha( -z612pkwc0HyiJ1lxzkO7JSeQ2HPN^Qh>8whjKz$8D^z~AHMck%oxDz`vBwf{Bim3m( -zY7*)L_B-|ZNqaFJ0BvZJO+n?r@s~UwI^T((w54 -z+}5X1^T^MOOJ)6*CcXk2H?T25u@I%o%o8JhD68_%)1k(nH-}bNK&w$c=v(}r_uc@`<{vTSLD~oa%4o7VIXPpv<2I -zrrTu+fIHHTa71dagt){`e$Xo~i*3!G2Ed@>0?cEiQe=MBRfnAr&A%zV<5>Z?f`cIX -zVv`{6cCV8E?r*Q2EBh;Wb^HN}{NvaAjxi4^m-ETp+;k#BZs&Cj|9Cm6as?)Y6jCDM -zK?YGhBB9xuxGmkGeCYS>Rjoa#AB~9;=E+k{U(oI -zPYW2tzIwjBJwWh)0m6Oq^XbyZ=1jdEHDopPv)5qZ*MVkv{S#Dz@6;DqW(V~_b^!S9 -zp)4I|N>bYEd-V37I4tBt|6c*%tnwQ|Nc|Ta -z9)zZMEY(7;^`jeD$_a6UA;7o1m%180Hg)gwAyeLYu*@VU*IbMIlX~)es)HoY;>%0b -z=ax!||4&_C9u9RMwQo-%6lJN1$}&%sB_hTW%5E$p+bCQ1%FbX^QY6$2NsO{(9sAfv -zNos6m-zg2*8S601`}Kt9z258h^B;3DpZR>xa-aL$=OAzsOsCt?3j`B0mNEDdQNSEWIkozD(uncdY>ZU4y70E;rn&@<^vtexfX`z{Yz!>< -zaKG@JhyfBFJmK)=Jr7A>DzLYzG2|iD*lPpKan+~8Zic^Yu(D^ji!yx -z*Ie*luy};{RS*2bQ8vHWd|X9rWgYLldi4hd%1Xus2$s{xvuM1fh{R2SUQL3A7J}yj -zejr1Z!V9zA^<#ZghU1=|ihEChZimE8eY7-EuITe^Ya^kpkr`2YeA#4*YU>N2G8}=Tx9jOF+u->bY6kMyw7oh*^+KTPFwER?k=fJ7Ze`fh7E^@hM&eyi9IrFBp>yQ2gEa1v|LoGz|wvGljH3z_x_ -zOC-Pz-ota!J>mTKRIj-2qQ05E%vx@lxhV^;`EPl)Wh8$f~8hgT=Ps2J%`$yDcB+Hm1XxWIAN$SHsrA -zd$!zL+=NEMMx}>1tqIng(=5BGJM+ia-n!n-z4$s&e*C+n&H;+JaL)dGSt_qE4M+88 -z=r+&q!b!v`UZLEg8J@p7WMZj{4VyLFnH}%SS^LeA;QSM2MwIo@b -zlHwhlEz}J+xAVygKi_iUv^CEBXlPUUo0h8&tit)2km0Xeax+?8j@u@URb7Hn+n4I& -zO{9sd`Lrc5C-;S0gIyUrgCrScWvFwxpD6#vLh}x$y^yuRluCo;?BH -zO_iO|ojdCmXOqFC`M{ib-PbLNJy%nd*tIOA$~VP-PdQ=CdSeCqekP|g&Y}`8w$c$i -zIK(Hwlv(GmQYw;Hj3j(i4(5(Gyf|IgAxh|Tm2djab?GLlOQKUUf5UoHA)KJ>`W9O+(WGNTS_;s#|fcTu`_Ctyg)Naz) -zeNaC3z7$*l=7GMJ0vbB0rLQX-bA{;()S0n}w8F})Y+bY`DjY<3v~?KY|Hj;i?`X}&sC0$H -zW-D%Gl(=?ZeBB-NW6^o-d@wH?B|Q`s*XCE3=N>|Ho;uk&@+(=UmM`^9cN-l)&y<GcE5l8!r6EJ5DyaCZn}|M^aF73u04cY -z_zsnlSDxsYHc2ht7z -zHdKD+{fgiTx>%%8VEQAa6OD}R6iO^y@0HVU>u8^TMcwZGXh>SZ>&!t`q83QiaO96R -z2J2D=X0 -zeK!X=)kFJW@blxRhvo#MEb})HVw3oVdxzEG?q*+)=63kn8Zzr3F){4gR+;(QNqE(p -zL6tFL+re)zk+iAA4it$HbD2Q%5MRCAi<^rs(>?m$k@ZF-xZ!Lu*AqmwrL-_RE2Ai4 -z?57c-P9lTt!hE=Y)!bHky9uGz|FdHM3D#ulQ(V#)y240f7_agD{~zG+5RJ$Cp6r%P -z-T()lSsD&t3!E*Lq4UT;tC1|sZ0j2K5SCSXl%3(H_po55fw5};;Ty3veF<8n(zZK! -z|GfQ`)!6y@eXvgn>%>SGX~a2mV7dU|rnnqJ%j?n&JJGkp^e%L6{h=WTMwZDesH%Pg -zmzY*i_a)wN@>POH=<}~ppf~)*s(&7B{_@GuOe8-kuR~pY6-_(8)yoQ4e+zd7WLybEh{$X=uaR;ZfJft7PLp -zE)GMuG-4~}p%vL!s#oPHY`5d#9`S=t3B$$5xRU1lh3Ufx;4 -zj0Y2cPPvqn3U>#4#UEdSyTf3j+c)ZsA+5|e0{Wyleg??K>T1ZX^cOjE9@$PP$?o2t -zb@+xQ%E_mvzhE3I28(Lre{LmQP9N0?rP)_%Dja3{3(qFXj{&L9YV+FUzf3^HNfF`m -z-G*NfThG-}^Ufqd^l=k4^N-5IESUk${zrOl78)lsmPJk<{hCBAqLGT~C)cFHtj+~H -zJgrL&vRAB>^0r=5-x&7`>C>3$<1pSCVEJMS}3!G%z9RWK5XN_!it)NdqXeY#9?HT~cUebM`JIb^N+0U?A -z{xHlkC1e#uSqkydX*@e>D+j(T?x!?Wu2)#JJ+pim-Vh>0khv&X(ouBJLV(+xG*%6- -z6w)$N-N}CN6MB8&B_wahoB9P^5+F%Z2q%pae^7E}joB?QC)%7w2XP-7cH-6gu6okM -zPU9py@5{?5zuPDyc#(f@s?NSYD1+pHo0Ge=yr=Q<7qw?LWh=(Y`dB-8svtWpr5mGX -zFQI=;9u7XIC?%`))-eDxL!+Y;YPxw{-RwipJB*&mrWu^XTObpA=;wGvoqz>uQ4Okf -z5j>xiM@?OVL$37oxjBljZGTR$T!=);+CVYC#n7t%2zH)n?q{m)WRv>*{?U27M2djh -zFuIav8Vepc6370}iW8%Js`Z-o&ZnL8J3NCdBfTaEf#D{d1Kpv&0ywn^=EMJVdgy%6f4Chqjn1qSI4q`O<2U8P;N<(K8n>cH -z%e>YQuJD{^&dKX`b(>Tw$S0;X_j>Kmp&ao6UZZ9uB?VboSmlW%eA}<;`PDX?&);6B -zzt|k=tQo1>mTT);7Z=&M6lR`bfCJcdqLyt;nb#%R<-K+YqLB~C3xGcg{AK9AA6qtc -z{?|W95y%{Q^kN@`WYhRUTO<2wikd6R@hGbs5gSj5r>z2H|-E$Edg@95o -z`LJ8jc4mn0q@m!4vR8>q+e6;qIUiFKsO0*OSJkpB(T&MRoWf=bL{deOp6%WrC}xfv -zpjNo8tbJH+TYGaGU7gAKFu_`Wxp1FcWFb>ZS0t1&LEaJbdIv+NAAK9Kp|=xxYw -zr_mwj97pm8%IZ8~P#m^nrgydeB1ZA*=0|Pb<^?CRWYQ#_=yxGcXp -z#=XKAsM2>8@keevJ)i>tckW7YX0cJOO=jEB{`0OYG|R*L34@9;yfzS2@@Po89oL;C -z8$1zLz6SJ&CWJCUn@@?+!)ZDsH(Y7N*8Oe$YZP?v@vw^`RrfADr@wEVUe`B)QXq#J -z2)70}B*6%%@E#;Z9h^On+qqIyC(uT9eLJg1K6Q!(yqA=^ceRYJ -zU>3B1qqR8-%co&NeR=6(hHw5`L@=h -znBx@Nos?7_pM=X0u?3OC~35~nd~fl;sy2r#Rigz=m5 -zUaf8#>iDzKo9)c}+#NRaEqkMmdd8C04Kcp@d_wFIf?6(JZuG!LM#3-OX6a!obSEZw`>U#&>2M4r1S*B*DUwmm!nSY=Ax+?aV{0Ib)WrYAM+SvVFO -z*783%#OYnp7~7sHNwD6i_D}$&Vl}uaNkUS{!a>Vx#-PKxCWe}bp5BF}mYg%ASuS}< -zo6Xx?+-YW|QLY`PPKDv|wuPCMZ`-xbqWWJ^+kOt2Qqi>aPVXZ|$G6Z8)cYhM>P)J* -z$kE`7cVugGF-J26eQzuTJV6viNa-l_NpFN>iGF>WwxYr(!95L6HMY&)u*+=gKHRuj -z8|T6GrzT?0@6?1@m*3Du(eLq93=D-DKw0pF?%Ptm6{;&Q4G&W=_l%UMSdcYMEZ{{w -z<5RiYK9XI!Nh#%U+cIHd&e`-n4AIZ6-rclqY_xCNZO}3Z0!Wq;3oB5U-~~D~a!}uSw_RTG?$ToukMmOJt(OV -z`Rgj6|Cd6?dXKc|ZUmfx60D7;6dY1G2=ZyEIU3_vUM~$qX@i(8qdhxL$496|d~scA -zP`5e`E~?{k5acsUCgzecL?n~hCY>$f{Chfee4r_d$+59T}{_Xa`0@SeWh*=^1D!VgH_nVxA$k-qae1u -zaEV{J^8rTX@;(icJqgFCaO4?&PuMz?5Gchn -z6F&qR%bhL3jJ6l_ -z;(Ra^@79nJ&B67uqrg9f&l$5EcV=ByEj8PGS$k+dhP-{iP

f(LUYB;~_IS9e}`! -zZ`r{+GH=AS8xpM&Y8zek-c-Ae*ZG@Nh}-yZ8a#j`V+2+8UHWGZP9d^E3Cb$T*o#8- -zQa8T7MzssM8Ox6Q-_||#GL_dJGbZYw8pX#i+o}1a8 -z+i+hTg2jbnsUbEd=G~jDcSo*P;pM9H!b>FVLMgA5`54WF4p6}YQ=oypGN;Bt5y282 -zCi%|%R^skb7f+0>B}hTaE0Obz#rYuFGv1(39|zId`SQV{4W)MQFCur<%P3h+x5V!V -z(Tl@oX@^_&)P=6adgFDxGuYPsa*>3XpRv>1g6<_$*qVWDKDfe_l6UiJK<%s9PW=}O -zb|A~={Y=g$MPrVYy>t`)`Y>b;$M14zh}ycGUThVu`e*%_qtFWQ{>Xk9|KcIRyQP@X -z27$hm)X6;*aqs^S999e#Kz7N74DmC`*;1OKE_>Rs<+Ibzl{BXDbQwFY+ts7r=MAjIJ)b>wbu>BQ;$GN(}4F$Yd3Ur~8C*S-|eb!83) -zNNHit5IP!*oV`&7b20@%N0v<^{G8SMJMBnsy@c+&>CF0Eh$2c6z~d`%<{am-cU+r% -z@q*#OpU|P*Ao@p2P2Ck}3M8#fP<}d}Q9yeuX$7lr=S!1Ds>FAGrn>N1I(a74r+qr!9>BKx410y=>v*6Ob`$(;BcEZqvke6f -zb*Lc61l$n_6`v4%cS_H-7sQzBp?9ljC`fXwt0wRue6)=Mx+gu*fanH||@0${CQf4!|-?LPt3vAZXT;Ccm -z{P3batA@Y)ks}oDUT9;PCRCUAlk#IRWB$S(+Ms6zI0KeV&&z+KFwiFC(^J#UPUSHY -z1Y|g`<6O6fsIeFiWTbH=%10_KZB829r!xRytLWlQ{FGnJ>)PxBO4e~reLsr2A2#i? -zOV@Z7Leq*2HpPZxgf`7rqPv@DN({V1c6KvE7d(MwQe+BIzC`9*m{u5eDa;vH)6m{x -z+txXf2jk`nz+IRKeNV+#>=b-i-e>9o%=_XmM~!IY*{#>DX4XIqS6FRS!Pa40v0H0) -zd`Fxo>$l|ut_!||d@5wp(@jl$XpQ}fj`lLhjrNvT6n8c@N&?S4%bA?S*HY)phGHLN -z40Q&ViuKY7Pd*y)B6O#7RewPNl~vwNC+!N%m1|-5dpATsS{4BP6QZN) -zO(xxZgC3v`f=g6i$DZ9-XhVo68ipQpeU`NVJ8Ss*m~XEvxXuLmxX91w!4*hBRTE7W -zpr%2o9L>GD1-{)8pH3IvFmDeYI^lFY{#emxv)}nTOj!`Z|ugxK*PP+Gwq|rf2%(3~pE<{~hwDR<0~T>`@_KTnaCOyHS~O&&K(7 -zrw#8)ecr2*zZ@%m@9Kc~{b7(EycsG;K*>VNo!CEECq8?VFJjAU4I)Z7F?~&RV5Zy2 -zS_%kj9z;KRSWs`;VBX;*a{moca>#r9{w$$*EIam(?&*&)SWLrE)x=7EsL#uueEMHv -z*))q{)Y--_2!9L2rrrjaIJrkJa&3w1#tugf_B~($Wq>FW3Ig$b6S=c#r|+p+RMULM -zX>|f@BH;q;H38%@n{M6I6XO!U^AxVMUx*hJU;l|E7H(Oz7h-q^;*Gl@&&J=?@xYZU -z#wE#lJ6f%{8?-Dkr;8~&hlEcFOB0$FffA`KJe%&#{=KR$?sv<604PocxTb|`tb5-B -zooG=+rVI0c@Pt?&Fdj!R3a>_5CnT#hdHLK#+ -zC|8b&t5Onlil$=5y%N=)=pcor+V3zDf^c}FZraeJBB8T#R_hr8%pNzAZzkT6teo=D -zL~Lrx^aLuO@f_Xye^Ngi5eOb5P(~WRu#>s&9E?=b`@r7>&0hc-4XLR`^(l?wOc8?2 -zm4*p%TT+Jjk`Xl}?`yr#v4WQS2}_Eo{?}b}Toi>Q&nF1`04YA|4!B09kIUZh&g1=- -z-~W}E{6h~aEL@wX+l?qy4{kS*$!MG0FATYLkiZNJ&|L1<9LZA&VbkDD=aso#GE?jA#S_zbg}WwXb@Rn2 -z+qP4;dEyJI^N%eMdjpM4S+0uUY_m^A5U+0VMc7bK5t%P)WupD~UiNyxRqPAN-(!;_ -z&Z^mXvU2&XZt^Akg#z@f^ZI}fbii-yO^M9kNH^CY0NZwq3fFmthhgM*>%_kNl+nDc -zcGKnWk11=lS59!ZAE&k@|GZ$g#MGlhGs)~=Db`SF!<@TCZ_@!OtQ#RlxLMOwsQ7zQ -zv5YT5<{-ejieg!vn=2^w#yWGuqER9Kj|Y{lC{P^nJ&WA$5BM?_JhpJEv=&TH`LZ}j -z-|BLr6}I!N_W@zvA~{BEqWX!Nc!`g!MOcQ-hmW%F8uhq4YR)MGmBFZ4(7GAS`efQ5 -z8+z5z>LP~Q@l4L?A4I<%KMO?FpD9Pxftx>Y54`fp6_`Svl2_b-hC;iYjZ69Ra;W -zscst=36J0t-TJsn*BdOFmU)0|96s&>-Af;{Kr=623iU{7NFuQY&Nxu -zCuNz^l4N62i9*_Rs05MuQix~1iM^tpFeKkZxC^=1%q&j`|ASf4#Z=caGuPfTPlg$i -z%&5v)qbw2gF67KRLZNbHWqM^F?`AzUALp95=FHpNCGq^g8DxO?d|~ps)IoeMNVl5( -zhI3)H%WYll&7Aax1OsAl~t46W+Xr{EH}72 -zo-w{7bCQ1$o|{?)KIp#mdkOFR@pdxh;!;|F1C%kcCMOWIeg@~F^Yoka=X`o^SSsj_ -zlUEwIaN4YQtt;PueN3@*_tnN`)5tm}zUvbQ7bVHifYfyx9vNx#1jftmTEk${fYg4s -z*@oR*Dommg6grSCotj|iSoU)Pi=gTPsNWOV$tBN|^Y>RI2NaXad|x&tzF5-Bu~r)W -z8Nse)OEF;7Op@RYAZ2~llQSjzXkQmlbH+yohWPoG#Ba@YLQxtpb -zq{=&2BIHA1Gbs7)gMw>bDD26kfVCIe_QSk#UKc7yMf5G0s}CNkR%4R+&7B<#``1gm5^uJz -zv_nE3sPL^?J;Jxuz)wQZso2*N!pSU=cg4|U9YyIO+fm}&`$hjAAqirzlJ~92S?^ek -z)2St*#guJF7s<>A2fw>H*g&kR6L~=YIbOFN%A}Nm_|+-Zj>N5HfC%~C?H< -zmfvUgv)&sm8i%YLxJI^U`W&^fndABxlwD+bU&Xhq!31|jm5;MT_$R833op-1#u7Qh -zLQ3jZzj@eN8p|GeFXqbihdgU(?ac&P!A37_@Svr5R(#opVGeHsgRrA#J|w(Zl(8i> -z$fN;lR%Gc*FG$czBrqpA%(kZ7tqIb!0k>-S?VTJ(0B^?9xn4KM)CAjjFYp}P-h5<#)MAkOlGL16KfbnA!tTC2W|kApvsgr~OnLm&Cn(AEa8iJTH0uYY -ztTk+O-T&U^*unBWpKi#5DIP|uOGJ>pfHk+O2la40_ke$DTujIq=X{~0?mqmv^a3GW -zQiy~_S1SlIof}?Qb#W)mjX$qoYS>-8_!xg?j>W?*p|Q#>=ef2|i8_d8Uf+3(aZt#1 -zbTWFWQntE#Yn2yaJX9-Eu`@avLCj$|L#Wt-30j28&{fru@hK!!AIUU1uAD?lCQf24 -zT05x|+E=kQCt1qRsy*m&r!Ue>#Rm)CQpWG8Ie`B7`R}%i?kiPa9oPpzH}QK1^*=k& -z>H#cFya0&d58rTe=U^a)lV^fS)8TS3>O@oIB{p^!)<3fg!I{0-bv-AcFxzyyDHM1; -zJgAGq`_L(ALT#TYGgzkgv7yA$ubM8mrsxpcbH?an1xG0Do$ZO3F-s75l_E&k&Z-9V -z>3G(OLAtcV6WcvZPZ0EXukCRNp`w@SwslbVe{9-*Y>-#e+V@`e!&?ww=B$2DSa=e_ -zPIraPQqpklFnm(^2qBU_+Zs(K-s18ZWnriqqGgL4^z=r5I;|(0HNQQ&n -zFis|zL(S_$dMj))PXyL<#3yA%vJn_R+bkf|&|RUs%YtxWxRLjqgZ>JgrI&Zkf`fm= -zK{L({0mNeW5xzh8xc!7S0c_lS37m<7R|9$S#H%-I(|>C*>TksbOFOgGfuUYt_Q-4a -zIgK<7d*BnqFP804gU)w;LBJ>c`u_T%Pbug_^PgrqKgp{W&?Tptq|(-*3ay;=6((W7 -zTjvO$29+q|cpR{5GZt9eZlx5B{B8NvfY-i?HtjT8F0%nYObhP97gEZVxlA>JMYIlP3V -zU6Gp1m{4=8L{POxlRBY{jnHu`>3+N17}~8Bq4S<}#)EyVmp&-}BAn`=J4gldyGs;y -zqF>?nLRmL=@yj9zQQ3ngo}uQ>$43ZXcFeEa`KBL4quK<~#%zL?tDn;kYM^SiTlv1% -z>!u^0#~({0J^2}E0qfT2Tc>lzXBkt+8|qBP8fr`$;GUpQvek6yJRu-jD6YwZda-a> -zZwS+8jr(*Etu}UdO;89oJ@<@Ag}azrEz*;<7$?svZg5+WeqM;zf9Tb;HGw*r{Q9zS -z*QMo&ZyrHpcx6Ems#L)JXhC6AxQbOw?Zf73$XWyK6MSnLi*I(4*hyJg3kOv~ifl$C -zr#pMZwjqCp4S=!A+Y+0Ajkv-2=&#f74LT^o0eEv1ef8phRiBAH68XSeGlF8VwNolD -zponR$gq$|1@(Yvn)~c5-Tn3F%O}b{J#uXSo7-Ut*{5R`$Zvtquo>IIcuuowcHL`7s -zyE-+P6BmTUfIV=%ve=(?m25SM@i1hMIfVAbf9&sdDu8 -zXM?7VBo=L+ip>XAY#<4^j5>0t -z8t0yY?W%jWY=3+RY>_aU3rIHU(#3UYlZLCV7=gpgMnCU|-DM)6Yzz#Vcio$eD#kEv -zvr&PyT7pZUt=h<(LNs|lxSTozm~d3fo5a0$h%!v~=DK(2OSAH5DQ0=;#FFoUs9Gll -zFtv>O@IR*yc%pq4-+ugX+hRzp+5RWon={xIoHr)yE^)cBbVh0aV*h(_ARI894YY*( -z{*CP+bl}gvkOR)86TJET>VBfzhYN^CD)3;6Y5KW)59YsL626zY+ZpPrjq2FHivM2R -zK<|2ri*D@xN8eBFbP=MvtVP0@itm0p+vA#mlNQsMsNvLa%-& -Date: Mon, 28 Jan 2019 14:39:26 -0600 -Subject: [PATCH 65/69] Doc: all: update revision history for 2.0.1 release - ---- - .../en-US/Pacemaker_Administration.ent | 2 +- - .../en-US/Revision_History.xml | 26 +++++++++++----------- - .../en-US/Pacemaker_Explained.ent | 2 +- - doc/Pacemaker_Explained/en-US/Revision_History.xml | 2 +- - doc/Pacemaker_Remote/en-US/Book_Info.xml | 2 +- - doc/Pacemaker_Remote/en-US/Revision_History.xml | 11 +++++++++ - 6 files changed, 28 insertions(+), 17 deletions(-) - -diff --git a/doc/Pacemaker_Administration/en-US/Pacemaker_Administration.ent b/doc/Pacemaker_Administration/en-US/Pacemaker_Administration.ent -index d9e752b..f67f950 100644 ---- a/doc/Pacemaker_Administration/en-US/Pacemaker_Administration.ent -+++ b/doc/Pacemaker_Administration/en-US/Pacemaker_Administration.ent -@@ -1,4 +1,4 @@ - - -- -+ - -diff --git a/doc/Pacemaker_Administration/en-US/Revision_History.xml b/doc/Pacemaker_Administration/en-US/Revision_History.xml -index eaaacd6..dae578f 100644 ---- a/doc/Pacemaker_Administration/en-US/Revision_History.xml -+++ b/doc/Pacemaker_Administration/en-US/Revision_History.xml -@@ -10,33 +10,33 @@ - - - -- 1-1 -- Tue Dec 4 2018 -+ 1-0 -+ Tue Jan 23 2018 - - KenGaillot - kgaillot@redhat.com - -- -- JanPokorný -- jpokorny@redhat.com -- - -- Add "Troubleshooting" chapter, minor -- clarifications and reformatting -+ Move administration-oriented information from -+ Pacemaker Explained into its own -+ book - - - - -- 1-0 -- Tue Jan 23 2018 -+ 1-1 -+ Mon Jan 28 2019 - - KenGaillot - kgaillot@redhat.com - -+ -+ JanPokorný -+ jpokorny@redhat.com -+ - -- Move administration-oriented information from -- Pacemaker Explained into its own -- book -+ Add "Troubleshooting" chapter, minor -+ clarifications and reformatting - - - -diff --git a/doc/Pacemaker_Explained/en-US/Pacemaker_Explained.ent b/doc/Pacemaker_Explained/en-US/Pacemaker_Explained.ent -index a767f5f..a79fcf3 100644 ---- a/doc/Pacemaker_Explained/en-US/Pacemaker_Explained.ent -+++ b/doc/Pacemaker_Explained/en-US/Pacemaker_Explained.ent -@@ -1,4 +1,4 @@ - - -- -+ - -diff --git a/doc/Pacemaker_Explained/en-US/Revision_History.xml b/doc/Pacemaker_Explained/en-US/Revision_History.xml -index 3da97ac..71c5a5b 100644 ---- a/doc/Pacemaker_Explained/en-US/Revision_History.xml -+++ b/doc/Pacemaker_Explained/en-US/Revision_History.xml -@@ -141,7 +141,7 @@ - - - 12-0 -- Fri Dec 7 2018 -+ Mon Jan 28 2019 - KenGaillotkgaillot@redhat.com - ReidWahlnwahl@redhat.com - JanPokornýjpokorny@redhat.com -diff --git a/doc/Pacemaker_Remote/en-US/Book_Info.xml b/doc/Pacemaker_Remote/en-US/Book_Info.xml -index bf11f23..8132402 100644 ---- a/doc/Pacemaker_Remote/en-US/Book_Info.xml -+++ b/doc/Pacemaker_Remote/en-US/Book_Info.xml -@@ -13,7 +13,7 @@ - simple textual changes (corrections, translations, etc.). - --> - 7 -- 1 -+ 2 - - - The document exists as both a reference and deployment guide for the Pacemaker Remote service. -diff --git a/doc/Pacemaker_Remote/en-US/Revision_History.xml b/doc/Pacemaker_Remote/en-US/Revision_History.xml -index b636049..9950f4c 100644 ---- a/doc/Pacemaker_Remote/en-US/Revision_History.xml -+++ b/doc/Pacemaker_Remote/en-US/Revision_History.xml -@@ -56,6 +56,17 @@ - KenGaillotkgaillot@redhat.com - Update banner for Pacemaker 2.0 and content for CentOS 7.4 with Pacemaker 1.1.16 - -+ -+ -+ 7-2 -+ Mon Jan 29 2019 -+ -+ JanPokorný -+ jpokorny@redhat.com -+ -+ Minor reformatting -+ -+ - - - --- -1.8.3.1 - - -From 9958bc9fde3dc4e9447a4e72c4ccbd9699ff4b58 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 28 Jan 2019 12:04:41 -0600 -Subject: [PATCH 66/69] Fix: controller: clear election dampening when DC is - lost - -same issue as 5c80a964 but for controller ---- - daemons/controld/controld_election.c | 9 ++++++++- - 1 file changed, 8 insertions(+), 1 deletion(-) - -diff --git a/daemons/controld/controld_election.c b/daemons/controld/controld_election.c -index 9fbf1e1..5d6858c 100644 ---- a/daemons/controld/controld_election.c -+++ b/daemons/controld/controld_election.c -@@ -1,5 +1,5 @@ - /* -- * Copyright 2004-2018 Andrew Beekhof -+ * Copyright 2004-2019 Andrew Beekhof - * - * This source code is licensed under the GNU General Public License version 2 - * or later (GPLv2+) WITHOUT ANY WARRANTY. -@@ -41,6 +41,13 @@ void - controld_remove_voter(const char *uname) - { - election_remove(fsa_election, uname); -+ -+ if (safe_str_eq(uname, fsa_our_dc)) { -+ /* Clear any election dampening in effect. Otherwise, if the lost DC had -+ * just won, an immediate new election could fizzle out with no new DC. -+ */ -+ election_clear_dampening(fsa_election); -+ } - } - - void --- -1.8.3.1 - - -From 0904789e74c59a33479b550befc82118a600fe36 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 29 Jan 2019 11:56:00 -0600 -Subject: [PATCH 67/69] Fix: controller: really avoid closing attrd IPC for - temporary failures - -Since 35d69f2f (1.0.3), the controller attempts to connect to the -attribute manager and update an attribute multiple times before giving up. - -b7c0e7f0 (1.1.9) attempted to avoid closing the IPC connection if the error -was "try again". However, rather than testing the errno from crm_ipc_connect(), -it tested the return value from attrd_update_delegate(), which would never -return that. - -Refactor so we can check the right function's result. Also tweak log messages. ---- - daemons/controld/controld_attrd.c | 41 ++++++++++++++++++++++++++------------- - 1 file changed, 27 insertions(+), 14 deletions(-) - -diff --git a/daemons/controld/controld_attrd.c b/daemons/controld/controld_attrd.c -index 0ce8be8..0653804 100644 ---- a/daemons/controld/controld_attrd.c -+++ b/daemons/controld/controld_attrd.c -@@ -81,31 +81,44 @@ update_attrd_helper(const char *host, const char *name, const char *value, - } - - for (int attempt = 1; attempt <= 4; ++attempt) { -+ rc = pcmk_ok; -+ -+ // If we're not already connected, try to connect - if (crm_ipc_connected(attrd_ipc) == FALSE) { -- crm_ipc_close(attrd_ipc); -- crm_info("Connecting to attribute manager (attempt %d of 4)", -- attempt); -+ if (attempt == 1) { -+ // Start with a clean slate -+ crm_ipc_close(attrd_ipc); -+ } - if (crm_ipc_connect(attrd_ipc) == FALSE) { -- crm_perror(LOG_INFO, "Connection to attribute manager failed"); -+ rc = errno; - } -+ crm_debug("Attribute manager connection attempt %d of 4: %s (%d)", -+ attempt, pcmk_strerror(rc), rc); - } - -- if (command) { -- rc = attrd_update_delegate(attrd_ipc, command, host, name, value, -+ if (rc == pcmk_ok) { -+ rc = command? -+ attrd_update_delegate(attrd_ipc, command, host, name, value, - XML_CIB_TAG_STATUS, NULL, NULL, -- user_name, attrd_opts); -- } else { -- /* (ab)using name/value as resource/operation */ -- rc = attrd_clear_delegate(attrd_ipc, host, name, value, -- interval_spec, user_name, attrd_opts); -+ user_name, attrd_opts) -+ -+ /* No command means clear fail count (name/value is really -+ * resource/operation) -+ */ -+ : attrd_clear_delegate(attrd_ipc, host, name, value, -+ interval_spec, user_name, attrd_opts); -+ crm_debug("Attribute manager request attempt %d of 4: %s (%d)", -+ attempt, pcmk_strerror(rc), rc); - } - - if (rc == pcmk_ok) { -+ // Success, we're done - break; - -- } else if (rc != -EAGAIN && rc != -EALREADY) { -- crm_info("Disconnecting from attribute manager: %s (%d)", -- pcmk_strerror(rc), rc); -+ } else if ((rc != EAGAIN) && (rc != EALREADY)) { -+ /* EAGAIN or EALREADY indicates a temporary block, so just try -+ * again. Otherwise, close the connection for a clean slate. -+ */ - crm_ipc_close(attrd_ipc); - } - --- -1.8.3.1 - - -From 77f6bf4a5762ae0b2192335a74894da7306c9674 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 29 Jan 2019 14:59:29 -0600 -Subject: [PATCH 68/69] Test: CTS: really don't require nodes to be specified - if listing tests - -9c993a7c was incomplete ---- - cts/CIB.py | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/cts/CIB.py b/cts/CIB.py -index 2e606b2..3b3c2ac 100644 ---- a/cts/CIB.py -+++ b/cts/CIB.py -@@ -441,7 +441,8 @@ class ConfigFactory(object): - self.register("pacemaker20", CIB20, CM, self) - self.register("pacemaker30", CIB30, CM, self) - # self.register("hae", HASI, CM, self) -- self.target = self.CM.Env["nodes"][0] -+ if self.CM.Env["ListTests"] == 0: -+ self.target = self.CM.Env["nodes"][0] - self.tmpfile = None - - def log(self, args): --- -1.8.3.1 - - -From 8de16b8a745e5b9d54c8441730d70c479ad84550 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 28 Jan 2019 14:53:58 -0600 -Subject: [PATCH 69/69] Doc: ChangeLog: update for 2.0.1-rc4 release - ---- - ChangeLog | 13 +++++++++++++ - 1 file changed, 13 insertions(+) - -diff --git a/ChangeLog b/ChangeLog -index ea65fbe..c51c439 100644 ---- a/ChangeLog -+++ b/ChangeLog -@@ -1,3 +1,16 @@ -+* Tue Jan 29 2019 Ken Gaillot Pacemaker-2.0.1-rc4 -+- Changesets: 42 -+ 15 files changed, 216 insertions(+), 137 deletions(-) -+ -+- Changes since Pacemaker-2.0.1-rc3 -+ + attrd: clear election dampening when the writer leaves -+ + controller: clear election dampening when DC is lost -+ + scheduler: don't order non-DC shutdowns before DC fencing -+ + libcrmservice: cancel DBus call when cancelling systemd/upstart actions -+ + tools: remove duplicate fence history state in crm_mon XML output -+ + build: offer configure option to disable tests broken with glib 2.59.0 -+ + build: minor logging fixes to allow compatibility with GCC 9 -Werror -+ - * Thu Jan 10 2019 Ken Gaillot Pacemaker-2.0.1-rc3 - - Changesets: 27 - - Diff: 20 files changed, 375 insertions(+), 195 deletions(-) --- -1.8.3.1 - diff --git a/SOURCES/001-xmldiffs.patch b/SOURCES/001-xmldiffs.patch new file mode 100644 index 0000000..3afcd94 --- /dev/null +++ b/SOURCES/001-xmldiffs.patch @@ -0,0 +1,284 @@ +From 66e5e4d83e90be3cecab7bf5f50d0e10fbaa7cea Mon Sep 17 00:00:00 2001 +From: "Gao,Yan" +Date: Fri, 26 Apr 2019 11:52:59 +0200 +Subject: [PATCH 1/3] Fix: libcrmcommon: correctly apply XML diffs with + multiple move/create changes + +Given a resource group: +``` + + + + + + + +``` + +, if we'd like to change it to: +``` + + + + + + + +``` + +, the generated XML diff would be like: +``` + + + + + + +``` + +Previously after applying the XML diff, the resulting XML would be a mess: +``` + + + + + + + +``` +It's because the positions of the already moved XML objects could be +affected by the later moved objects. + +This commit fixes it by temporarily putting "move" objects after the +last sibling and also delaying the adding of any "create" objects, then +placing them to the target positions in the right order. +--- + lib/common/xml.c | 126 ++++++++++++++++++++++++++++++++++++++++++------------- + 1 file changed, 97 insertions(+), 29 deletions(-) + +diff --git a/lib/common/xml.c b/lib/common/xml.c +index 66b5f66..d815a48 100644 +--- a/lib/common/xml.c ++++ b/lib/common/xml.c +@@ -1466,11 +1466,40 @@ __xml_find_path(xmlNode *top, const char *key, int target_position) + return target; + } + ++typedef struct xml_change_obj_s { ++ xmlNode *change; ++ xmlNode *match; ++} xml_change_obj_t; ++ ++static gint ++sort_change_obj_by_position(gconstpointer a, gconstpointer b) ++{ ++ const xml_change_obj_t *change_obj_a = a; ++ const xml_change_obj_t *change_obj_b = b; ++ int position_a = -1; ++ int position_b = -1; ++ ++ crm_element_value_int(change_obj_a->change, XML_DIFF_POSITION, &position_a); ++ crm_element_value_int(change_obj_b->change, XML_DIFF_POSITION, &position_b); ++ ++ if (position_a < position_b) { ++ return -1; ++ ++ } else if (position_a > position_b) { ++ return 1; ++ } ++ ++ return 0; ++} ++ + static int + xml_apply_patchset_v2(xmlNode *xml, xmlNode *patchset) + { + int rc = pcmk_ok; + xmlNode *change = NULL; ++ GListPtr change_objs = NULL; ++ GListPtr gIter = NULL; ++ + for (change = __xml_first_child(patchset); change != NULL; change = __xml_next(change)) { + xmlNode *match = NULL; + const char *op = crm_element_value(change, XML_DIFF_OP); +@@ -1482,6 +1511,7 @@ xml_apply_patchset_v2(xmlNode *xml, xmlNode *patchset) + continue; + } + ++ // "delete" changes for XML comments are generated with "position" + if(strcmp(op, "delete") == 0) { + crm_element_value_int(change, XML_DIFF_POSITION, &position); + } +@@ -1497,7 +1527,71 @@ xml_apply_patchset_v2(xmlNode *xml, xmlNode *patchset) + rc = -pcmk_err_diff_failed; + continue; + +- } else if(strcmp(op, "create") == 0) { ++ } else if (strcmp(op, "create") == 0 || strcmp(op, "move") == 0) { ++ // Delay the adding of a "create" object ++ xml_change_obj_t *change_obj = calloc(1, sizeof(xml_change_obj_t)); ++ ++ CRM_ASSERT(change_obj != NULL); ++ ++ change_obj->change = change; ++ change_obj->match = match; ++ ++ change_objs = g_list_append(change_objs, change_obj); ++ ++ if (strcmp(op, "move") == 0) { ++ // Temporarily put the "move" object after the last sibling ++ if (match->parent != NULL && match->parent->last != NULL) { ++ xmlAddNextSibling(match->parent->last, match); ++ } ++ } ++ ++ } else if(strcmp(op, "delete") == 0) { ++ free_xml(match); ++ ++ } else if(strcmp(op, "modify") == 0) { ++ xmlAttr *pIter = pcmk__first_xml_attr(match); ++ xmlNode *attrs = __xml_first_child(first_named_child(change, XML_DIFF_RESULT)); ++ ++ if(attrs == NULL) { ++ rc = -ENOMSG; ++ continue; ++ } ++ while(pIter != NULL) { ++ const char *name = (const char *)pIter->name; ++ ++ pIter = pIter->next; ++ xml_remove_prop(match, name); ++ } ++ ++ for (pIter = pcmk__first_xml_attr(attrs); pIter != NULL; pIter = pIter->next) { ++ const char *name = (const char *)pIter->name; ++ const char *value = crm_element_value(attrs, name); ++ ++ crm_xml_add(match, name, value); ++ } ++ ++ } else { ++ crm_err("Unknown operation: %s", op); ++ } ++ } ++ ++ // Changes should be generated in the right order. Double checking. ++ change_objs = g_list_sort(change_objs, sort_change_obj_by_position); ++ ++ for (gIter = change_objs; gIter; gIter = gIter->next) { ++ xml_change_obj_t *change_obj = gIter->data; ++ xmlNode *match = change_obj->match; ++ const char *op = NULL; ++ const char *xpath = NULL; ++ ++ change = change_obj->change; ++ ++ op = crm_element_value(change, XML_DIFF_OP); ++ xpath = crm_element_value(change, XML_DIFF_PATH); ++ ++ crm_trace("Continue performing %s on %s with %p", op, xpath, match); ++ ++ if(strcmp(op, "create") == 0) { + int position = 0; + xmlNode *child = NULL; + xmlNode *match_child = NULL; +@@ -1565,36 +1659,10 @@ xml_apply_patchset_v2(xmlNode *xml, xmlNode *patchset) + match->name, ID(match), __xml_offset(match), position, match->prev); + rc = -pcmk_err_diff_failed; + } +- +- } else if(strcmp(op, "delete") == 0) { +- free_xml(match); +- +- } else if(strcmp(op, "modify") == 0) { +- xmlAttr *pIter = pcmk__first_xml_attr(match); +- xmlNode *attrs = __xml_first_child(first_named_child(change, XML_DIFF_RESULT)); +- +- if(attrs == NULL) { +- rc = -ENOMSG; +- continue; +- } +- while(pIter != NULL) { +- const char *name = (const char *)pIter->name; +- +- pIter = pIter->next; +- xml_remove_prop(match, name); +- } +- +- for (pIter = pcmk__first_xml_attr(attrs); pIter != NULL; pIter = pIter->next) { +- const char *name = (const char *)pIter->name; +- const char *value = crm_element_value(attrs, name); +- +- crm_xml_add(match, name, value); +- } +- +- } else { +- crm_err("Unknown operation: %s", op); + } + } ++ ++ g_list_free_full(change_objs, free); + return rc; + } + +-- +1.8.3.1 + + +From f8d008d8d3a29900ee0c6decbb71a243fa4c2d8c Mon Sep 17 00:00:00 2001 +From: "Gao,Yan" +Date: Tue, 30 Apr 2019 00:15:03 +0200 +Subject: [PATCH 2/3] Fix: libcrmcommon: avoid possible use-of-NULL when + applying XML diffs + +--- + lib/common/xml.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/lib/common/xml.c b/lib/common/xml.c +index d815a48..fe87de6 100644 +--- a/lib/common/xml.c ++++ b/lib/common/xml.c +@@ -1506,11 +1506,12 @@ xml_apply_patchset_v2(xmlNode *xml, xmlNode *patchset) + const char *xpath = crm_element_value(change, XML_DIFF_PATH); + int position = -1; + +- crm_trace("Processing %s %s", change->name, op); + if(op == NULL) { + continue; + } + ++ crm_trace("Processing %s %s", change->name, op); ++ + // "delete" changes for XML comments are generated with "position" + if(strcmp(op, "delete") == 0) { + crm_element_value_int(change, XML_DIFF_POSITION, &position); +-- +1.8.3.1 + + +From e6b2bf0cf7e7ed839583d529b190a7a6cd1bd594 Mon Sep 17 00:00:00 2001 +From: "Gao,Yan" +Date: Tue, 30 Apr 2019 00:19:46 +0200 +Subject: [PATCH 3/3] Fix: libcrmcommon: return error when applying XML diffs + containing unknown operations + +--- + lib/common/xml.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/lib/common/xml.c b/lib/common/xml.c +index fe87de6..940c4b9 100644 +--- a/lib/common/xml.c ++++ b/lib/common/xml.c +@@ -1573,6 +1573,7 @@ xml_apply_patchset_v2(xmlNode *xml, xmlNode *patchset) + + } else { + crm_err("Unknown operation: %s", op); ++ rc = -pcmk_err_diff_failed; + } + } + +-- +1.8.3.1 + diff --git a/SOURCES/002-failed-monitors.patch b/SOURCES/002-failed-monitors.patch new file mode 100644 index 0000000..1adf1a4 --- /dev/null +++ b/SOURCES/002-failed-monitors.patch @@ -0,0 +1,273 @@ +From 5470f1d9c776dbf753e015fa96153b6a63c17b83 Mon Sep 17 00:00:00 2001 +From: "Gao,Yan" +Date: Thu, 9 May 2019 13:24:35 +0200 +Subject: [PATCH] Fix: controller: confirm cancel of failed monitors + +Usually after a monitor has been cancelled from executor, contoller +erases the corresponding lrm_rsc_op from the cib, and DC will confirm +the cancel action by process_op_deletion() according to the cib diff. + +But if a monitor has failed, the lrm_rsc_op will be recorded as +"last_failure". When cancelling it, the lrm_rsc_op won't get erased from +the cib given the logic on purpose in erase_lrm_history_by_op(). So that +the cancel action won't have a chance to get confirmed by DC with +process_op_deletion(). + +Previously cluster transition would get stuck waiting for the remaining +action timer to time out. + +This commit fixes the issue by directly acknowledging the cancel action +in this case and enabling DC to be able to confirm it. + +This also moves get_node_id() function into controld_utils.c for common +use. + +Producer: +``` + # Insert a 10s sleep in the monitor action of RA + # /usr/lib/ocf/resource.d/pacemaker/Stateful: + + stateful_monitor() { + + sleep 10 + stateful_check_state "master" + + # Add a promotable clone resource: + + crm configure primitive stateful ocf:pacemaker:Stateful \ + op monitor interval=5 role=Master \ + op monitor interval=10 role=Slave + crm configure clone p-clone stateful \ + meta promotable=true + + # Wait for the resource instance to be started, promoted to be master, + # and monitor for master role to complete. + + # Set is-managed=false for the promotable clone: + crm_resource --meta -p is-managed -v false -r p-clone + + # Change the status of the master instance to be slave and immediately + # enforce refresh of it: + echo slave > /var/run/Stateful-stateful.state; crm_resource --refresh -r stateful --force + + # Wait for probe to complete, and then monitor for slave role to be + # issued: + sleep 15 + + # While the monitor for slave role is still in progress, change the + # status to be master again: + echo master > /var/run/Stateful-stateful.state + + # The monitor for slave role returns error. Cluster issues monitor for + # master role instead and tries to cancel the failed one for slave role. + # But cluster transition gets stuck. Depending on the monitor timeout + # configured for the slave role plus cluster-delay, only after that + # controller eventually says: + + pacemaker-controld[21205] error: Node opensuse150 did not send cancel result (via controller) within 20000ms (action timeout plus cluster-delay) + pacemaker-controld[21205] error: [Action 1]: In-flight rsc op stateful_monitor_10000 on opensuse150 (priority: 0, waiting: none) + pacemaker-controld[21205] notice: Transition 6 aborted: Action lost + +``` +--- + daemons/controld/controld_execd.c | 38 ++++++++++++++++++++++++++++++++ + daemons/controld/controld_te_callbacks.c | 21 ++---------------- + daemons/controld/controld_te_events.c | 32 +++++++++++++++++++++++++++ + daemons/controld/controld_transition.h | 1 + + daemons/controld/controld_utils.c | 13 +++++++++++ + daemons/controld/controld_utils.h | 2 ++ + 6 files changed, 88 insertions(+), 19 deletions(-) + +diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c +index 976fed1..8282fed 100644 +--- a/daemons/controld/controld_execd.c ++++ b/daemons/controld/controld_execd.c +@@ -2476,6 +2476,30 @@ unescape_newlines(const char *string) + return ret; + } + ++static bool ++did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id, ++ const char * op_type, guint interval_ms) ++{ ++ rsc_history_t *entry = NULL; ++ ++ CRM_CHECK(lrm_state != NULL, return FALSE); ++ CRM_CHECK(rsc_id != NULL, return FALSE); ++ CRM_CHECK(op_type != NULL, return FALSE); ++ ++ entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id); ++ if (entry == NULL || entry->failed == NULL) { ++ return FALSE; ++ } ++ ++ if (crm_str_eq(entry->failed->rsc_id, rsc_id, TRUE) ++ && safe_str_eq(entry->failed->op_type, op_type) ++ && entry->failed->interval_ms == interval_ms) { ++ return TRUE; ++ } ++ ++ return FALSE; ++} ++ + void + process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, + struct recurring_op_s *pending, xmlNode *action_xml) +@@ -2605,6 +2629,20 @@ process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, + erase_lrm_history_by_op(lrm_state, op); + } + ++ /* If the recurring operation had failed, the lrm_rsc_op is recorded as ++ * "last_failure" which won't get erased from the cib given the logic on ++ * purpose in erase_lrm_history_by_op(). So that the cancel action won't ++ * have a chance to get confirmed by DC with process_op_deletion(). ++ * Cluster transition would get stuck waiting for the remaining action ++ * timer to time out. ++ * ++ * Directly acknowledge the cancel operation in this case. ++ */ ++ if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id, ++ pending->op_type, pending->interval_ms)) { ++ need_direct_ack = TRUE; ++ } ++ + } else if (op->rsc_deleted) { + /* This recurring operation was cancelled (but not by us, and the + * executor does not have resource information, likely due to resource +diff --git a/daemons/controld/controld_te_callbacks.c b/daemons/controld/controld_te_callbacks.c +index 51d908e..22b5f4b 100644 +--- a/daemons/controld/controld_te_callbacks.c ++++ b/daemons/controld/controld_te_callbacks.c +@@ -32,19 +32,6 @@ static unsigned long int stonith_max_attempts = 10; + /* #define RSC_OP_TEMPLATE "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_CIB_TAG_STATE"[@uname='%s']"//"XML_LRM_TAG_RSC_OP"[@id='%s]" */ + #define RSC_OP_TEMPLATE "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_LRM_TAG_RSC_OP"[@id='%s']" + +-static const char * +-get_node_id(xmlNode * rsc_op) +-{ +- xmlNode *node = rsc_op; +- +- while (node != NULL && safe_str_neq(XML_CIB_TAG_STATE, TYPE(node))) { +- node = node->parent; +- } +- +- CRM_CHECK(node != NULL, return NULL); +- return ID(node); +-} +- + void + update_stonith_max_attempts(const char* value) + { +@@ -374,12 +361,8 @@ process_op_deletion(const char *xpath, xmlNode *change) + node_uuid = extract_node_uuid(xpath); + cancel = get_cancel_action(key, node_uuid); + if (cancel) { +- crm_info("Cancellation of %s on %s confirmed (%d)", +- key, node_uuid, cancel->id); +- stop_te_timer(cancel->timer); +- te_action_confirmed(cancel); +- update_graph(transition_graph, cancel); +- trigger_graph(); ++ confirm_cancel_action(cancel); ++ + } else { + abort_transition(INFINITY, tg_restart, "Resource operation removal", + change); +diff --git a/daemons/controld/controld_te_events.c b/daemons/controld/controld_te_events.c +index c0d096f..b7b48a4 100644 +--- a/daemons/controld/controld_te_events.c ++++ b/daemons/controld/controld_te_events.c +@@ -355,6 +355,27 @@ get_cancel_action(const char *id, const char *node) + return NULL; + } + ++void ++confirm_cancel_action(crm_action_t *cancel) ++{ ++ const char *op_key = NULL; ++ const char *node_name = NULL; ++ ++ CRM_ASSERT(cancel != NULL); ++ ++ op_key = crm_element_value(cancel->xml, XML_LRM_ATTR_TASK_KEY); ++ node_name = crm_element_value(cancel->xml, XML_LRM_ATTR_TARGET); ++ ++ stop_te_timer(cancel->timer); ++ te_action_confirmed(cancel); ++ update_graph(transition_graph, cancel); ++ ++ crm_info("Cancellation of %s on %s confirmed (action %d)", ++ op_key, node_name, cancel->id); ++ ++ trigger_graph(); ++} ++ + /* downed nodes are listed like: ... */ + #define XPATH_DOWNED "//" XML_GRAPH_TAG_DOWNED \ + "/" XML_CIB_TAG_NODE "[@" XML_ATTR_UUID "='%s']" +@@ -471,6 +492,17 @@ process_graph_event(xmlNode *event, const char *event_node) + /* Recurring actions have the transition number they were first + * scheduled in. + */ ++ ++ if (status == PCMK_LRM_OP_CANCELLED) { ++ const char *node_id = get_node_id(event); ++ ++ action = get_cancel_action(id, node_id); ++ if (action) { ++ confirm_cancel_action(action); ++ } ++ goto bail; ++ } ++ + desc = "arrived after initial scheduling"; + abort_transition(INFINITY, tg_restart, "Change in recurring result", + event); +diff --git a/daemons/controld/controld_transition.h b/daemons/controld/controld_transition.h +index 0a33599..a162f99 100644 +--- a/daemons/controld/controld_transition.h ++++ b/daemons/controld/controld_transition.h +@@ -25,6 +25,7 @@ void execute_stonith_cleanup(void); + /* tengine */ + extern crm_action_t *match_down_event(const char *target); + extern crm_action_t *get_cancel_action(const char *id, const char *node); ++void confirm_cancel_action(crm_action_t *cancel); + + void controld_record_action_timeout(crm_action_t *action); + extern gboolean fail_incompletable_actions(crm_graph_t * graph, const char *down_node); +diff --git a/daemons/controld/controld_utils.c b/daemons/controld/controld_utils.c +index ca7e15d..35922f0 100644 +--- a/daemons/controld/controld_utils.c ++++ b/daemons/controld/controld_utils.c +@@ -1073,3 +1073,16 @@ feature_set_compatible(const char *dc_version, const char *join_version) + // DC's minor version must be the same or older + return dc_v <= join_v; + } ++ ++const char * ++get_node_id(xmlNode *lrm_rsc_op) ++{ ++ xmlNode *node = lrm_rsc_op; ++ ++ while (node != NULL && safe_str_neq(XML_CIB_TAG_STATE, TYPE(node))) { ++ node = node->parent; ++ } ++ ++ CRM_CHECK(node != NULL, return NULL); ++ return ID(node); ++} +diff --git a/daemons/controld/controld_utils.h b/daemons/controld/controld_utils.h +index 2a92db5..68992f5 100644 +--- a/daemons/controld/controld_utils.h ++++ b/daemons/controld/controld_utils.h +@@ -95,6 +95,8 @@ unsigned int cib_op_timeout(void); + bool feature_set_compatible(const char *dc_version, const char *join_version); + bool controld_action_is_recordable(const char *action); + ++const char *get_node_id(xmlNode *lrm_rsc_op); ++ + /* Convenience macro for registering a CIB callback + * (assumes that data can be freed with free()) + */ +-- +1.8.3.1 + diff --git a/SOURCES/002-security.patch b/SOURCES/002-security.patch deleted file mode 100644 index e3701d0..0000000 --- a/SOURCES/002-security.patch +++ /dev/null @@ -1,2441 +0,0 @@ -From 32ded3e0172e0fae89cf70965e1c0406c1db883b Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Tue, 2 Apr 2019 10:13:21 +0200 -Subject: [PATCH 1/7] High: libservices: fix use-after-free wrt. alert handling - -This could possibly lead to unsolicited information disclosure by the -means of standard output of the immediately preceding agent/resource -execution leaking into the log stream under some circumstances. -It was hence assigned CVE-2019-3885. - -The provoked pathological state of pacemaker-execd daemon progresses -towards crashing it for hitting segmentation fault. ---- - lib/services/services.c | 40 +--------------------------------------- - lib/services/services_linux.c | 35 +++++++++++++++++++++++++++++++---- - 2 files changed, 32 insertions(+), 43 deletions(-) - -diff --git a/lib/services/services.c b/lib/services/services.c -index 313567f..d3537d0 100644 ---- a/lib/services/services.c -+++ b/lib/services/services.c -@@ -373,35 +373,6 @@ services_action_user(svc_action_t *op, const char *user) - return crm_user_lookup(user, &(op->opaque->uid), &(op->opaque->gid)); - } - --static void --set_alert_env(gpointer key, gpointer value, gpointer user_data) --{ -- int rc; -- -- if (value) { -- rc = setenv(key, value, 1); -- } else { -- rc = unsetenv(key); -- } -- -- if (rc < 0) { -- crm_perror(LOG_ERR, "setenv %s=%s", -- (char*)key, (value? (char*)value : "")); -- } else { -- crm_trace("setenv %s=%s", (char*)key, (value? (char*)value : "")); -- } --} -- --static void --unset_alert_env(gpointer key, gpointer value, gpointer user_data) --{ -- if (unsetenv(key) < 0) { -- crm_perror(LOG_ERR, "unset %s", (char*)key); -- } else { -- crm_trace("unset %s", (char*)key); -- } --} -- - /*! - * \brief Execute an alert agent action - * -@@ -416,18 +387,9 @@ unset_alert_env(gpointer key, gpointer value, gpointer user_data) - gboolean - services_alert_async(svc_action_t *action, void (*cb)(svc_action_t *op)) - { -- gboolean responsible; -- - action->synchronous = false; - action->opaque->callback = cb; -- if (action->params) { -- g_hash_table_foreach(action->params, set_alert_env, NULL); -- } -- responsible = services_os_action_execute(action); -- if (action->params) { -- g_hash_table_foreach(action->params, unset_alert_env, NULL); -- } -- return responsible; -+ return services_os_action_execute(action); - } - - #if SUPPORT_DBUS -diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c -index a04a8f9..90c1f44 100644 ---- a/lib/services/services_linux.c -+++ b/lib/services/services_linux.c -@@ -160,6 +160,25 @@ set_ocf_env_with_prefix(gpointer key, gpointer value, gpointer user_data) - set_ocf_env(buffer, value, user_data); - } - -+static void -+set_alert_env(gpointer key, gpointer value, gpointer user_data) -+{ -+ int rc; -+ -+ if (value != NULL) { -+ rc = setenv(key, value, 1); -+ } else { -+ rc = unsetenv(key); -+ } -+ -+ if (rc < 0) { -+ crm_perror(LOG_ERR, "setenv %s=%s", -+ (char*)key, (value? (char*)value : "")); -+ } else { -+ crm_trace("setenv %s=%s", (char*)key, (value? (char*)value : "")); -+ } -+} -+ - /*! - * \internal - * \brief Add environment variables suitable for an action -@@ -169,12 +188,20 @@ set_ocf_env_with_prefix(gpointer key, gpointer value, gpointer user_data) - static void - add_action_env_vars(const svc_action_t *op) - { -- if (safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_OCF) == FALSE) { -- return; -+ void (*env_setter)(gpointer, gpointer, gpointer) = NULL; -+ if (op->agent == NULL) { -+ env_setter = set_alert_env; /* we deal with alert handler */ -+ -+ } else if (safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_OCF)) { -+ env_setter = set_ocf_env_with_prefix; - } - -- if (op->params) { -- g_hash_table_foreach(op->params, set_ocf_env_with_prefix, NULL); -+ if (env_setter != NULL && op->params != NULL) { -+ g_hash_table_foreach(op->params, env_setter, NULL); -+ } -+ -+ if (env_setter == NULL || env_setter == set_alert_env) { -+ return; - } - - set_ocf_env("OCF_RA_VERSION_MAJOR", "1", NULL); --- -1.8.3.1 - - -From 912f5d9ce983339e939e4cc55f27791f8c9baa18 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Mon, 15 Apr 2019 17:09:50 +0200 -Subject: [PATCH 2/7] High: pacemakerd vs. IPC/procfs confused deputy - authenticity issue (0/4) - -[0/4: make crm_pid_active more precise as to when detections fail] - -It would be bad if the function claimed the process is not active -when the only obstacle in the detection process was that none of the -detection methods worked for a plain lack of permissions to apply -them. Also, do some other minor cleanup of the function and add its -documentation. As an additional measure, log spamming is kept at -minimum for repeated queries about the same PID. ---- - include/crm/common/internal.h | 21 +++++++++++++ - lib/common/pid.c | 68 +++++++++++++++++++++++++++---------------- - 2 files changed, 64 insertions(+), 25 deletions(-) - -diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h -index f2944f5..6775d14 100644 ---- a/include/crm/common/internal.h -+++ b/include/crm/common/internal.h -@@ -49,7 +49,28 @@ void crm_schema_cleanup(void); - - /* internal functions related to process IDs (from pid.c) */ - -+/*! -+ * \internal -+ * \brief Detect if process per PID and optionally exe path (component) exists -+ * -+ * \param[in] pid PID of process assumed alive, disproving of which to try -+ * \param[in] daemon exe path (component) to possibly match with procfs entry -+ * -+ * \return -1 on invalid PID specification, -2 when the calling process has no -+ * (is refused an) ability to (dis)prove the predicate, -+ * 0 if the negation of the predicate is confirmed (check-through-kill -+ * indicates so, or the subsequent check-through-procfs-match on -+ * \p daemon when provided and procfs available at the standard path), -+ * 1 if it cannot be disproved (reliably [modulo race conditions] -+ * when \p daemon provided, procfs available at the standard path -+ * and the calling process has permissions to access the respective -+ * procfs location, less so otherwise, since mere check-through-kill -+ * is exercised without powers to exclude PID recycled in the interim). -+ * -+ * \note This function cannot be used to verify \e authenticity of the process. -+ */ - int crm_pid_active(long pid, const char *daemon); -+ - long crm_pidfile_inuse(const char *filename, long mypid, const char *daemon); - long crm_read_pidfile(const char *filename); - int crm_lock_pidfile(const char *filename, const char *name); -diff --git a/lib/common/pid.c b/lib/common/pid.c -index 803799e..2439680 100644 ---- a/lib/common/pid.c -+++ b/lib/common/pid.c -@@ -1,5 +1,7 @@ - /* -- * Copyright 2004-2018 Andrew Beekhof -+ * Copyright 2004-2019 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. - * - * This source code is licensed under the GNU Lesser General Public License - * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -@@ -20,17 +22,21 @@ - int - crm_pid_active(long pid, const char *daemon) - { -+ static int last_asked_pid = 0; /* log spam prevention */ -+#if SUPPORT_PROCFS - static int have_proc_pid = 0; -+#else -+ static int have_proc_pid = -1; -+#endif -+ int rc = 0; - - if (have_proc_pid == 0) { -+ /* evaluation of /proc/PID/exe applicability via self-introspection */ - char proc_path[PATH_MAX], exe_path[PATH_MAX]; -- -- // Make sure pid hasn't been reused by another process - snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", -- (long unsigned int)getpid()); -- -+ (long unsigned int) getpid()); - have_proc_pid = 1; -- if (readlink(proc_path, exe_path, PATH_MAX - 1) < 0) { -+ if (readlink(proc_path, exe_path, sizeof(exe_path) - 1) < 0) { - have_proc_pid = -1; - } - } -@@ -38,40 +44,52 @@ crm_pid_active(long pid, const char *daemon) - if (pid <= 0) { - return -1; - -- } else if ((kill(pid, 0) < 0) && (errno == ESRCH)) { -- return 0; -+ } else if ((rc = kill(pid, 0)) < 0 && errno == ESRCH) { -+ return 0; /* no such PID detected */ - -- } else if ((daemon == NULL) || (have_proc_pid == -1)) { -- return 1; -+ } else if (rc < 0 && have_proc_pid == -1) { -+ if (last_asked_pid != pid) { -+ crm_info("Cannot examine PID %ld: %s", pid, strerror(errno)); -+ last_asked_pid = pid; -+ } -+ return -2; /* errno != ESRCH */ -+ -+ } else if (rc == 0 && (daemon == NULL || have_proc_pid == -1)) { -+ return 1; /* kill as the only indicator, cannot double check */ - - } else { -- int rc = 0; -+ /* make sure PID hasn't been reused by another process -+ XXX: might still be just a zombie, which could confuse decisions */ -+ bool checked_through_kill = (rc == 0); - char proc_path[PATH_MAX], exe_path[PATH_MAX], myexe_path[PATH_MAX]; -- -- // Make sure pid hasn't been reused by another process - snprintf(proc_path, sizeof(proc_path), "/proc/%ld/exe", pid); - -- rc = readlink(proc_path, exe_path, PATH_MAX - 1); -+ rc = readlink(proc_path, exe_path, sizeof(exe_path) - 1); - if ((rc < 0) && (errno == EACCES)) { -- crm_perror(LOG_INFO, "Could not read from %s", proc_path); -- return 1; -+ if (last_asked_pid != pid) { -+ crm_info("Could not read from %s: %s", proc_path, -+ strerror(errno)); -+ last_asked_pid = pid; -+ } -+ return checked_through_kill ? 1 : -2; - } else if (rc < 0) { -- crm_perror(LOG_ERR, "Could not read from %s", proc_path); -- return 0; -+ if (last_asked_pid != pid) { -+ crm_err("Could not read from %s: %s (%d)", proc_path, -+ strerror(errno), errno); -+ last_asked_pid = pid; -+ } -+ return 0; /* most likely errno == ENOENT */ - } -- -- exe_path[rc] = 0; -+ exe_path[rc] = '\0'; - - if (daemon[0] != '/') { -- rc = snprintf(myexe_path, sizeof(proc_path), CRM_DAEMON_DIR"/%s", -+ rc = snprintf(myexe_path, sizeof(myexe_path), CRM_DAEMON_DIR"/%s", - daemon); -- myexe_path[rc] = 0; - } else { -- rc = snprintf(myexe_path, sizeof(proc_path), "%s", daemon); -- myexe_path[rc] = 0; -+ rc = snprintf(myexe_path, sizeof(myexe_path), "%s", daemon); - } - -- if (strcmp(exe_path, myexe_path) == 0) { -+ if (rc > 0 && rc < sizeof(myexe_path) && !strcmp(exe_path, myexe_path)) { - return 1; - } - } --- -1.8.3.1 - - -From 1148f45da977113dff588cdd1cfebb7a47760b32 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Mon, 15 Apr 2019 17:10:07 +0200 -Subject: [PATCH 3/7] High: pacemakerd vs. IPC/procfs confused deputy - authenticity issue (1/4) - -[1/4: new helpers to allow IPC client side to authenticate the server] - -The title problem here could possibly lead to local privilege escalation -up to the root's level (and implicitly unguarded by some additional -protection layers like SELinux unless the defaults constrained further). - -Main problem is that the authenticity assumptions were built on two, -seemingly mutually supporting legs leading to two CVEs assigned: - -* procfs (mere process existence and right path to binary check) - used to verify (this part was assigned CVE-2018-16878), and - -* one-way only client-server authentication, putting the client - here at the mercy of the server not necessarily cross-verified - per the above point if at all (this part was assigned - CVE-2018-16877) - -whereas these two were in fact orthogonal, tearing security -assumptions about the "passive influencers" in the pacemaker's daemon -resilience-friendly constellation (orchestrated by the main of them, -pacemakerd) apart. Moreover, procfs-based approach is discouraged -for other reasons. - -The layout of the basic fix is as follows: -* 1/4: new helpers to allow IPC client side to authenticate the server - (this commit, along with unifying subsequent solution for - both CVEs) -* 2/4: pacemakerd to trust pre-existing processes via new checks instead - (along with unifying solution for both CVEs) -* 3/4: other daemons to authenticate IPC servers of fellow processes - (along with addressing CVE-2018-16877 alone, for parts of - pacemaker not covered earlier) -* 4/4: CPG users to be careful about now-more-probable rival processes - (this is merely to mitigate corner case fallout from the new - approaches taken to face CVE-2018-16878 in particular; - courtesy of Yan Gao of SUSE for the reporting this) - -With "basic", it is meant that it constitutes a self-contained best -effort solution with some compromises that can only be overcome with the -assistance of IPC library, libqb, as is also elaborated in messages of -remaining "fix" commits. Beside that, also conventional encapsulation -of server-by-client authentication would be useful, but lack thereof -is not an obstacle (more so should there by any security related -neglectations on the IPC client side and its connection initiating -arrangement within libqb that has a potential to strike as early as -when the authenticity of the server side is yet to be examined). - -One extra kludge that's introduced for FreeBSD lacking Unix socket to -remote peer PID mapping is masquerading such an unspecified PID with -value of 1, since that shall always be around as "init" task and, -deferring to proof by contradiction, cannot be pacemakerd-spawned -child either even if PID 1 was pacemakerd (and running such a child -alone is rather nonsensical). The code making decisions based on that -value must acknowledge this craze and refrain from killing/signalling -the underlying process on this platform (but shall in general follow -the same elsewhere, keep in mind systemd socket-based activation for -instance, which would end up in such a situation easily!). ---- - configure.ac | 43 +++++++++++ - include/crm/common/Makefile.am | 3 +- - include/crm/common/ipc.h | 40 ++++++++++- - include/crm/common/ipc_internal.h | 69 ++++++++++++++++++ - lib/common/ipc.c | 145 +++++++++++++++++++++++++++++++++++++- - 5 files changed, 296 insertions(+), 4 deletions(-) - create mode 100644 include/crm/common/ipc_internal.h - -diff --git a/configure.ac b/configure.ac -index f272587..7228989 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -425,6 +425,48 @@ do - fi - done - -+us_auth= -+AC_CHECK_HEADER([sys/socket.h], [ -+ AC_CHECK_DECL([SO_PEERCRED], [ -+ # Linux -+ AC_CHECK_TYPE([struct ucred], [ -+ us_auth=peercred_ucred; -+ AC_DEFINE([US_AUTH_PEERCRED_UCRED], [1], -+ [Define if Unix socket auth method is -+ getsockopt(s, SO_PEERCRED, &ucred, ...)]) -+ ], [ -+ # OpenBSD -+ AC_CHECK_TYPE([struct sockpeercred], [ -+ us_auth=localpeercred_sockepeercred; -+ AC_DEFINE([US_AUTH_PEERCRED_SOCKPEERCRED], [1], -+ [Define if Unix socket auth method is -+ getsockopt(s, SO_PEERCRED, &sockpeercred, ...)]) -+ ], [], [[#include ]]) -+ ], [[#define _GNU_SOURCE -+ #include ]]) -+ ], [], [[#include ]]) -+]) -+ -+if test -z "${us_auth}"; then -+ # FreeBSD -+ AC_CHECK_DECL([getpeereid], [ -+ us_auth=getpeereid; -+ AC_DEFINE([US_AUTH_GETPEEREID], [1], -+ [Define if Unix socket auth method is -+ getpeereid(s, &uid, &gid)]) -+ ], [ -+ # Solaris/OpenIndiana -+ AC_CHECK_DECL([getpeerucred], [ -+ us_auth=getpeerucred; -+ AC_DEFINE([US_AUTH_GETPEERUCRED], [1], -+ [Define if Unix socket auth method is -+ getpeercred(s, &ucred)]) -+ ], [ -+ AC_MSG_ERROR([No way to authenticate a Unix socket peer]) -+ ], [[#include ]]) -+ ]) -+fi -+ - dnl This OS-based decision-making is poor autotools practice; - dnl feature-based mechanisms are strongly preferred. - dnl -@@ -1845,3 +1887,4 @@ AC_MSG_RESULT([ LDFLAGS_HARDENED_EXE = ${LDFLAGS_HARDENED_EXE}]) - AC_MSG_RESULT([ LDFLAGS_HARDENED_LIB = ${LDFLAGS_HARDENED_LIB}]) - AC_MSG_RESULT([ Libraries = ${LIBS}]) - AC_MSG_RESULT([ Stack Libraries = ${CLUSTERLIBS}]) -+AC_MSG_RESULT([ Unix socket auth method = ${us_auth}]) -diff --git a/include/crm/common/Makefile.am b/include/crm/common/Makefile.am -index 7400152..be19e5e 100644 ---- a/include/crm/common/Makefile.am -+++ b/include/crm/common/Makefile.am -@@ -14,3 +14,4 @@ headerdir=$(pkgincludedir)/crm/common - header_HEADERS = xml.h ipc.h util.h iso8601.h mainloop.h logging.h results.h - noinst_HEADERS = cib_secrets.h ipcs.h internal.h alerts_internal.h \ -- iso8601_internal.h remote_internal.h xml_internal.h -+ iso8601_internal.h remote_internal.h xml_internal.h \ -+ ipc_internal.h -diff --git a/include/crm/common/ipc.h b/include/crm/common/ipc.h -index 84a3a71..3249662 100644 ---- a/include/crm/common/ipc.h -+++ b/include/crm/common/ipc.h -@@ -1,5 +1,7 @@ - /* -- * Copyright 2013-2018 Andrew Beekhof -+ * Copyright 2004-2019 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. - * - * This source code is licensed under the GNU Lesser General Public License - * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -@@ -75,6 +75,44 @@ uint32_t crm_ipc_buffer_flags(crm_ipc_t * client); - const char *crm_ipc_name(crm_ipc_t * client); - unsigned int crm_ipc_default_buffer_size(void); - -+/*! -+ * \brief Check the authenticity of the IPC socket peer process -+ * -+ * If everything goes well, peer's authenticity is verified by the means -+ * of comparing against provided referential UID and GID (either satisfies), -+ * and the result of this check can be deduced from the return value. -+ * As an exception, detected UID of 0 ("root") satisfies arbitrary -+ * provided referential daemon's credentials. -+ * -+ * \param[in] sock IPC related, connected Unix socket to check peer of -+ * \param[in] refuid referential UID to check against -+ * \param[in] refgid referential GID to check against -+ * \param[out] gotpid to optionally store obtained PID of the peer -+ * (not available on FreeBSD, special value of 1 -+ * used instead, and the caller is required to -+ * special case this value respectively) -+ * \param[out] gotuid to optionally store obtained UID of the peer -+ * \param[out] gotgid to optionally store obtained GID of the peer -+ * -+ * \return 0 if IPC related socket's peer is not authentic given the -+ * referential credentials (see above), 1 if it is, -+ * negative value on error (generally expressing -errno unless -+ * it was zero even on nonhappy path, -pcmk_err_generic is -+ * returned then; no message is directly emitted) -+ * -+ * \note While this function is tolerant on what constitutes authorized -+ * IPC daemon process (its effective user matches UID=0 or \p refuid, -+ * or at least its group matches \p refroup), either or both (in case -+ * of UID=0) mismatches on the expected credentials of such peer -+ * process \e shall be investigated at the caller when value of 1 -+ * gets returned there, since higher-than-expected privileges in -+ * respect to the expected/intended credentials possibly violate -+ * the least privilege principle and may pose an additional risk -+ * (i.e. such accidental inconsistency shall be eventually fixed). -+ */ -+int crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid, -+ pid_t *gotpid, uid_t *gotuid, gid_t *gotgid); -+ - /* Utils */ - xmlNode *create_hello_message(const char *uuid, const char *client_name, - const char *major_version, const char *minor_version); -diff --git a/include/crm/common/ipc_internal.h b/include/crm/common/ipc_internal.h -new file mode 100644 -index 0000000..41a6653 ---- /dev/null -+++ b/include/crm/common/ipc_internal.h -@@ -0,0 +1,69 @@ -+/* -+ * Copyright 2019 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. -+ * -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -+ */ -+ -+#ifndef PCMK__IPC_INTERNAL_H -+#define PCMK__IPC_INTERNAL_H -+ -+#include -+ -+#include /* US_AUTH_GETPEEREID */ -+ -+ -+/* denotes "non yieldable PID" on FreeBSD, or actual PID1 in scenarios that -+ require a delicate handling anyway (socket-based activation with systemd); -+ we can be reasonably sure that this PID is never possessed by the actual -+ child daemon, as it gets taken either by the proper init, or by pacemakerd -+ itself (i.e. this precludes anything else); note that value of zero -+ is meant to carry "unset" meaning, and better not to bet on/conditionalize -+ over signedness of pid_t */ -+#define PCMK__SPECIAL_PID 1 -+ -+#if defined(US_AUTH_GETPEEREID) -+/* on FreeBSD, we don't want to expose "non-yieldable PID" (leading to -+ "IPC liveness check only") as its nominal representation, which could -+ cause confusion -- this is unambiguous as long as there's no -+ socket-based activation like with systemd (very improbable) */ -+#define PCMK__SPECIAL_PID_AS_0(p) (((p) == PCMK__SPECIAL_PID) ? 0 : (p)) -+#else -+#define PCMK__SPECIAL_PID_AS_0(p) (p) -+#endif -+ -+/*! -+ * \internal -+ * \brief Check the authenticity and liveness of the process via IPC end-point -+ * -+ * When IPC daemon under given IPC end-point (name) detected, its authenticity -+ * is verified by the means of comparing against provided referential UID and -+ * GID, and the result of this check can be deduced from the return value. -+ * As an exception, referential UID of 0 (~ root) satisfies arbitrary -+ * detected daemon's credentials. -+ * -+ * \param[in] name IPC name to base the search on -+ * \param[in] refuid referential UID to check against -+ * \param[in] refgid referential GID to check against -+ * \param[out] gotpid to optionally store obtained PID of the found process -+ * upon returning 1 or -2 -+ * (not available on FreeBSD, special value of 1, -+ * see PCMK__SPECIAL_PID, used instead, and the caller -+ * is required to special case this value respectively) -+ * -+ * \return 0 if no trace of IPC peer's liveness detected, 1 if it was, -+ * -1 on error, and -2 when the IPC blocked with unauthorized -+ * process (log message emitted in both latter cases) -+ * -+ * \note This function emits a log message also in case there isn't a perfect -+ * match in respect to \p reguid and/or \p refgid, for a possible -+ * least privilege principle violation. -+ * -+ * \see crm_ipc_is_authentic_process -+ */ -+int pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid, -+ gid_t refgid, pid_t *gotpid); -+ -+#endif -diff --git a/lib/common/ipc.c b/lib/common/ipc.c -index c582ccf..aa055d3 100644 ---- a/lib/common/ipc.c -+++ b/lib/common/ipc.c -@@ -1,5 +1,7 @@ - /* -- * Copyright 2004-2018 Andrew Beekhof -+ * Copyright 2004-2019 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. - * - * This source code is licensed under the GNU Lesser General Public License - * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -@@ -7,6 +9,17 @@ - - #include - -+#if defined(US_AUTH_PEERCRED_UCRED) || defined(US_AUTH_PEERCRED_SOCKPEERCRED) -+# ifdef US_AUTH_PEERCRED_UCRED -+# ifndef _GNU_SOURCE -+# define _GNU_SOURCE -+# endif -+# endif -+# include -+#elif defined(US_AUTH_GETPEERUCRED) -+# include -+#endif -+ - #include - - #include -@@ -19,11 +32,13 @@ - #include - #include - --#include -+#include /* indirectly: pcmk_err_generic */ - #include - #include - #include - -+#include /* PCMK__SPECIAL_PID* */ -+ - #define PCMK_IPC_VERSION 1 - - /* Evict clients whose event queue grows this large (by default) */ -@@ -1394,6 +1409,132 @@ crm_ipc_send(crm_ipc_t * client, xmlNode * message, enum crm_ipc_flags flags, in - return rc; - } - -+int -+crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid, -+ pid_t *gotpid, uid_t *gotuid, gid_t *gotgid) { -+ int ret = 0; -+ pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; -+#if defined(US_AUTH_PEERCRED_UCRED) -+ struct ucred ucred; -+ socklen_t ucred_len = sizeof(ucred); -+ -+ if (!getsockopt(sock, SOL_SOCKET, SO_PEERCRED, -+ &ucred, &ucred_len) -+ && ucred_len == sizeof(ucred)) { -+ found_pid = ucred.pid; found_uid = ucred.uid; found_gid = ucred.gid; -+ -+#elif defined(US_AUTH_PEERCRED_SOCKPEERCRED) -+ struct sockpeercred sockpeercred; -+ socklen_t sockpeercred_len = sizeof(sockpeercred); -+ -+ if (!getsockopt(sock, SOL_SOCKET, SO_PEERCRED, -+ &sockpeercred, &sockpeercred_len) -+ && sockpeercred_len == sizeof(sockpeercred_len)) { -+ found_pid = sockpeercred.pid; -+ found_uid = sockpeercred.uid; found_gid = sockpeercred.gid; -+ -+#elif defined(US_AUTH_GETPEEREID) -+ if (!getpeereid(sock, &found_uid, &found_gid)) { -+ found_pid = PCMK__SPECIAL_PID; /* cannot obtain PID (FreeBSD) */ -+ -+#elif defined(US_AUTH_GETPEERUCRED) -+ ucred_t *ucred; -+ if (!getpeerucred(sock, &ucred)) { -+ errno = 0; -+ found_pid = ucred_getpid(ucred); -+ found_uid = ucred_geteuid(ucred); found_gid = ucred_getegid(ucred); -+ ret = -errno; -+ ucred_free(ucred); -+ if (ret) { -+ return (ret < 0) ? ret : -pcmk_err_generic; -+ } -+ -+#else -+# error "No way to authenticate a Unix socket peer" -+ errno = 0; -+ if (0) { -+#endif -+ if (gotpid != NULL) { -+ *gotpid = found_pid; -+ } -+ if (gotuid != NULL) { -+ *gotuid = found_uid; -+ } -+ if (gotgid != NULL) { -+ *gotgid = found_gid; -+ } -+ ret = (found_uid == 0 || found_uid == refuid || found_gid == refgid); -+ } else { -+ ret = (errno > 0) ? -errno : -pcmk_err_generic; -+ } -+ -+ return ret; -+} -+ -+int -+pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid, -+ gid_t refgid, pid_t *gotpid) { -+ static char last_asked_name[PATH_MAX / 2] = ""; /* log spam prevention */ -+ int fd, ret = 0; -+ pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; -+ qb_ipcc_connection_t *c; -+ -+ if ((c = qb_ipcc_connect(name, 0)) == NULL) { -+ crm_info("Could not connect to %s IPC: %s", name, strerror(errno)); -+ -+ } else if ((ret = qb_ipcc_fd_get(c, &fd))) { -+ crm_err("Could not get fd from %s IPC: %s (%d)", name, -+ strerror(-ret), -ret); -+ ret = -1; -+ -+ } else if ((ret = crm_ipc_is_authentic_process(fd, refuid, refgid, -+ &found_pid, &found_uid, -+ &found_gid)) < 0) { -+ if (ret == -pcmk_err_generic) { -+ crm_err("Could not get peer credentials from %s IPC", name); -+ } else { -+ crm_err("Could not get peer credentials from %s IPC: %s (%d)", -+ name, strerror(-ret), -ret); -+ } -+ ret = -1; -+ -+ } else { -+ if (gotpid != NULL) { -+ *gotpid = found_pid; -+ } -+ -+ if (!ret) { -+ crm_err("Daemon (IPC %s) effectively blocked with unauthorized" -+ " process %lld (uid: %lld, gid: %lld)", -+ name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ ret = -2; -+ } else if ((found_uid != refuid || found_gid != refgid) -+ && strncmp(last_asked_name, name, sizeof(last_asked_name))) { -+ if (!found_uid && refuid) { -+ crm_warn("Daemon (IPC %s) runs as root, whereas the expected" -+ " credentials are %lld:%lld, hazard of violating" -+ " the least privilege principle", -+ name, (long long) refuid, (long long) refgid); -+ } else { -+ crm_notice("Daemon (IPC %s) runs as %lld:%lld, whereas the" -+ " expected credentials are %lld:%lld, which may" -+ " mean a different set of privileges than expected", -+ name, (long long) found_uid, (long long) found_gid, -+ (long long) refuid, (long long) refgid); -+ } -+ memccpy(last_asked_name, name, '\0', sizeof(last_asked_name)); -+ } -+ } -+ -+ if (ret) { /* here, !ret only when we could not initially connect */ -+ qb_ipcc_disconnect(c); -+ } -+ -+ return ret; -+} -+ -+ - /* Utils */ - - xmlNode * --- -1.8.3.1 - - -From 970736b1c7ad5c78cc5295a4231e546104d55893 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Tue, 16 Apr 2019 00:13:06 +0200 -Subject: [PATCH 4/7] High: pacemakerd vs. IPC/procfs confused deputy - authenticity issue (2/4) - -[2/4: pacemakerd to trust pre-existing processes via new checks instead] - -In pacemakerd in the context of entrusting pre-existing processes, -we now resort to procfs-based solution only in boundary, fouled cases, -and primarily examine the credentials of the processes already -occupying known IPC end-points before adopting them. - -The commit applies the new helpers from 1/1 so as to close the both -related sensitive problems, CVE-2018-16877 and CVE-2018-16878, in -a unified manner, this time limited to the main daemon of pacemaker -(pacemakerd). - -To be noted that it is clearly not 100% for this purpose for still -allowing for TOCTTOU, but that's what commit (3/3) is meant to solve -for the most part, plus there may be optimizations solving this concern -as a side effect, but that requires an active assistance on the libqb -side (https://github.com/ClusterLabs/libqb/issues/325) since any -improvement on pacemaker side in isolation would be very -cumbersome if generally possible at all, but either way -means a new, soft compatibility encumberment. - -As a follow-up to what was put in preceding 1/3 commit, PID of 1 tracked -as child's identification on FreeBSD (or when socket-based activation is -used with systemd) is treated specially, incl. special precaution with -child's PID discovered as 1 elsewhere. - -v2: courtesy of Yan Gao of SUSE for early discovery and report for - what's primarily solved with 4/4 commit, in extension, child - daemons in the initialization phase coinciding with IPC-feasibility - based process scan in pacemakerd in a way that those are missed - (although they are to come up fully just moments later only - to interfere with naturally spawned ones) are now considered so - that if any native children later fail for said clash, the - pre-existing counterpart may get adopted instead of ending up - with repeated spawn-bury loop ad nauseam without real progress - (note that PCMK_fail_fast=true could possibly help, but that's - rather a big hammer not suitable for all the use cases, not - the ones we try to deal with gracefully here) ---- - daemons/pacemakerd/pacemakerd.c | 406 ++++++++++++++++++++++++++++++++++------ - 1 file changed, 345 insertions(+), 61 deletions(-) - -diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c -index eca7f4d..bebf938 100644 ---- a/daemons/pacemakerd/pacemakerd.c -+++ b/daemons/pacemakerd/pacemakerd.c -@@ -12,17 +12,23 @@ - - #include - #include -+#include - #include - #include - #include - #include - #include - -+#include /* indirectly: CRM_EX_* */ -+#include /* cib_channel_ro */ - #include - #include - #include - #include - #include -+ -+#include /* PCMK__SPECIAL_PID*, ... */ -+ - #ifdef SUPPORT_COROSYNC - #include - #endif -@@ -33,6 +39,7 @@ - static gboolean pcmk_quorate = FALSE; - static gboolean fatal_error = FALSE; - static GMainLoop *mainloop = NULL; -+static bool global_keep_tracking = false; - - #define PCMK_PROCESS_CHECK_INTERVAL 5 - -@@ -50,6 +57,7 @@ typedef struct pcmk_child_s { - const char *name; - const char *uid; - const char *command; -+ const char *endpoint; /* IPC server name */ - - gboolean active_before_startup; - } pcmk_child_t; -@@ -64,30 +72,38 @@ static pcmk_child_t pcmk_children[] = { - }, - { - 0, crm_proc_execd, 3, 0, TRUE, "pacemaker-execd", -- NULL, CRM_DAEMON_DIR "/pacemaker-execd" -+ NULL, CRM_DAEMON_DIR "/pacemaker-execd", -+ CRM_SYSTEM_LRMD - }, - { - 0, crm_proc_based, 1, 0, TRUE, "pacemaker-based", -- CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-based" -+ CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-based", -+ CIB_CHANNEL_RO - }, - { - 0, crm_proc_controld, 6, 0, TRUE, "pacemaker-controld", -- CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-controld" -+ CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-controld", -+ CRM_SYSTEM_CRMD - }, - { - 0, crm_proc_attrd, 4, 0, TRUE, "pacemaker-attrd", -- CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-attrd" -+ CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-attrd", -+ T_ATTRD - }, - { - 0, crm_proc_schedulerd, 5, 0, TRUE, "pacemaker-schedulerd", -- CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-schedulerd" -+ CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-schedulerd", -+ CRM_SYSTEM_PENGINE - }, - { - 0, crm_proc_fenced, 2, 0, TRUE, "pacemaker-fenced", -- NULL, CRM_DAEMON_DIR "/pacemaker-fenced" -+ NULL, CRM_DAEMON_DIR "/pacemaker-fenced", -+ "stonith-ng" - }, - }; - -+static gboolean check_active_before_startup_processes(gpointer user_data); -+static int pcmk_child_active(pcmk_child_t *child); - static gboolean start_child(pcmk_child_t * child); - static gboolean update_node_processes(uint32_t id, const char *uname, - uint32_t procs); -@@ -130,14 +146,31 @@ pcmk_process_exit(pcmk_child_t * child) - } - - if (shutdown_trigger) { -+ /* resume step-wise shutdown (returned TRUE yields no parallelizing) */ - mainloop_set_trigger(shutdown_trigger); -+ /* intended to speed up propagating expected lay-off of the daemons? */ - update_node_processes(local_nodeid, NULL, get_process_list()); - -- } else if (child->respawn && crm_is_true(getenv("PCMK_fail_fast"))) { -+ } else if (!child->respawn) { -+ /* nothing to do */ -+ -+ } else if (crm_is_true(getenv("PCMK_fail_fast"))) { - crm_err("Rebooting system because of %s", child->name); - pcmk_panic(__FUNCTION__); - -- } else if (child->respawn) { -+ } else if (pcmk_child_active(child) == 1) { -+ crm_warn("One-off suppressing strict respawning of a child process %s," -+ " appears alright per %s IPC end-point", -+ child->name, child->endpoint); -+ /* need to monitor how it evolves, and start new process if badly */ -+ child->active_before_startup = TRUE; -+ if (!global_keep_tracking) { -+ global_keep_tracking = true; -+ g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL, -+ check_active_before_startup_processes, NULL); -+ } -+ -+ } else { - crm_notice("Respawning failed child process: %s", child->name); - start_child(child); - } -@@ -218,8 +251,13 @@ stop_child(pcmk_child_t * child, int signal) - signal = SIGTERM; - } - -- if (child->command == NULL) { -- crm_debug("Nothing to do for child \"%s\"", child->name); -+ /* why to skip PID of 1? -+ - FreeBSD ~ how untrackable process behind IPC is masqueraded as -+ - elsewhere: how "init" task is designated; in particular, in systemd -+ arrangement of socket-based activation, this is pretty real */ -+ if (child->command == NULL || child->pid == PCMK__SPECIAL_PID) { -+ crm_debug("Nothing to do for child \"%s\" (process %lld)", -+ child->name, (long long) PCMK__SPECIAL_PID_AS_0(child->pid)); - return TRUE; - } - -@@ -244,6 +282,11 @@ stop_child(pcmk_child_t * child, int signal) - static char *opts_default[] = { NULL, NULL }; - static char *opts_vgrind[] = { NULL, NULL, NULL, NULL, NULL }; - -+/* TODO once libqb is taught to juggle with IPC end-points carried over as -+ bare file descriptor (https://github.com/ClusterLabs/libqb/issues/325) -+ it shall hand over these descriptors here if/once they are successfully -+ pre-opened in (presumably) pcmk_child_active, to avoid any remaining -+ room for races */ - static gboolean - start_child(pcmk_child_t * child) - { -@@ -380,7 +423,10 @@ escalate_shutdown(gpointer data) - - pcmk_child_t *child = data; - -- if (child->pid) { -+ if (child->pid == PCMK__SPECIAL_PID) { -+ pcmk_process_exit(child); -+ -+ } else if (child->pid) { - /* Use SIGSEGV instead of SIGKILL to create a core so we can see what it was up to */ - crm_err("Child %s not terminating in a timely manner, forcing", child->name); - stop_child(child, SIGSEGV); -@@ -388,6 +434,8 @@ escalate_shutdown(gpointer data) - return FALSE; - } - -+#define SHUTDOWN_ESCALATION_PERIOD 180000 /* 3m */ -+ - static gboolean - pcmk_shutdown_worker(gpointer user_data) - { -@@ -416,11 +464,24 @@ pcmk_shutdown_worker(gpointer user_data) - time_t now = time(NULL); - - if (child->respawn) { -+ if (child->pid == PCMK__SPECIAL_PID) { -+ crm_warn("The process behind %s IPC cannot be" -+ " terminated, so either wait the graceful" -+ " period of %ld s for its native termination" -+ " if it vitally depends on some other daemons" -+ " going down in a controlled way already," -+ " or locate and kill the correct %s process" -+ " on your own; set PCMK_fail_fast=1 to avoid" -+ " this altogether next time around", -+ child->name, (long) SHUTDOWN_ESCALATION_PERIOD, -+ child->command); -+ } - next_log = now + 30; - child->respawn = FALSE; - stop_child(child, SIGTERM); - if (phase < pcmk_children[PCMK_CHILD_CONTROLD].start_seq) { -- g_timeout_add(180000 /* 3m */ , escalate_shutdown, child); -+ g_timeout_add(SHUTDOWN_ESCALATION_PERIOD, -+ escalate_shutdown, child); - } - - } else if (now >= next_log) { -@@ -702,7 +763,102 @@ mcp_chown(const char *path, uid_t uid, gid_t gid) - } - } - --#if SUPPORT_PROCFS -+/*! -+ * \internal -+ * \brief Check the liveness of the child based on IPC name and PID if tracked -+ * -+ * \param[inout] child Child tracked data -+ * -+ * \return 0 if no trace of child's liveness detected (while it is detectable -+ * to begin with, at least according to one of the two properties), -+ * 1 if everything is fine, 2 if it's up per PID, but not per IPC -+ * end-point (still starting?), -1 on error, and -2 when the child -+ * (its IPC) blocked with an unauthorized process (log message -+ * emitted in both latter cases) -+ * -+ * \note This function doesn't modify any of \p child members but \c pid, -+ * and is not actively toying with processes as such but invoking -+ * \c stop_child in one particular case (there's for some reason -+ * a different authentic holder of the IPC end-point). -+ */ -+static int -+pcmk_child_active(pcmk_child_t *child) { -+ static uid_t cl_uid = 0; -+ static gid_t cl_gid = 0; -+ const uid_t root_uid = 0; -+ const gid_t root_gid = 0; -+ const uid_t *ref_uid; -+ const gid_t *ref_gid; -+ int ret = 0; -+ pid_t ipc_pid = 0; -+ -+ if (child->endpoint == NULL -+ && (child->pid <= 0 || child->pid == PCMK__SPECIAL_PID)) { -+ crm_err("Cannot track child %s for missing both API end-point and PID", -+ child->name); -+ ret = -1; /* misuse of the function when child is not trackable */ -+ -+ } else if (child->endpoint != NULL) { -+ -+ ref_uid = (child->uid != NULL) ? &cl_uid : &root_uid; -+ ref_gid = (child->uid != NULL) ? &cl_gid : &root_gid; -+ -+ if (child->uid != NULL && !cl_uid && !cl_gid -+ && crm_user_lookup(CRM_DAEMON_USER, &cl_uid, &cl_gid) < 0) { -+ crm_err("Could not find user and group IDs for user %s", -+ CRM_DAEMON_USER); -+ ret = -1; -+ } else if ((ret = pcmk__ipc_is_authentic_process_active(child->endpoint, -+ *ref_uid, *ref_gid, -+ &ipc_pid)) < 0) { -+ /* game over */ -+ } else if (child->pid <= 0) { -+ /* hit new child to be initialized, or reset to zero -+ and investigate further for ret == 0 */ -+ child->pid = ipc_pid; -+ } else if (ipc_pid && child->pid != ipc_pid) { -+ /* ultimately strange for ret == 1; either way, investigate */ -+ ret = 0; -+ } -+ } -+ -+ if (!ret) { -+ /* when no IPC based liveness detected (incl. if ever a child without -+ IPC is tracked), or detected for a different _authentic_ process; -+ safe on FreeBSD since the only change possible from a proper child's -+ PID into "special" PID of 1 behind more loosely related process */ -+ ret = crm_pid_active(child->pid, child->name); -+ if (ipc_pid && (ret != 1 -+ || ipc_pid == PCMK__SPECIAL_PID -+ || crm_pid_active(ipc_pid, child->name) == 1)) { -+ if (ret == 1) { -+ /* assume there's no forking-while-retaining-IPC-socket -+ involved in the "children's" lifecycle, hence that the -+ tracking got out of sync purely because of some external -+ (esotheric?) forces (user initiated process "refresh" by -+ force? or intentionally racing on start-up, even?), and -+ that switching over to this other detected, authentic -+ instance with an IPC already in possession is a better -+ trade-off than "neutralizing" it first so as to give -+ either the original or possibly a new to-be-spawned -+ daemon process a leeway for operation, which would -+ otherwise have to be carried out */ -+ /* not possessing IPC, afterall (what about corosync CPG?) */ -+ stop_child(child, SIGKILL); -+ } else { -+ ret = 1; -+ } -+ child->pid = ipc_pid; -+ } else if (ret == 1) { -+ ret = 2; /* up per PID, but not per IPC (still starting?) */ -+ } else if (!child->pid && ret == -1) { -+ ret = 0; /* correct -1 on FreeBSD from above back to 0 */ -+ } -+ } -+ -+ return ret; -+} -+ - static gboolean - check_active_before_startup_processes(gpointer user_data) - { -@@ -719,12 +875,41 @@ check_active_before_startup_processes(gpointer user_data) - continue; - } else { - const char *name = pcmk_children[lpc].name; -- -- if (crm_pid_active(pcmk_children[lpc].pid, name) != 1) { -- crm_notice("Process %s terminated (pid=%d)", -- name, pcmk_children[lpc].pid); -- pcmk_process_exit(&(pcmk_children[lpc])); -- continue; -+ int ret; -+ -+ switch ((ret = pcmk_child_active(&pcmk_children[lpc]))) { -+ case 1: -+ break; -+ case 0: -+ case 2: /* this very case: it was OK once already */ -+ if (pcmk_children[lpc].respawn == TRUE) { -+ /* presumably after crash, hence critical */ -+ crm_crit("Process %s terminated (pid=%lld)%s", \ -+ name, (long long) -+ PCMK__SPECIAL_PID_AS_0(pcmk_children[lpc].pid), -+ ret ? ", at least per IPC end-point that went AWOL" -+ : ""); -+ } else { -+ /* orderly shutdown */ -+ crm_notice("Process %s terminated (pid=%lld)%s", \ -+ name, (long long) -+ PCMK__SPECIAL_PID_AS_0(pcmk_children[lpc].pid), -+ ret ? ", at least per IPC end-point that went AWOL" -+ : ""); -+ } -+ pcmk_process_exit(&(pcmk_children[lpc])); -+ continue; -+ default: -+ crm_crit("Unexpected value from pcmk_child_active:" -+ " %d (pid=%lld)", ret, -+ (long long) PCMK__SPECIAL_PID_AS_0( -+ pcmk_children[lpc].pid)); -+ /* fall through */ -+ case -1: -+ case -2: -+ /* message(s) already emitted */ -+ crm_exit(CRM_EX_FATAL); -+ break; /* static analysis/noreturn */ - } - } - /* at least one of the processes found at startup -@@ -733,57 +918,147 @@ check_active_before_startup_processes(gpointer user_data) - } - } - -+ global_keep_tracking = keep_tracking; - return keep_tracking; - } --#endif // SUPPORT_PROCFS - --static void -+/*! -+ * \internal -+ * \brief Initial one-off check of the pre-existing "child" processes -+ * -+ * With "child" process, we mean the subdaemon that defines an API end-point -+ * (all of them do as of the comment) -- the possible complement is skipped -+ * as it is deemed it has no such shared resources to cause conflicts about, -+ * hence it can presumably be started anew without hesitation. -+ * If that won't hold true in the future, the concept of a shared resource -+ * will have to be generalized beyond the API end-point. -+ * -+ * For boundary cases that the "child" is still starting (IPC end-point is yet -+ * to be witnessed), or more rarely (practically FreeBSD only), when there's -+ * a pre-existing "untrackable" authentic process, we give the situation some -+ * time to possibly unfold in the right direction, meaning that said socket -+ * will appear or the unattainable process will disappear per the observable -+ * IPC, respectively. -+ * -+ * \return 0 if no such "child" process found, positive number X when X -+ * "children" detected, -1 on an internal error, -2 when any -+ * would-be-used IPC is blocked with an unauthorized process -+ * -+ * \note Since this gets run at the very start, \c respawn_count fields -+ * for particular children get temporarily overloaded with "rounds -+ * of waiting" tracking, restored once we are about to finish with -+ * success (i.e. returning value >=0) and will remain unrestored -+ * otherwise. One way to suppress liveness detection logic for -+ * particular child is to set the said value to a negative number. -+ */ -+#define WAIT_TRIES 4 /* together with interleaved sleeps, worst case ~ 1s */ -+static int - find_and_track_existing_processes(void) - { --#if SUPPORT_PROCFS -- DIR *dp; -- struct dirent *entry; -- bool start_tracker = FALSE; -- char entry_name[16]; -- -- dp = opendir("/proc"); -- if (!dp) { -- /* no proc directory to search through */ -- crm_notice("Can not read /proc directory to track existing components"); -- return; -- } -- -- while ((entry = readdir(dp)) != NULL) { -- int pid; -- int max = SIZEOF(pcmk_children); -- int i; -- -- if (crm_procfs_process_info(entry, entry_name, &pid) < 0) { -- continue; -- } -- for (i = 0; i < max; i++) { -- if ((pcmk_children[i].start_seq != 0) -- && !strncmp(entry_name, pcmk_children[i].name, 15) -- && (crm_pid_active(pid, NULL) == 1)) { -- -- crm_notice("Tracking existing %s process (pid=%d)", -- pcmk_children[i].name, pid); -- pcmk_children[i].pid = pid; -- pcmk_children[i].active_before_startup = TRUE; -- start_tracker = TRUE; -- break; -+ unsigned tracking = 0U; -+ bool wait_in_progress; -+ int cur; -+ size_t i, rounds; -+ -+ for (rounds = 1; rounds <= WAIT_TRIES; rounds++) { -+ wait_in_progress = false; -+ for (i = 0; i < SIZEOF(pcmk_children); i++) { -+ if (!pcmk_children[i].endpoint -+ || pcmk_children[i].respawn_count < 0 -+ || !(cur = pcmk_child_active(&pcmk_children[i]))) { -+ /* as a speculation, don't give up in the context of -+ pcmk_child_active check if there are more rounds to -+ come for other reasons, but don't artificially wait just -+ because of this, since we would preferably start ASAP */ -+ continue; -+ } -+ pcmk_children[i].respawn_count = rounds; -+ switch (cur) { -+ case 1: -+ if (pcmk_children[i].pid == PCMK__SPECIAL_PID) { -+ if (crm_is_true(getenv("PCMK_fail_fast"))) { -+ crm_crit("Cannot reliably track pre-existing" -+ " authentic process behind %s IPC on this" -+ " platform and PCMK_fail_fast requested", -+ pcmk_children[i].endpoint); -+ return -1; -+ } else if (pcmk_children[i].respawn_count == WAIT_TRIES) { -+ crm_notice("Assuming pre-existing authentic, though" -+ " on this platform untrackable, process" -+ " behind %s IPC is stable (was in %d" -+ " previous samples) so rather than" -+ " bailing out (PCMK_fail_fast not" -+ " requested), we just switch to a less" -+ " optimal IPC liveness monitoring" -+ " (not very suitable for heavy load)", -+ pcmk_children[i].name, WAIT_TRIES - 1); -+ crm_warn("The process behind %s IPC cannot be" -+ " terminated, so the overall shutdown" -+ " will get delayed implicitly (%ld s)," -+ " which serves as a graceful period for" -+ " its native termination if it vitally" -+ " depends on some other daemons going" -+ " down in a controlled way already", -+ pcmk_children[i].name, -+ (long) SHUTDOWN_ESCALATION_PERIOD); -+ } else { -+ wait_in_progress = true; -+ crm_warn("Cannot reliably track pre-existing" -+ " authentic process behind %s IPC on this" -+ " platform, can still disappear in %d" -+ " attempt(s)", pcmk_children[i].endpoint, -+ WAIT_TRIES - pcmk_children[i].respawn_count); -+ continue; -+ } -+ } -+ crm_notice("Tracking existing %s process (pid=%lld)", -+ pcmk_children[i].name, -+ (long long) PCMK__SPECIAL_PID_AS_0( -+ pcmk_children[i].pid)); -+ pcmk_children[i].respawn_count = -1; /* 0~keep watching */ -+ pcmk_children[i].active_before_startup = TRUE; -+ tracking++; -+ break; -+ case 2: -+ if (pcmk_children[i].respawn_count == WAIT_TRIES) { -+ crm_crit("%s IPC end-point for existing authentic" -+ " process %lld did not (re)appear", -+ pcmk_children[i].endpoint, -+ (long long) PCMK__SPECIAL_PID_AS_0( -+ pcmk_children[i].pid)); -+ return -1; -+ } -+ wait_in_progress = true; -+ crm_warn("Cannot find %s IPC end-point for existing" -+ " authentic process %lld, can still (re)appear" -+ " in %d attempts (?)", -+ pcmk_children[i].endpoint, -+ (long long) PCMK__SPECIAL_PID_AS_0( -+ pcmk_children[i].pid), -+ WAIT_TRIES - pcmk_children[i].respawn_count); -+ continue; -+ case -1: -+ case -2: -+ return cur; /* messages already emitted */ -+ default: -+ crm_crit("Unexpected condition"CRM_XS"cur=%d", cur); -+ return -1; /* unexpected condition */ - } - } -+ if (!wait_in_progress) { -+ break; -+ } -+ (void) poll(NULL, 0, 250); /* a bit for changes to possibly happen */ -+ } -+ for (i = 0; i < SIZEOF(pcmk_children); i++) { -+ pcmk_children[i].respawn_count = 0; /* restore pristine state */ - } - -- if (start_tracker) { -- g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL, check_active_before_startup_processes, -- NULL); -+ if (tracking) { -+ g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL, -+ check_active_before_startup_processes, NULL); - } -- closedir(dp); --#else -- crm_notice("No procfs support, so skipping check for existing components"); --#endif // SUPPORT_PROCFS -+ return (tracking > INT_MAX) ? INT_MAX : tracking; - } - - static void -@@ -1091,7 +1366,16 @@ main(int argc, char **argv) - setenv("PCMK_watchdog", "false", 1); - } - -- find_and_track_existing_processes(); -+ switch (find_and_track_existing_processes()) { -+ case -1: -+ crm_crit("Internal fatality, see the log"); -+ crm_exit(CRM_EX_FATAL); -+ case -2: -+ crm_crit("Blocked by foreign process, kill the offender"); -+ crm_exit(CRM_EX_CANTCREAT); -+ default: -+ break; -+ }; - - cluster.destroy = mcp_cpg_destroy; - cluster.cpg.cpg_deliver_fn = mcp_cpg_deliver; --- -1.8.3.1 - - -From 052e6045eea77685aabeed12c519c7c9eb9b5287 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Tue, 16 Apr 2019 00:13:31 +0200 -Subject: [PATCH 5/7] High: pacemakerd vs. IPC/procfs confused deputy - authenticity issue (3/4) - -[3/4: other daemons to authenticate IPC servers of fellow processes] - -Now that CVE-2018-16877 issue alone is still only partially covered -based on the preceding commits in the set, put the server-by-client -authentication (enabled and 1/3 and partially sported in 2/3) into -practice widely amongst the communicating pacemaker child daemons and -towards CPG API provided by 3rd party but principally using the same -underlying IPC mechanism facilitated by libqb, and consequently close -the remaining "big gap". - -As a small justification to introducing yet another "return -value" int variable, type-correctness is restored for those -that shall be cs_error_t to begin with. ---- - daemons/pacemakerd/pcmkd_corosync.c | 61 +++++++++++- - lib/cluster/corosync.c | 178 ++++++++++++++++++++++++++++++------ - lib/cluster/cpg.c | 81 +++++++++++++--- - lib/common/ipc.c | 43 ++++++++- - 4 files changed, 317 insertions(+), 46 deletions(-) - -diff --git a/daemons/pacemakerd/pcmkd_corosync.c b/daemons/pacemakerd/pcmkd_corosync.c -index c73a1f5..65595fa 100644 ---- a/daemons/pacemakerd/pcmkd_corosync.c -+++ b/daemons/pacemakerd/pcmkd_corosync.c -@@ -1,5 +1,7 @@ - /* -- * Copyright 2010-2018 Andrew Beekhof -+ * Copyright 2010-2019 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. - * - * This source code is licensed under the GNU General Public License version 2 - * or later (GPLv2+) WITHOUT ANY WARRANTY. -@@ -21,8 +23,11 @@ - #include - - #include -+#include /* for crm_ipc_is_authentic_process */ - #include - -+#include /* PCMK__SPECIAL_PID* */ -+ - enum cluster_type_e stack = pcmk_cluster_unknown; - static corosync_cfg_handle_t cfg_handle; - -@@ -91,7 +96,10 @@ gboolean - cluster_connect_cfg(uint32_t * nodeid) - { - cs_error_t rc; -- int fd = 0, retries = 0; -+ int fd = -1, retries = 0, rv; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; - - static struct mainloop_fd_callbacks cfg_fd_callbacks = { - .dispatch = pcmk_cfg_dispatch, -@@ -101,13 +109,27 @@ cluster_connect_cfg(uint32_t * nodeid) - cs_repeat(retries, 30, rc = corosync_cfg_initialize(&cfg_handle, &cfg_callbacks)); - - if (rc != CS_OK) { -- crm_err("corosync cfg init error %d", rc); -+ crm_err("corosync cfg init: %s (%d)", cs_strerror(rc), rc); - return FALSE; - } - - rc = corosync_cfg_fd_get(cfg_handle, &fd); - if (rc != CS_OK) { -- crm_err("corosync cfg fd_get error %d", rc); -+ crm_err("corosync cfg fd_get: %s (%d)", cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CFG provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CFG provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CFG provider: %s (%d)", -+ strerror(-rv), -rv); - goto bail; - } - -@@ -152,10 +174,15 @@ get_config_opt(uint64_t unused, cmap_handle_t object_handle, const char *key, ch - gboolean - mcp_read_config(void) - { -- int rc = CS_OK; -+ cs_error_t rc = CS_OK; - int retries = 0; - cmap_handle_t local_handle; - uint64_t config = 0; -+ int fd = -1; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - // There can be only one possibility - do { -@@ -178,6 +205,30 @@ mcp_read_config(void) - return FALSE; - } - -+ rc = cmap_fd_get(local_handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CMAP API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ cmap_finalize(local_handle); -+ return FALSE; -+ } -+ -+ /* CMAP provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CMAP provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ cmap_finalize(local_handle); -+ return FALSE; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CMAP provider: %s (%d)", -+ strerror(-rv), -rv); -+ cmap_finalize(local_handle); -+ return FALSE; -+ } -+ - stack = get_cluster_type(); - crm_info("Reading configure for stack: %s", name_for_cluster_type(stack)); - -diff --git a/lib/cluster/corosync.c b/lib/cluster/corosync.c -index fcdfc0e..648c0d5 100644 ---- a/lib/cluster/corosync.c -+++ b/lib/cluster/corosync.c -@@ -1,19 +1,10 @@ - /* -- * Copyright (C) 2004 Andrew Beekhof -+ * Copyright 2004-2019 the Pacemaker project contributors - * -- * This library is free software; you can redistribute it and/or -- * modify it under the terms of the GNU Lesser General Public -- * License as published by the Free Software Foundation; either -- * version 2.1 of the License, or (at your option) any later version. -+ * The version control history for this file may have further details. - * -- * This library is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- * Lesser General Public License for more details. -- * -- * You should have received a copy of the GNU Lesser General Public -- * License along with this library; if not, write to the Free Software -- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. - */ - - #include -@@ -43,6 +34,8 @@ - - #include - -+#include /* PCMK__SPECIAL_PID* */ -+ - quorum_handle_t pcmk_quorum_handle = 0; - - gboolean(*quorum_app_callback) (unsigned long long seq, gboolean quorate) = NULL; -@@ -68,10 +61,15 @@ char * - corosync_node_name(uint64_t /*cmap_handle_t */ cmap_handle, uint32_t nodeid) - { - int lpc = 0; -- int rc = CS_OK; -+ cs_error_t rc = CS_OK; - int retries = 0; - char *name = NULL; - cmap_handle_t local_handle = 0; -+ int fd = -1; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - if (nodeid == 0) { - nodeid = get_local_nodeid(0); -@@ -100,6 +98,27 @@ corosync_node_name(uint64_t /*cmap_handle_t */ cmap_handle, uint32_t nodeid) - - if (cmap_handle == 0) { - cmap_handle = local_handle; -+ -+ rc = cmap_fd_get(cmap_handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CMAP API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CMAP provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CMAP provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CMAP provider: %s (%d)", -+ strerror(-rv), -rv); -+ goto bail; -+ } - } - - while (name == NULL && cmap_handle != 0) { -@@ -140,6 +159,7 @@ corosync_node_name(uint64_t /*cmap_handle_t */ cmap_handle, uint32_t nodeid) - lpc++; - } - -+bail: - if(local_handle) { - cmap_finalize(local_handle); - } -@@ -251,11 +271,15 @@ gboolean - cluster_connect_quorum(gboolean(*dispatch) (unsigned long long, gboolean), - void (*destroy) (gpointer)) - { -- int rc = -1; -+ cs_error_t rc; - int fd = 0; - int quorate = 0; - uint32_t quorum_type = 0; - struct mainloop_fd_callbacks quorum_fd_callbacks; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - quorum_fd_callbacks.dispatch = pcmk_quorum_dispatch; - quorum_fd_callbacks.destroy = destroy; -@@ -264,7 +288,8 @@ cluster_connect_quorum(gboolean(*dispatch) (unsigned long long, gboolean), - - rc = quorum_initialize(&pcmk_quorum_handle, &quorum_callbacks, &quorum_type); - if (rc != CS_OK) { -- crm_err("Could not connect to the Quorum API: %d", rc); -+ crm_err("Could not connect to the Quorum API: %s (%d)", -+ cs_strerror(rc), rc); - goto bail; - - } else if (quorum_type != QUORUM_SET) { -@@ -272,6 +297,29 @@ cluster_connect_quorum(gboolean(*dispatch) (unsigned long long, gboolean), - goto bail; - } - -+ rc = quorum_fd_get(pcmk_quorum_handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the Quorum API connection: %s (%d)", -+ strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* Quorum provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("Quorum provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ rc = CS_ERR_ACCESS; -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of Quorum provider: %s (%d)", -+ strerror(-rv), -rv); -+ rc = CS_ERR_ACCESS; -+ goto bail; -+ } -+ - rc = quorum_getquorate(pcmk_quorum_handle, &quorate); - if (rc != CS_OK) { - crm_err("Could not obtain the current Quorum API state: %d", rc); -@@ -292,12 +340,6 @@ cluster_connect_quorum(gboolean(*dispatch) (unsigned long long, gboolean), - goto bail; - } - -- rc = quorum_fd_get(pcmk_quorum_handle, &fd); -- if (rc != CS_OK) { -- crm_err("Could not obtain the Quorum API connection: %d", rc); -- goto bail; -- } -- - mainloop_add_fd("quorum", G_PRIORITY_HIGH, fd, dispatch, &quorum_fd_callbacks); - - corosync_initialize_nodelist(NULL, FALSE, NULL); -@@ -488,10 +530,15 @@ gboolean - corosync_initialize_nodelist(void *cluster, gboolean force_member, xmlNode * xml_parent) - { - int lpc = 0; -- int rc = CS_OK; -+ cs_error_t rc = CS_OK; - int retries = 0; - gboolean any = FALSE; - cmap_handle_t cmap_handle; -+ int fd = -1; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - do { - rc = cmap_initialize(&cmap_handle); -@@ -509,6 +556,27 @@ corosync_initialize_nodelist(void *cluster, gboolean force_member, xmlNode * xml - return FALSE; - } - -+ rc = cmap_fd_get(cmap_handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CMAP API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CMAP provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CMAP provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CMAP provider: %s (%d)", -+ strerror(-rv), -rv); -+ goto bail; -+ } -+ - crm_peer_init(); - crm_trace("Initializing corosync nodelist"); - for (lpc = 0; TRUE; lpc++) { -@@ -562,6 +630,7 @@ corosync_initialize_nodelist(void *cluster, gboolean force_member, xmlNode * xml - - free(name); - } -+bail: - cmap_finalize(cmap_handle); - return any; - } -@@ -571,36 +640,68 @@ corosync_cluster_name(void) - { - cmap_handle_t handle; - char *cluster_name = NULL; -- int rc = CS_OK; -+ cs_error_t rc = CS_OK; -+ int fd = -1; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - rc = cmap_initialize(&handle); - if (rc != CS_OK) { -- crm_info("Failed to initialize the cmap API: %s (%d)", ais_error2text(rc), rc); -+ crm_info("Failed to initialize the cmap API: %s (%d)", -+ cs_strerror(rc), rc); - return NULL; - } - -+ rc = cmap_fd_get(handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CMAP API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CMAP provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CMAP provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CMAP provider: %s (%d)", -+ strerror(-rv), -rv); -+ goto bail; -+ } -+ - rc = cmap_get_string(handle, "totem.cluster_name", &cluster_name); - if (rc != CS_OK) { -- crm_info("Cannot get totem.cluster_name: %s (%d)", ais_error2text(rc), rc); -+ crm_info("Cannot get totem.cluster_name: %s (%d)", cs_strerror(rc), rc); - - } else { - crm_debug("cmap totem.cluster_name = '%s'", cluster_name); - } - -+bail: - cmap_finalize(handle); -- - return cluster_name; - } - - int - corosync_cmap_has_config(const char *prefix) - { -- int rc = CS_OK; -+ cs_error_t rc = CS_OK; - int retries = 0; - static int found = -1; - cmap_handle_t cmap_handle; - cmap_iter_handle_t iter_handle; - char key_name[CMAP_KEYNAME_MAXLEN + 1]; -+ int fd = -1; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - if(found != -1) { - return found; -@@ -623,6 +724,27 @@ corosync_cmap_has_config(const char *prefix) - return -1; - } - -+ rc = cmap_fd_get(cmap_handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CMAP API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CMAP provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CMAP provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CMAP provider: %s (%d)", -+ strerror(-rv), -rv); -+ goto bail; -+ } -+ - rc = cmap_iter_init(cmap_handle, prefix, &iter_handle); - if (rc != CS_OK) { - crm_warn("Failed to initialize iteration for corosync cmap '%s': %s (rc=%d)", -diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c -index 75af131..430195c 100644 ---- a/lib/cluster/cpg.c -+++ b/lib/cluster/cpg.c -@@ -1,5 +1,7 @@ - /* -- * Copyright 2004-2018 Andrew Beekhof -+ * Copyright 2004-2019 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. - * - * This source code is licensed under the GNU Lesser General Public License - * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -@@ -27,6 +29,8 @@ - - #include - -+#include /* PCMK__SPECIAL_PID* */ -+ - cpg_handle_t pcmk_cpg_handle = 0; /* TODO: Remove, use cluster.cpg_handle */ - - static bool cpg_evicted = FALSE; -@@ -60,11 +64,16 @@ cluster_disconnect_cpg(crm_cluster_t *cluster) - - uint32_t get_local_nodeid(cpg_handle_t handle) - { -- int rc = CS_OK; -+ cs_error_t rc = CS_OK; - int retries = 0; - static uint32_t local_nodeid = 0; - cpg_handle_t local_handle = handle; - cpg_callbacks_t cb = { }; -+ int fd = -1; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - if(local_nodeid != 0) { - return local_nodeid; -@@ -73,6 +82,32 @@ uint32_t get_local_nodeid(cpg_handle_t handle) - if(handle == 0) { - crm_trace("Creating connection"); - cs_repeat(retries, 5, rc = cpg_initialize(&local_handle, &cb)); -+ if (rc != CS_OK) { -+ crm_err("Could not connect to the CPG API: %s (%d)", -+ cs_strerror(rc), rc); -+ return 0; -+ } -+ -+ rc = cpg_fd_get(local_handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CPG API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CPG provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CPG provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CPG provider: %s (%d)", -+ strerror(-rv), -rv); -+ goto bail; -+ } - } - - if (rc == CS_OK) { -@@ -84,6 +119,8 @@ uint32_t get_local_nodeid(cpg_handle_t handle) - if (rc != CS_OK) { - crm_err("Could not get local node id from the CPG API: %s (%d)", ais_error2text(rc), rc); - } -+ -+bail: - if(handle == 0) { - crm_trace("Closing connection"); - cpg_finalize(local_handle); -@@ -403,13 +440,17 @@ pcmk_cpg_membership(cpg_handle_t handle, - gboolean - cluster_connect_cpg(crm_cluster_t *cluster) - { -- int rc = -1; -- int fd = 0; -+ cs_error_t rc; -+ int fd = -1; - int retries = 0; - uint32_t id = 0; - crm_node_t *peer = NULL; - cpg_handle_t handle = 0; - const char *message_name = pcmk_message_name(crm_system_name); -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - struct mainloop_fd_callbacks cpg_fd_callbacks = { - .dispatch = pcmk_cpg_dispatch, -@@ -434,7 +475,31 @@ cluster_connect_cpg(crm_cluster_t *cluster) - - cs_repeat(retries, 30, rc = cpg_initialize(&handle, &cpg_callbacks)); - if (rc != CS_OK) { -- crm_err("Could not connect to the Cluster Process Group API: %d", rc); -+ crm_err("Could not connect to the CPG API: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ rc = cpg_fd_get(handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CPG API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CPG provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CPG provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ rc = CS_ERR_ACCESS; -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CPG provider: %s (%d)", -+ strerror(-rv), -rv); -+ rc = CS_ERR_ACCESS; - goto bail; - } - -@@ -453,12 +518,6 @@ cluster_connect_cpg(crm_cluster_t *cluster) - goto bail; - } - -- rc = cpg_fd_get(handle, &fd); -- if (rc != CS_OK) { -- crm_err("Could not obtain the CPG API connection: %d", rc); -- goto bail; -- } -- - pcmk_cpg_handle = handle; - cluster->cpg_handle = handle; - mainloop_add_fd("corosync-cpg", G_PRIORITY_MEDIUM, fd, cluster, &cpg_fd_callbacks); -diff --git a/lib/common/ipc.c b/lib/common/ipc.c -index aa055d3..84d7cb9 100644 ---- a/lib/common/ipc.c -+++ b/lib/common/ipc.c -@@ -947,11 +947,18 @@ crm_ipc_new(const char *name, size_t max_size) - * - * \param[in] client Connection instance obtained from crm_ipc_new() - * -- * \return TRUE on success, FALSE otherwise (in which case errno will be set) -+ * \return TRUE on success, FALSE otherwise (in which case errno will be set; -+ * specifically, in case of discovering the remote side is not -+ * authentic, its value is set to ECONNABORTED). - */ - bool - crm_ipc_connect(crm_ipc_t * client) - { -+ static uid_t cl_uid = 0; -+ static gid_t cl_gid = 0; -+ pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; -+ int rv; -+ - client->need_reply = FALSE; - client->ipc = qb_ipcc_connect(client->name, client->buf_size); - -@@ -962,7 +969,39 @@ crm_ipc_connect(crm_ipc_t * client) - - client->pfd.fd = crm_ipc_get_fd(client); - if (client->pfd.fd < 0) { -- crm_debug("Could not obtain file descriptor for %s connection: %s (%d)", client->name, pcmk_strerror(errno), errno); -+ rv = errno; -+ /* message already omitted */ -+ crm_ipc_close(client); -+ errno = rv; -+ return FALSE; -+ } -+ -+ if (!cl_uid && !cl_gid -+ && (rv = crm_user_lookup(CRM_DAEMON_USER, &cl_uid, &cl_gid)) < 0) { -+ errno = -rv; -+ /* message already omitted */ -+ crm_ipc_close(client); -+ errno = -rv; -+ return FALSE; -+ } -+ -+ if (!(rv = crm_ipc_is_authentic_process(client->pfd.fd, cl_uid, cl_gid, -+ &found_pid, &found_uid, -+ &found_gid))) { -+ crm_err("Daemon (IPC %s) is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ client->name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ crm_ipc_close(client); -+ errno = ECONNABORTED; -+ return FALSE; -+ -+ } else if (rv < 0) { -+ errno = -rv; -+ crm_perror(LOG_ERR, "Could not verify authenticity of daemon (IPC %s)", -+ client->name); -+ crm_ipc_close(client); -+ errno = -rv; - return FALSE; - } - --- -1.8.3.1 - - -From d324e407c0e2695f405974d567d79eb91d0ee69a Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Tue, 16 Apr 2019 00:13:40 +0200 -Subject: [PATCH 6/7] High: pacemakerd vs. IPC/procfs confused deputy - authenticity issue (4/4) - -[4/4: CPG users to be careful about now-more-probable rival processes] - -In essence, this comes down to pacemaker confusing at-node CPG members -with effectively the only plausible to co-exist at particular node, -which doesn't hold and asks for a wider reconciliation of this -reality-check. - -However, in practical terms, since there are two factors lowering the -priority of doing so: - -1/ possibly the only non-self-inflicted scenario is either that - some of the cluster stack processes fail -- this the problem - that shall rather be deferred to arranged node disarming/fencing - to stay on the safe side with 100% certainty, at the cost of - possibly long-lasting failover process at other nodes - (for other possibility, someone running some of these by accident - so they effectively become rival processes, it's like getting - hands cut when playing with a lawnmower in an unintended way) - -2/ for state tracking of the peer nodes, it may possibly cause troubles - in case the process observed as left wasn't the last for the - particular node, even if presumably just temporary, since the - situation may eventually resolve with imposed serialization of - the rival processes via API end-point singleton restriction (this - is also the most likely cause of why such non-final leave gets - observed in the first place), except in one case -- the legitimate - API end-point carrier won't possibly acknowledged as returned - by its peers, at least not immediately, unless it tries to join - anew, which verges on undefined behaviour (at least per corosync - documentation) - -we make do just with a light code change so as to - -* limit 1/ some more with in-daemon self-check for pre-existing - end-point existence (this is to complement the checks already made in - the parent daemon prior to spawning new instances, only some moments - later; note that we don't have any lock file etc. mechanisms to - prevent parallel runs of the same daemons, and people could run these - on their own deliberation), and to - -* guard against the interferences from the rivals at the same node - per 2/ with ignoring their non-final leave messages altogether. - -Note that CPG at this point is already expected to be authenticity-safe. - -Regarding now-more-probable part, we actually traded the inherently racy -procfs scanning for something (exactly that singleton mentioned above) -rather firm (and unfakeable), but we admittedly got lost track of -processes that are after CPG membership (that is, another form of -a shared state) prior to (or in non-deterministic order allowing for -the same) carring about publishing the end-point. - -Big thanks is owed to Yan Gao of SUSE, for early discovery and reporting -this discrepancy arising from the earlier commits in the set. ---- - daemons/attrd/pacemaker-attrd.c | 15 ++++++ - daemons/based/pacemaker-based.c | 18 ++++++- - daemons/controld/pacemaker-controld.c | 18 ++++++- - daemons/fenced/pacemaker-fenced.c | 15 ++++++ - lib/cluster/cpg.c | 91 ++++++++++++++++++++++++++++++++--- - 5 files changed, 146 insertions(+), 11 deletions(-) - -diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c -index 837459f..9ffe201 100644 ---- a/daemons/attrd/pacemaker-attrd.c -+++ b/daemons/attrd/pacemaker-attrd.c -@@ -344,6 +344,7 @@ main(int argc, char **argv) - int flag = 0; - int index = 0; - int argerr = 0; -+ crm_ipc_t *old_instance = NULL; - - attrd_init_mainloop(); - crm_log_preinit(NULL, argc, argv); -@@ -380,6 +381,20 @@ main(int argc, char **argv) - - crm_log_init(T_ATTRD, LOG_INFO, TRUE, FALSE, argc, argv, FALSE); - crm_info("Starting up"); -+ -+ old_instance = crm_ipc_new(T_ATTRD, 0); -+ if (crm_ipc_connect(old_instance)) { -+ /* IPC end-point already up */ -+ crm_ipc_close(old_instance); -+ crm_ipc_destroy(old_instance); -+ crm_err("pacemaker-attrdd is already active, aborting startup"); -+ crm_exit(CRM_EX_OK); -+ } else { -+ /* not up or not authentic, we'll proceed either way */ -+ crm_ipc_destroy(old_instance); -+ old_instance = NULL; -+ } -+ - attributes = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_attribute); - - /* Connect to the CIB before connecting to the cluster or listening for IPC. -diff --git a/daemons/based/pacemaker-based.c b/daemons/based/pacemaker-based.c -index bd98ca8..f004f671 100644 ---- a/daemons/based/pacemaker-based.c -+++ b/daemons/based/pacemaker-based.c -@@ -90,13 +90,12 @@ main(int argc, char **argv) - int index = 0; - int argerr = 0; - struct passwd *pwentry = NULL; -+ crm_ipc_t *old_instance = NULL; - - crm_log_preinit(NULL, argc, argv); - crm_set_options(NULL, "[options]", - long_options, "Daemon for storing and replicating the cluster configuration"); - -- crm_peer_init(); -- - mainloop_add_signal(SIGTERM, cib_shutdown); - mainloop_add_signal(SIGPIPE, cib_enable_writes); - -@@ -171,6 +170,19 @@ main(int argc, char **argv) - - crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE); - -+ old_instance = crm_ipc_new(CIB_CHANNEL_RO, 0); -+ if (crm_ipc_connect(old_instance)) { -+ /* IPC end-point already up */ -+ crm_ipc_close(old_instance); -+ crm_ipc_destroy(old_instance); -+ crm_err("pacemaker-based is already active, aborting startup"); -+ crm_exit(CRM_EX_OK); -+ } else { -+ /* not up or not authentic, we'll proceed either way */ -+ crm_ipc_destroy(old_instance); -+ old_instance = NULL; -+ } -+ - if (cib_root == NULL) { - cib_root = CRM_CONFIG_DIR; - } else { -@@ -185,6 +197,8 @@ main(int argc, char **argv) - return CRM_EX_FATAL; - } - -+ crm_peer_init(); -+ - /* read local config file */ - cib_init(); - -diff --git a/daemons/controld/pacemaker-controld.c b/daemons/controld/pacemaker-controld.c -index 11f909e..38842fb 100644 ---- a/daemons/controld/pacemaker-controld.c -+++ b/daemons/controld/pacemaker-controld.c -@@ -1,5 +1,7 @@ - /* -- * Copyright 2004-2018 Andrew Beekhof -+ * Copyright 2004-2019 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. - * - * This source code is licensed under the GNU General Public License version 2 - * or later (GPLv2+) WITHOUT ANY WARRANTY. -@@ -50,6 +52,7 @@ main(int argc, char **argv) - int flag; - int index = 0; - int argerr = 0; -+ crm_ipc_t *old_instance = NULL; - - crmd_mainloop = g_main_loop_new(NULL, FALSE); - crm_log_preinit(NULL, argc, argv); -@@ -93,6 +96,19 @@ main(int argc, char **argv) - crm_help('?', CRM_EX_USAGE); - } - -+ old_instance = crm_ipc_new(CRM_SYSTEM_CRMD, 0); -+ if (crm_ipc_connect(old_instance)) { -+ /* IPC end-point already up */ -+ crm_ipc_close(old_instance); -+ crm_ipc_destroy(old_instance); -+ crm_err("pacemaker-controld is already active, aborting startup"); -+ crm_exit(CRM_EX_OK); -+ } else { -+ /* not up or not authentic, we'll proceed either way */ -+ crm_ipc_destroy(old_instance); -+ old_instance = NULL; -+ } -+ - if (pcmk__daemon_can_write(PE_STATE_DIR, NULL) == FALSE) { - crm_err("Terminating due to bad permissions on " PE_STATE_DIR); - fprintf(stderr, -diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c -index 105012b..da0dce9 100644 ---- a/daemons/fenced/pacemaker-fenced.c -+++ b/daemons/fenced/pacemaker-fenced.c -@@ -1262,6 +1262,7 @@ main(int argc, char **argv) - int option_index = 0; - crm_cluster_t cluster; - const char *actions[] = { "reboot", "off", "on", "list", "monitor", "status" }; -+ crm_ipc_t *old_instance = NULL; - - crm_log_preinit(NULL, argc, argv); - crm_set_options(NULL, "mode [options]", long_options, -@@ -1432,6 +1433,20 @@ main(int argc, char **argv) - } - - crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE); -+ -+ old_instance = crm_ipc_new("stonith-ng", 0); -+ if (crm_ipc_connect(old_instance)) { -+ /* IPC end-point already up */ -+ crm_ipc_close(old_instance); -+ crm_ipc_destroy(old_instance); -+ crm_err("pacemaker-fenced is already active, aborting startup"); -+ crm_exit(CRM_EX_OK); -+ } else { -+ /* not up or not authentic, we'll proceed either way */ -+ crm_ipc_destroy(old_instance); -+ old_instance = NULL; -+ } -+ - mainloop_add_signal(SIGTERM, stonith_shutdown); - - crm_peer_init(); -diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c -index 430195c..2898c51 100644 ---- a/lib/cluster/cpg.c -+++ b/lib/cluster/cpg.c -@@ -362,6 +362,20 @@ pcmk_message_common_cs(cpg_handle_t handle, uint32_t nodeid, uint32_t pid, void - - #define PEER_NAME(peer) ((peer)? ((peer)->uname? (peer)->uname : "") : "") - -+static int cmp_member_list_nodeid(const void *first, -+ const void *second) -+{ -+ const struct cpg_address *const a = *((const struct cpg_address **) first), -+ *const b = *((const struct cpg_address **) second); -+ if (a->nodeid < b->nodeid) { -+ return -1; -+ } else if (a->nodeid > b->nodeid) { -+ return 1; -+ } -+ /* don't bother with "reason" nor "pid" */ -+ return 0; -+} -+ - void - pcmk_cpg_membership(cpg_handle_t handle, - const struct cpg_name *groupName, -@@ -373,29 +387,89 @@ pcmk_cpg_membership(cpg_handle_t handle, - gboolean found = FALSE; - static int counter = 0; - uint32_t local_nodeid = get_local_nodeid(handle); -+ const struct cpg_address *key, **rival, **sorted; -+ -+ sorted = malloc(member_list_entries * sizeof(const struct cpg_address *)); -+ CRM_ASSERT(sorted != NULL); -+ -+ for (size_t iter = 0; iter < member_list_entries; iter++) { -+ sorted[iter] = member_list + iter; -+ } -+ /* so that the cross-matching multiply-subscribed nodes is then cheap */ -+ qsort(sorted, member_list_entries, sizeof(const struct cpg_address *), -+ cmp_member_list_nodeid); - - for (i = 0; i < left_list_entries; i++) { - crm_node_t *peer = crm_find_peer(left_list[i].nodeid, NULL); - -- crm_info("Group event %s.%d: node %u (%s) left", -+ crm_info("Group event %s.%d: node %u (%s) left: %llu", - groupName->value, counter, left_list[i].nodeid, -- PEER_NAME(peer)); -+ PEER_NAME(peer), (unsigned long long) left_list[i].pid); -+ -+ /* in CPG world, NODE:PROCESS-IN-MEMBERSHIP-OF-G is an 1:N relation -+ and not playing by this rule may go wild in case of multiple -+ residual instances of the same pacemaker daemon at the same node -+ -- we must ensure that the possible local rival(s) won't make us -+ cry out and bail (e.g. when they quit themselves), since all the -+ surrounding logic denies this simple fact that the full membership -+ is discriminated also per the PID of the process beside mere node -+ ID (and implicitly, group ID); practically, this will be sound in -+ terms of not preventing progress, since all the CPG joiners are -+ also API end-point carriers, and that's what matters locally -+ (who's the winner); -+ remotely, we will just compare leave_list and member_list and if -+ the left process has it's node retained in member_list (under some -+ other PID, anyway) we will just ignore it as well -+ XXX: long-term fix is to establish in-out PID-aware tracking? */ - if (peer) { -- crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, OFFLINESTATUS); -+ key = &left_list[i]; -+ rival = bsearch(&key, sorted, member_list_entries, -+ sizeof(const struct cpg_address *), -+ cmp_member_list_nodeid); -+ if (rival == NULL) { -+ crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, -+ OFFLINESTATUS); -+ } else if (left_list[i].nodeid == local_nodeid) { -+ crm_info("Ignoring the above event %s.%d, comes from a local" -+ " rival process (presumably not us): %llu", -+ groupName->value, counter, -+ (unsigned long long) left_list[i].pid); -+ } else { -+ crm_info("Ignoring the above event %s.%d, comes from" -+ " a rival-rich node: %llu (e.g. %llu process" -+ " carries on)", -+ groupName->value, counter, -+ (unsigned long long) left_list[i].pid, -+ (unsigned long long) (*rival)->pid); -+ } - } - } -+ free(sorted); -+ sorted = NULL; - - for (i = 0; i < joined_list_entries; i++) { -- crm_info("Group event %s.%d: node %u joined", -- groupName->value, counter, joined_list[i].nodeid); -+ crm_info("Group event %s.%d: node %u joined: %llu" -+ " (unchecked for rivals)", -+ groupName->value, counter, joined_list[i].nodeid, -+ (unsigned long long) joined_list[i].pid); - } - - for (i = 0; i < member_list_entries; i++) { - crm_node_t *peer = crm_get_peer(member_list[i].nodeid, NULL); - -- crm_info("Group event %s.%d: node %u (%s) is member", -+ crm_info("Group event %s.%d: node %u (%s) is member: %llu" -+ " (at least once)", - groupName->value, counter, member_list[i].nodeid, -- PEER_NAME(peer)); -+ PEER_NAME(peer), member_list[i].pid); -+ -+ if (member_list[i].nodeid == local_nodeid -+ && member_list[i].pid != getpid()) { -+ /* see the note above */ -+ crm_info("Ignoring the above event %s.%d, comes from a local rival" -+ " process: %llu", groupName->value, counter, -+ (unsigned long long) member_list[i].pid); -+ continue; -+ } - - /* If the caller left auto-reaping enabled, this will also update the - * state to member. -@@ -418,7 +492,8 @@ pcmk_cpg_membership(cpg_handle_t handle, - - } else if (now > (peer->when_lost + 60)) { - // If it persists for more than a minute, update the state -- crm_warn("Node %u member of group %s but believed offline", -+ crm_warn("Node %u member of group %s but believed offline" -+ " (unchecked for rivals)", - member_list[i].nodeid, groupName->value); - crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_MEMBER, 0); - } --- -1.8.3.1 - - -From 3ad7b2509d78f95b5dfc8fffc4d9a91be1da5113 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Wed, 17 Apr 2019 15:04:21 +0200 -Subject: [PATCH 7/7] Med: controld: fix possible NULL pointer dereference - -This is now more likely triggerable once the problems related to -CVE-2018-16878 are avoided. ---- - daemons/controld/controld_control.c | 15 +++++++++------ - 1 file changed, 9 insertions(+), 6 deletions(-) - -diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c -index ee95698..0ac358c 100644 ---- a/daemons/controld/controld_control.c -+++ b/daemons/controld/controld_control.c -@@ -77,12 +77,15 @@ do_ha_control(long long action, - registered = crm_connect_corosync(cluster); - #endif - } -- controld_election_init(cluster->uname); -- fsa_our_uname = cluster->uname; -- fsa_our_uuid = cluster->uuid; -- if(cluster->uuid == NULL) { -- crm_err("Could not obtain local uuid"); -- registered = FALSE; -+ -+ if (registered == TRUE) { -+ controld_election_init(cluster->uname); -+ fsa_our_uname = cluster->uname; -+ fsa_our_uuid = cluster->uuid; -+ if(cluster->uuid == NULL) { -+ crm_err("Could not obtain local uuid"); -+ registered = FALSE; -+ } - } - - if (registered == FALSE) { --- -1.8.3.1 - diff --git a/SOURCES/003-fencer-logs.patch b/SOURCES/003-fencer-logs.patch new file mode 100644 index 0000000..072b4cd --- /dev/null +++ b/SOURCES/003-fencer-logs.patch @@ -0,0 +1,652 @@ +From 0a884f325e1049febc28bf0419ab307dd0bce5af Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 16 May 2019 20:04:57 -0500 +Subject: [PATCH] Log: various: improve fencer connection messages + +Previously, log messages around fencer connections were inconsistent. + +This attempts to make them more consistent by: having stonith_api_signon() log +only at debug level, letting the callers log at a level appropriate to the +situation using the return code; functionizing retrying a connection; and +using similar wording across clients. + +This also does a bit of refactoring for better error checking and improved +efficiency. +--- + daemons/controld/controld_control.c | 7 +- + daemons/controld/controld_te_utils.c | 59 ++++++----- + daemons/execd/pacemaker-execd.c | 28 ++--- + daemons/fenced/cts-fence-helper.c | 38 +++---- + include/crm/stonith-ng.h | 4 + + lib/fencing/st_client.c | 195 ++++++++++++++++++++--------------- + tools/crm_mon.c | 1 - + tools/stonith_admin.c | 29 +----- + 8 files changed, 181 insertions(+), 180 deletions(-) + +diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c +index 89b5b5d..6d9f335 100644 +--- a/daemons/controld/controld_control.c ++++ b/daemons/controld/controld_control.c +@@ -628,10 +628,11 @@ do_started(long long action, + register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); + } + ++ // Try connecting to fencer (retrying later in mainloop if failed) + if (stonith_reconnect == NULL) { +- int dummy; +- +- stonith_reconnect = mainloop_add_trigger(G_PRIORITY_LOW, te_connect_stonith, &dummy); ++ stonith_reconnect = mainloop_add_trigger(G_PRIORITY_LOW, ++ te_connect_stonith, ++ GINT_TO_POINTER(TRUE)); + } + set_bit(fsa_input_register, R_ST_REQUIRED); + mainloop_set_trigger(stonith_reconnect); +diff --git a/daemons/controld/controld_te_utils.c b/daemons/controld/controld_te_utils.c +index 5606ed6..22f83ad 100644 +--- a/daemons/controld/controld_te_utils.c ++++ b/daemons/controld/controld_te_utils.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2018 Andrew Beekhof ++ * Copyright 2004-2019 the Pacemaker project contributors + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. +@@ -385,10 +385,18 @@ te_trigger_stonith_history_sync(void) + mainloop_timer_start(stonith_history_sync_timer); + } + ++/*! ++ * \brief Connect to fencer ++ * ++ * \param[in] user_data If NULL, retry failures now, otherwise retry in main loop ++ * ++ * \return TRUE ++ * \note If user_data is NULL, this will wait 2s between attempts, for up to ++ * 30 attempts, meaning the controller could be blocked as long as 58s. ++ */ + gboolean + te_connect_stonith(gpointer user_data) + { +- int lpc = 0; + int rc = pcmk_ok; + + if (stonith_api == NULL) { +@@ -396,42 +404,41 @@ te_connect_stonith(gpointer user_data) + } + + if (stonith_api->state != stonith_disconnected) { +- crm_trace("Still connected"); ++ crm_trace("Already connected to fencer, no need to retry"); + return TRUE; + } + +- for (lpc = 0; lpc < 30; lpc++) { +- crm_debug("Attempting connection to fencing daemon..."); +- +- sleep(1); +- rc = stonith_api->cmds->connect(stonith_api, crm_system_name, NULL); +- +- if (rc == pcmk_ok) { +- break; ++ if (user_data == NULL) { ++ // Blocking (retry failures now until successful) ++ rc = stonith_api_connect_retry(stonith_api, crm_system_name, 30); ++ if (rc != pcmk_ok) { ++ crm_err("Could not connect to fencer in 30 attempts: %s " ++ CRM_XS " rc=%d", pcmk_strerror(rc), rc); + } +- +- if (user_data != NULL) { ++ } else { ++ // Non-blocking (retry failures later in main loop) ++ rc = stonith_api->cmds->connect(stonith_api, crm_system_name, NULL); ++ if (rc != pcmk_ok) { + if (is_set(fsa_input_register, R_ST_REQUIRED)) { +- crm_err("Sign-in failed: triggered a retry"); ++ crm_err("Fencer connection failed (will retry): %s " ++ CRM_XS " rc=%d", pcmk_strerror(rc), rc); + mainloop_set_trigger(stonith_reconnect); + } else { +- crm_info("Sign-in failed, but no longer required"); ++ crm_info("Fencer connection failed (ignoring because no longer required): %s " ++ CRM_XS " rc=%d", pcmk_strerror(rc), rc); + } + return TRUE; + } +- +- crm_err("Sign-in failed: pausing and trying again in 2s..."); +- sleep(1); + } + +- CRM_CHECK(rc == pcmk_ok, return TRUE); /* If not, we failed 30 times... just get out */ +- stonith_api->cmds->register_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT, +- tengine_stonith_connection_destroy); +- +- stonith_api->cmds->register_notification(stonith_api, T_STONITH_NOTIFY_FENCE, +- tengine_stonith_notify); +- +- crm_trace("Connected"); ++ if (rc == pcmk_ok) { ++ stonith_api->cmds->register_notification(stonith_api, ++ T_STONITH_NOTIFY_DISCONNECT, ++ tengine_stonith_connection_destroy); ++ stonith_api->cmds->register_notification(stonith_api, ++ T_STONITH_NOTIFY_FENCE, ++ tengine_stonith_notify); ++ } + return TRUE; + } + +diff --git a/daemons/execd/pacemaker-execd.c b/daemons/execd/pacemaker-execd.c +index 21bb0ed..e2fdfca 100644 +--- a/daemons/execd/pacemaker-execd.c ++++ b/daemons/execd/pacemaker-execd.c +@@ -65,28 +65,20 @@ get_stonith_connection(void) + stonith_api = NULL; + } + +- if (!stonith_api) { +- int rc = 0; +- int tries = 10; ++ if (stonith_api == NULL) { ++ int rc = pcmk_ok; + + stonith_api = stonith_api_new(); +- do { +- rc = stonith_api->cmds->connect(stonith_api, "pacemaker-execd", NULL); +- if (rc == pcmk_ok) { +- stonith_api->cmds->register_notification(stonith_api, +- T_STONITH_NOTIFY_DISCONNECT, +- stonith_connection_destroy_cb); +- break; +- } +- sleep(1); +- tries--; +- } while (tries); +- +- if (rc) { +- crm_err("Unable to connect to stonith daemon to execute command. error: %s", +- pcmk_strerror(rc)); ++ rc = stonith_api_connect_retry(stonith_api, crm_system_name, 10); ++ if (rc != pcmk_ok) { ++ crm_err("Could not connect to fencer in 10 attempts: %s " ++ CRM_XS " rc=%d", pcmk_strerror(rc), rc); + stonith_api_delete(stonith_api); + stonith_api = NULL; ++ } else { ++ stonith_api->cmds->register_notification(stonith_api, ++ T_STONITH_NOTIFY_DISCONNECT, ++ stonith_connection_destroy_cb); + } + } + return stonith_api; +diff --git a/daemons/fenced/cts-fence-helper.c b/daemons/fenced/cts-fence-helper.c +index c5ce1ab..4552fc1 100644 +--- a/daemons/fenced/cts-fence-helper.c ++++ b/daemons/fenced/cts-fence-helper.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2009-2018 Andrew Beekhof ++ * Copyright 2009-2019 the Pacemaker project contributors + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. +@@ -124,8 +124,10 @@ passive_test(void) + int rc = 0; + + rc = st->cmds->connect(st, crm_system_name, &pollfd.fd); +- crm_debug("Connect: %d", rc); +- ++ if (rc != pcmk_ok) { ++ stonith_api_delete(st); ++ crm_exit(CRM_EX_DISCONNECT); ++ } + st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT, st_callback); + st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, st_callback); + st->cmds->register_notification(st, STONITH_OP_DEVICE_ADD, st_callback); +@@ -271,8 +273,10 @@ sanity_tests(void) + int rc = 0; + + rc = st->cmds->connect(st, crm_system_name, &pollfd.fd); +- crm_debug("Connect: %d", rc); +- ++ if (rc != pcmk_ok) { ++ stonith_api_delete(st); ++ crm_exit(CRM_EX_DISCONNECT); ++ } + st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT, st_callback); + st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, st_callback); + st->cmds->register_notification(st, STONITH_OP_DEVICE_ADD, st_callback); +@@ -295,7 +299,10 @@ standard_dev_test(void) + stonith_key_value_t *params = NULL; + + rc = st->cmds->connect(st, crm_system_name, &pollfd.fd); +- crm_debug("Connect: %d", rc); ++ if (rc != pcmk_ok) { ++ stonith_api_delete(st); ++ crm_exit(CRM_EX_DISCONNECT); ++ } + + params = stonith_key_value_add(params, "pcmk_host_map", "some-host=pcmk-7 true_1_node1=3,4"); + +@@ -502,23 +509,12 @@ test_register_async_devices(int check_event) + static void + try_mainloop_connect(int check_event) + { +- int tries = 10; +- int i = 0; +- int rc = 0; ++ int rc = stonith_api_connect_retry(st, crm_system_name, 10); + +- for (i = 0; i < tries; i++) { +- rc = st->cmds->connect(st, crm_system_name, NULL); +- +- if (!rc) { +- crm_info("stonith client connection established"); +- mainloop_test_done(TRUE); +- return; +- } else { +- crm_info("stonith client connection failed"); +- } +- sleep(1); ++ if (rc == pcmk_ok) { ++ mainloop_test_done(TRUE); ++ return; + } +- + crm_err("API CONNECTION FAILURE"); + mainloop_test_done(FALSE); + } +diff --git a/include/crm/stonith-ng.h b/include/crm/stonith-ng.h +index b7365a9..b640732 100644 +--- a/include/crm/stonith-ng.h ++++ b/include/crm/stonith-ng.h +@@ -430,6 +430,10 @@ void stonith_key_value_freeall(stonith_key_value_t * kvp, int keys, int values); + + void stonith_history_free(stonith_history_t *history); + ++// Convenience functions ++int stonith_api_connect_retry(stonith_t *st, const char *name, ++ int max_attempts); ++ + /* Basic helpers that allows nodes to be fenced and the history to be + * queried without mainloop or the caller understanding the full API + * +diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c +index 270ef8d..ceee944 100644 +--- a/lib/fencing/st_client.c ++++ b/lib/fencing/st_client.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2018 Andrew Beekhof ++ * Copyright 2004-2019 the Pacemaker project contributors + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. +@@ -1415,14 +1415,21 @@ static int + stonith_api_signon(stonith_t * stonith, const char *name, int *stonith_fd) + { + int rc = pcmk_ok; +- stonith_private_t *native = stonith->st_private; ++ stonith_private_t *native = NULL; ++ const char *display_name = name? name : "client"; + + static struct ipc_client_callbacks st_callbacks = { + .dispatch = stonith_dispatch_internal, + .destroy = stonith_connection_destroy + }; + +- crm_trace("Connecting command channel"); ++ CRM_CHECK(stonith != NULL, return -EINVAL); ++ ++ native = stonith->st_private; ++ CRM_ASSERT(native != NULL); ++ ++ crm_debug("Attempting fencer connection by %s with%s mainloop", ++ display_name, (stonith_fd? "out" : "")); + + stonith->state = stonith_connected_command; + if (stonith_fd) { +@@ -1432,8 +1439,9 @@ stonith_api_signon(stonith_t * stonith, const char *name, int *stonith_fd) + if (native->ipc && crm_ipc_connect(native->ipc)) { + *stonith_fd = crm_ipc_get_fd(native->ipc); + } else if (native->ipc) { +- crm_perror(LOG_ERR, "Connection to fencer failed"); +- rc = -ENOTCONN; ++ crm_ipc_close(native->ipc); ++ crm_ipc_destroy(native->ipc); ++ native->ipc = NULL; + } + + } else { +@@ -1444,11 +1452,8 @@ stonith_api_signon(stonith_t * stonith, const char *name, int *stonith_fd) + } + + if (native->ipc == NULL) { +- crm_debug("Could not connect to the Stonith API"); + rc = -ENOTCONN; +- } +- +- if (rc == pcmk_ok) { ++ } else { + xmlNode *reply = NULL; + xmlNode *hello = create_xml_node(NULL, "stonith_command"); + +@@ -1458,11 +1463,12 @@ stonith_api_signon(stonith_t * stonith, const char *name, int *stonith_fd) + rc = crm_ipc_send(native->ipc, hello, crm_ipc_client_response, -1, &reply); + + if (rc < 0) { +- crm_perror(LOG_DEBUG, "Couldn't complete registration with the fencing API: %d", rc); ++ crm_debug("Couldn't register with the fencer: %s " ++ CRM_XS " rc=%d", pcmk_strerror(rc), rc); + rc = -ECOMM; + + } else if (reply == NULL) { +- crm_err("Did not receive registration reply"); ++ crm_debug("Couldn't register with the fencer: no reply"); + rc = -EPROTO; + + } else { +@@ -1470,18 +1476,23 @@ stonith_api_signon(stonith_t * stonith, const char *name, int *stonith_fd) + const char *tmp_ticket = crm_element_value(reply, F_STONITH_CLIENTID); + + if (safe_str_neq(msg_type, CRM_OP_REGISTER)) { +- crm_err("Invalid registration message: %s", msg_type); +- crm_log_xml_err(reply, "Bad reply"); ++ crm_debug("Couldn't register with the fencer: invalid reply type '%s'", ++ (msg_type? msg_type : "(missing)")); ++ crm_log_xml_debug(reply, "Invalid fencer reply"); + rc = -EPROTO; + + } else if (tmp_ticket == NULL) { +- crm_err("No registration token provided"); +- crm_log_xml_err(reply, "Bad reply"); ++ crm_debug("Couldn't register with the fencer: no token in reply"); ++ crm_log_xml_debug(reply, "Invalid fencer reply"); + rc = -EPROTO; + + } else { +- crm_trace("Obtained registration token: %s", tmp_ticket); + native->token = strdup(tmp_ticket); ++#if HAVE_MSGFROMIPC_TIMEOUT ++ stonith->call_timeout = MAX_IPC_DELAY; ++#endif ++ crm_debug("Connection to fencer by %s succeeded (registration token: %s)", ++ display_name, native->token); + rc = pcmk_ok; + } + } +@@ -1490,16 +1501,11 @@ stonith_api_signon(stonith_t * stonith, const char *name, int *stonith_fd) + free_xml(hello); + } + +- if (rc == pcmk_ok) { +-#if HAVE_MSGFROMIPC_TIMEOUT +- stonith->call_timeout = MAX_IPC_DELAY; +-#endif +- crm_debug("Connection to fencer successful"); +- return pcmk_ok; ++ if (rc != pcmk_ok) { ++ crm_debug("Connection attempt to fencer by %s failed: %s " ++ CRM_XS " rc=%d", display_name, pcmk_strerror(rc), rc); ++ stonith->cmds->disconnect(stonith); + } +- +- crm_debug("Connection to fencer failed: %s", pcmk_strerror(rc)); +- stonith->cmds->disconnect(stonith); + return rc; + } + +@@ -2071,6 +2077,36 @@ stonith_api_new(void) + return new_stonith; + } + ++/*! ++ * \brief Make a blocking connection attempt to the fencer ++ * ++ * \param[in,out] st Fencer API object ++ * \param[in] name Client name to use with fencer ++ * \param[in] max_attempts Return error if this many attempts fail ++ * ++ * \return pcmk_ok on success, result of last attempt otherwise ++ */ ++int ++stonith_api_connect_retry(stonith_t *st, const char *name, int max_attempts) ++{ ++ int rc = -EINVAL; // if max_attempts is not positive ++ ++ for (int attempt = 1; attempt <= max_attempts; attempt++) { ++ rc = st->cmds->connect(st, name, NULL); ++ if (rc == pcmk_ok) { ++ return pcmk_ok; ++ } else if (attempt < max_attempts) { ++ crm_notice("Fencer connection attempt %d of %d failed (retrying in 2s): %s " ++ CRM_XS " rc=%d", ++ attempt, max_attempts, pcmk_strerror(rc), rc); ++ sleep(2); ++ } ++ } ++ crm_notice("Could not connect to fencer: %s " CRM_XS " rc=%d", ++ pcmk_strerror(rc), rc); ++ return rc; ++} ++ + stonith_key_value_t * + stonith_key_value_add(stonith_key_value_t * head, const char *key, const char *value) + { +@@ -2122,85 +2158,78 @@ stonith_key_value_freeall(stonith_key_value_t * head, int keys, int values) + int + stonith_api_kick(uint32_t nodeid, const char *uname, int timeout, bool off) + { +- char *name = NULL; +- const char *action = "reboot"; +- +- int rc = -EPROTO; +- stonith_t *st = NULL; +- enum stonith_call_options opts = st_opt_sync_call | st_opt_allow_suicide; ++ int rc = pcmk_ok; ++ stonith_t *st = stonith_api_new(); ++ const char *action = off? "off" : "reboot"; + + api_log_open(); +- st = stonith_api_new(); +- if (st) { +- rc = st->cmds->connect(st, "stonith-api", NULL); +- if(rc != pcmk_ok) { +- api_log(LOG_ERR, "Connection failed, could not kick (%s) node %u/%s : %s (%d)", action, nodeid, uname, pcmk_strerror(rc), rc); +- } ++ if (st == NULL) { ++ api_log(LOG_ERR, "API initialization failed, could not kick (%s) node %u/%s", ++ action, nodeid, uname); ++ return -EPROTO; + } + +- if (uname != NULL) { +- name = strdup(uname); +- +- } else if (nodeid > 0) { +- opts |= st_opt_cs_nodeid; +- name = crm_itoa(nodeid); +- } +- +- if (off) { +- action = "off"; +- } +- +- if (rc == pcmk_ok) { ++ rc = st->cmds->connect(st, "stonith-api", NULL); ++ if (rc != pcmk_ok) { ++ api_log(LOG_ERR, "Connection failed, could not kick (%s) node %u/%s : %s (%d)", ++ action, nodeid, uname, pcmk_strerror(rc), rc); ++ } else { ++ char *name = NULL; ++ enum stonith_call_options opts = st_opt_sync_call | st_opt_allow_suicide; ++ ++ if (uname != NULL) { ++ name = strdup(uname); ++ } else if (nodeid > 0) { ++ opts |= st_opt_cs_nodeid; ++ name = crm_itoa(nodeid); ++ } + rc = st->cmds->fence(st, opts, name, action, timeout, 0); +- if(rc != pcmk_ok) { +- api_log(LOG_ERR, "Could not kick (%s) node %u/%s : %s (%d)", action, nodeid, uname, pcmk_strerror(rc), rc); ++ free(name); ++ ++ if (rc != pcmk_ok) { ++ api_log(LOG_ERR, "Could not kick (%s) node %u/%s : %s (%d)", ++ action, nodeid, uname, pcmk_strerror(rc), rc); + } else { +- api_log(LOG_NOTICE, "Node %u/%s kicked: %s ", nodeid, uname, action); ++ api_log(LOG_NOTICE, "Node %u/%s kicked: %s", nodeid, uname, action); + } + } + +- if (st) { +- st->cmds->disconnect(st); +- stonith_api_delete(st); +- } +- +- free(name); ++ stonith_api_delete(st); + return rc; + } + + time_t + stonith_api_time(uint32_t nodeid, const char *uname, bool in_progress) + { +- int rc = 0; +- char *name = NULL; +- ++ int rc = pcmk_ok; + time_t when = 0; +- stonith_t *st = NULL; ++ stonith_t *st = stonith_api_new(); + stonith_history_t *history = NULL, *hp = NULL; +- enum stonith_call_options opts = st_opt_sync_call; +- +- st = stonith_api_new(); +- if (st) { +- rc = st->cmds->connect(st, "stonith-api", NULL); +- if(rc != pcmk_ok) { +- api_log(LOG_NOTICE, "Connection failed: %s (%d)", pcmk_strerror(rc), rc); +- } +- } +- +- if (uname != NULL) { +- name = strdup(uname); + +- } else if (nodeid > 0) { +- opts |= st_opt_cs_nodeid; +- name = crm_itoa(nodeid); ++ if (st == NULL) { ++ api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: " ++ "API initialization failed", nodeid, uname); ++ return when; + } + +- if (st && rc == pcmk_ok) { ++ rc = st->cmds->connect(st, "stonith-api", NULL); ++ if (rc != pcmk_ok) { ++ api_log(LOG_NOTICE, "Connection failed: %s (%d)", pcmk_strerror(rc), rc); ++ } else { + int entries = 0; + int progress = 0; + int completed = 0; +- ++ char *name = NULL; ++ enum stonith_call_options opts = st_opt_sync_call; ++ ++ if (uname != NULL) { ++ name = strdup(uname); ++ } else if (nodeid > 0) { ++ opts |= st_opt_cs_nodeid; ++ name = crm_itoa(nodeid); ++ } + rc = st->cmds->history(st, opts, name, &history, 120); ++ free(name); + + for (hp = history; hp; hp = hp->next) { + entries++; +@@ -2227,15 +2256,11 @@ stonith_api_time(uint32_t nodeid, const char *uname, bool in_progress) + } + } + +- if (st) { +- st->cmds->disconnect(st); +- stonith_api_delete(st); +- } ++ stonith_api_delete(st); + + if(when) { + api_log(LOG_INFO, "Node %u/%s last kicked at: %ld", nodeid, uname, (long int)when); + } +- free(name); + return when; + } + +diff --git a/tools/crm_mon.c b/tools/crm_mon.c +index e101b62..bed0796 100644 +--- a/tools/crm_mon.c ++++ b/tools/crm_mon.c +@@ -298,7 +298,6 @@ cib_connect(gboolean full) + } + + if ((fence_connect) && (st->state == stonith_disconnected)) { +- crm_trace("Connecting to stonith"); + rc = st->cmds->connect(st, crm_system_name, NULL); + if (rc == pcmk_ok) { + crm_trace("Setting up stonith callbacks"); +diff --git a/tools/stonith_admin.c b/tools/stonith_admin.c +index d960fb1..6be66c6 100644 +--- a/tools/stonith_admin.c ++++ b/tools/stonith_admin.c +@@ -198,31 +198,6 @@ struct { + int rc; + } async_fence_data; + +-static int +-try_mainloop_connect(void) +-{ +- stonith_t *st = async_fence_data.st; +- int tries = 10; +- int i = 0; +- int rc = 0; +- +- for (i = 0; i < tries; i++) { +- crm_debug("Connecting as %s", async_fence_data.name); +- rc = st->cmds->connect(st, async_fence_data.name, NULL); +- +- if (!rc) { +- crm_debug("stonith client connection established"); +- return 0; +- } else { +- crm_debug("stonith client connection failed"); +- } +- sleep(1); +- } +- +- crm_err("Could not connect to the fencer"); +- return -1; +-} +- + static void + notify_callback(stonith_t * st, stonith_event_t * e) + { +@@ -251,8 +226,10 @@ async_fence_helper(gpointer user_data) + { + stonith_t *st = async_fence_data.st; + int call_id = 0; ++ int rc = stonith_api_connect_retry(st, async_fence_data.name, 10); + +- if (try_mainloop_connect()) { ++ if (rc != pcmk_ok) { ++ fprintf(stderr, "Could not connect to fencer: %s\n", pcmk_strerror(rc)); + g_main_loop_quit(mainloop); + return TRUE; + } +-- +1.8.3.1 + diff --git a/SOURCES/003-security-log.patch b/SOURCES/003-security-log.patch deleted file mode 100644 index cbb521b..0000000 --- a/SOURCES/003-security-log.patch +++ /dev/null @@ -1,186 +0,0 @@ -From bccf845261c6e69fc4e6bdb8cf4e630a4a4ec7a8 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 12 Apr 2019 09:46:51 -0500 -Subject: [PATCH] Log: libcrmcluster: improve CPG membership messages - -Show CPG event reason when provided by corosync, make messages more readable, -upgrade duplicate pid messages to warnings (and log only one message in those -cases). ---- - lib/cluster/cpg.c | 91 ++++++++++++++++++++++++++++++++++--------------------- - 1 file changed, 56 insertions(+), 35 deletions(-) - -diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c -index 2898c51..ef6fa36 100644 ---- a/lib/cluster/cpg.c -+++ b/lib/cluster/cpg.c -@@ -360,8 +360,6 @@ pcmk_message_common_cs(cpg_handle_t handle, uint32_t nodeid, uint32_t pid, void - return NULL; - } - --#define PEER_NAME(peer) ((peer)? ((peer)->uname? (peer)->uname : "") : "") -- - static int cmp_member_list_nodeid(const void *first, - const void *second) - { -@@ -376,6 +374,32 @@ static int cmp_member_list_nodeid(const void *first, - return 0; - } - -+static const char * -+cpgreason2str(cpg_reason_t reason) -+{ -+ switch (reason) { -+ case CPG_REASON_JOIN: return " via cpg_join"; -+ case CPG_REASON_LEAVE: return " via cpg_leave"; -+ case CPG_REASON_NODEDOWN: return " via cluster exit"; -+ case CPG_REASON_NODEUP: return " via cluster join"; -+ case CPG_REASON_PROCDOWN: return " for unknown reason"; -+ default: break; -+ } -+ return ""; -+} -+ -+static inline const char * -+peer_name(crm_node_t *peer) -+{ -+ if (peer == NULL) { -+ return "unknown node"; -+ } else if (peer->uname == NULL) { -+ return "peer node"; -+ } else { -+ return peer->uname; -+ } -+} -+ - void - pcmk_cpg_membership(cpg_handle_t handle, - const struct cpg_name *groupName, -@@ -387,7 +411,7 @@ pcmk_cpg_membership(cpg_handle_t handle, - gboolean found = FALSE; - static int counter = 0; - uint32_t local_nodeid = get_local_nodeid(handle); -- const struct cpg_address *key, **rival, **sorted; -+ const struct cpg_address *key, **sorted; - - sorted = malloc(member_list_entries * sizeof(const struct cpg_address *)); - CRM_ASSERT(sorted != NULL); -@@ -401,10 +425,7 @@ pcmk_cpg_membership(cpg_handle_t handle, - - for (i = 0; i < left_list_entries; i++) { - crm_node_t *peer = crm_find_peer(left_list[i].nodeid, NULL); -- -- crm_info("Group event %s.%d: node %u (%s) left: %llu", -- groupName->value, counter, left_list[i].nodeid, -- PEER_NAME(peer), (unsigned long long) left_list[i].pid); -+ const struct cpg_address **rival = NULL; - - /* in CPG world, NODE:PROCESS-IN-MEMBERSHIP-OF-G is an 1:N relation - and not playing by this rule may go wild in case of multiple -@@ -418,7 +439,7 @@ pcmk_cpg_membership(cpg_handle_t handle, - also API end-point carriers, and that's what matters locally - (who's the winner); - remotely, we will just compare leave_list and member_list and if -- the left process has it's node retained in member_list (under some -+ the left process has its node retained in member_list (under some - other PID, anyway) we will just ignore it as well - XXX: long-term fix is to establish in-out PID-aware tracking? */ - if (peer) { -@@ -426,50 +447,51 @@ pcmk_cpg_membership(cpg_handle_t handle, - rival = bsearch(&key, sorted, member_list_entries, - sizeof(const struct cpg_address *), - cmp_member_list_nodeid); -- if (rival == NULL) { -+ } -+ -+ if (rival == NULL) { -+ crm_info("Group %s event %d: %s (node %u pid %u) left%s", -+ groupName->value, counter, peer_name(peer), -+ left_list[i].nodeid, left_list[i].pid, -+ cpgreason2str(left_list[i].reason)); -+ if (peer) { - crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, - OFFLINESTATUS); -- } else if (left_list[i].nodeid == local_nodeid) { -- crm_info("Ignoring the above event %s.%d, comes from a local" -- " rival process (presumably not us): %llu", -- groupName->value, counter, -- (unsigned long long) left_list[i].pid); -- } else { -- crm_info("Ignoring the above event %s.%d, comes from" -- " a rival-rich node: %llu (e.g. %llu process" -- " carries on)", -- groupName->value, counter, -- (unsigned long long) left_list[i].pid, -- (unsigned long long) (*rival)->pid); - } -+ } else if (left_list[i].nodeid == local_nodeid) { -+ crm_warn("Group %s event %d: duplicate local pid %u left%s", -+ groupName->value, counter, -+ left_list[i].pid, cpgreason2str(left_list[i].reason)); -+ } else { -+ crm_warn("Group %s event %d: " -+ "%s (node %u) duplicate pid %u left%s (%u remains)", -+ groupName->value, counter, peer_name(peer), -+ left_list[i].nodeid, left_list[i].pid, -+ cpgreason2str(left_list[i].reason), (*rival)->pid); - } - } - free(sorted); - sorted = NULL; - - for (i = 0; i < joined_list_entries; i++) { -- crm_info("Group event %s.%d: node %u joined: %llu" -- " (unchecked for rivals)", -+ crm_info("Group %s event %d: node %u pid %u joined%s", - groupName->value, counter, joined_list[i].nodeid, -- (unsigned long long) joined_list[i].pid); -+ joined_list[i].pid, cpgreason2str(joined_list[i].reason)); - } - - for (i = 0; i < member_list_entries; i++) { - crm_node_t *peer = crm_get_peer(member_list[i].nodeid, NULL); - -- crm_info("Group event %s.%d: node %u (%s) is member: %llu" -- " (at least once)", -- groupName->value, counter, member_list[i].nodeid, -- PEER_NAME(peer), member_list[i].pid); -- - if (member_list[i].nodeid == local_nodeid - && member_list[i].pid != getpid()) { - /* see the note above */ -- crm_info("Ignoring the above event %s.%d, comes from a local rival" -- " process: %llu", groupName->value, counter, -- (unsigned long long) member_list[i].pid); -+ crm_warn("Group %s event %d: detected duplicate local pid %u", -+ groupName->value, counter, member_list[i].pid); - continue; - } -+ crm_info("Group %s event %d: %s (node %u pid %u) is member", -+ groupName->value, counter, peer_name(peer), -+ member_list[i].nodeid, member_list[i].pid); - - /* If the caller left auto-reaping enabled, this will also update the - * state to member. -@@ -492,8 +514,7 @@ pcmk_cpg_membership(cpg_handle_t handle, - - } else if (now > (peer->when_lost + 60)) { - // If it persists for more than a minute, update the state -- crm_warn("Node %u member of group %s but believed offline" -- " (unchecked for rivals)", -+ crm_warn("Node %u is member of group %s but was believed offline", - member_list[i].nodeid, groupName->value); - crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_MEMBER, 0); - } -@@ -505,7 +526,7 @@ pcmk_cpg_membership(cpg_handle_t handle, - } - - if (!found) { -- crm_err("We're not part of CPG group '%s' anymore!", groupName->value); -+ crm_err("Local node was evicted from group %s", groupName->value); - cpg_evicted = TRUE; - } - --- -1.8.3.1 - diff --git a/SOURCES/004-concurrent-fencing.patch b/SOURCES/004-concurrent-fencing.patch new file mode 100644 index 0000000..4bab3e6 --- /dev/null +++ b/SOURCES/004-concurrent-fencing.patch @@ -0,0 +1,49 @@ +From 463eb8e36e2d2bf10a0e37938e0924ea6699f041 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 30 May 2019 08:37:52 -0500 +Subject: [PATCH] Low: libpe_status: offer compile-time option to change + concurrent-fencing default + +We most likely want to make concurrent-fencing default to true at some point. +For now, offer that possibility via a compile-time constant, for experimenting. +--- + lib/pengine/common.c | 8 +++++++- + lib/pengine/status.c | 3 +++ + 2 files changed, 10 insertions(+), 1 deletion(-) + +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index 9513633..3a283b4 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -95,7 +95,13 @@ static pe_cluster_option pe_opts[] = { + "How long to wait for the STONITH action (reboot,on,off) to complete", NULL }, + { XML_ATTR_HAVE_WATCHDOG, NULL, "boolean", NULL, "false", &check_boolean, + "Enable watchdog integration", "Set automatically by the cluster if SBD is detected. User configured values are ignored." }, +- { "concurrent-fencing", NULL, "boolean", NULL, "false", &check_boolean, ++ { "concurrent-fencing", NULL, "boolean", NULL, ++#ifdef DEFAULT_CONCURRENT_FENCING_TRUE ++ "true", ++#else ++ "false", ++#endif ++ &check_boolean, + "Allow performing fencing operations in parallel", NULL }, + { "startup-fencing", NULL, "boolean", NULL, "true", &check_boolean, + "STONITH unseen nodes", "Advanced Use Only! Not using the default is very unsafe!" }, +diff --git a/lib/pengine/status.c b/lib/pengine/status.c +index 3ccfac4..a8b0947 100644 +--- a/lib/pengine/status.c ++++ b/lib/pengine/status.c +@@ -354,6 +354,9 @@ set_working_set_defaults(pe_working_set_t * data_set) + set_bit(data_set->flags, pe_flag_stop_rsc_orphans); + set_bit(data_set->flags, pe_flag_symmetric_cluster); + set_bit(data_set->flags, pe_flag_stop_action_orphans); ++#ifdef DEFAULT_CONCURRENT_FENCING_TRUE ++ set_bit(data_set->flags, pe_flag_concurrent_fencing); ++#endif + } + + resource_t * +-- +1.8.3.1 + diff --git a/SOURCES/004-security-active.patch b/SOURCES/004-security-active.patch deleted file mode 100644 index 806dffc..0000000 --- a/SOURCES/004-security-active.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 7dda20dac25f07eae959ca25cc974ef2fa6daf02 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 24 Apr 2019 16:25:46 -0500 -Subject: [PATCH] Fix: libcrmcommon: avoid use-of-NULL when checking whether - process is active - ---- - lib/common/pid.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/lib/common/pid.c b/lib/common/pid.c -index 2439680..4fbf2dd 100644 ---- a/lib/common/pid.c -+++ b/lib/common/pid.c -@@ -57,7 +57,7 @@ crm_pid_active(long pid, const char *daemon) - } else if (rc == 0 && (daemon == NULL || have_proc_pid == -1)) { - return 1; /* kill as the only indicator, cannot double check */ - -- } else { -+ } else if (daemon != NULL) { - /* make sure PID hasn't been reused by another process - XXX: might still be just a zombie, which could confuse decisions */ - bool checked_through_kill = (rc == 0); --- -1.8.3.1 - diff --git a/SOURCES/005-glib-priorities.patch b/SOURCES/005-glib-priorities.patch new file mode 100644 index 0000000..3106932 --- /dev/null +++ b/SOURCES/005-glib-priorities.patch @@ -0,0 +1,211 @@ +From 65170ffd5fa10cbda176b3f88e817d534b6331d6 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= +Date: Wed, 29 Aug 2018 15:49:58 +0200 +Subject: [PATCH 1/2] Low: mainloop: make it possible to specify server's + priority in mainloop + +--- + include/crm/common/mainloop.h | 24 +++++++++++++ + lib/common/mainloop.c | 82 +++++++++++++++++++++++++++++++++++++++++-- + 2 files changed, 103 insertions(+), 3 deletions(-) + +diff --git a/include/crm/common/mainloop.h b/include/crm/common/mainloop.h +index 85da1cd..2cfb63e 100644 +--- a/include/crm/common/mainloop.h ++++ b/include/crm/common/mainloop.h +@@ -79,6 +79,30 @@ struct ipc_client_callbacks { + qb_ipcs_service_t *mainloop_add_ipc_server(const char *name, enum qb_ipc_type type, + struct qb_ipcs_service_handlers *callbacks); + ++/*! ++ * \brief Start server-side API end-point, hooked into the internal event loop ++ * ++ * \param[in] name name of the IPC end-point ("address" for the client) ++ * \param[in] type selects libqb's IPC back-end (or use #QB_IPC_NATIVE) ++ * \param[in] callbacks defines libqb's IPC service-level handlers ++ * \param[in] priority priority relative to other events handled in the ++ * abstract handling loop, use #QB_LOOP_MED when unsure ++ * ++ * \return libqb's opaque handle to the created service abstraction ++ * ++ * \note For portability concerns, do not use this function if you keep ++ * \p priority as #QB_LOOP_MED, stick with #mainloop_add_ipc_server ++ * (with exactly such semantics) instead (once you link with this new ++ * symbol employed, you can't downgrade the library freely anymore). ++ * ++ * \note The intended effect will only get fully reflected when run-time ++ * linked to patched libqb: https://github.com/ClusterLabs/libqb/pull/352 ++ */ ++qb_ipcs_service_t *mainloop_add_ipc_server_with_prio(const char *name, ++ enum qb_ipc_type type, ++ struct qb_ipcs_service_handlers *callbacks, ++ enum qb_loop_priority prio); ++ + void mainloop_del_ipc_server(qb_ipcs_service_t * server); + + mainloop_io_t *mainloop_add_ipc_client(const char *name, int priority, size_t max_size, +diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c +index 18f7014..17e69f0 100644 +--- a/lib/common/mainloop.c ++++ b/lib/common/mainloop.c +@@ -509,6 +509,65 @@ gio_poll_destroy(gpointer data) + } + } + ++/*! ++ * \internal ++ * \brief Convert libqb's poll priority into GLib's one ++ * ++ * \param[in] prio libqb's poll priority (#QB_LOOP_MED assumed as fallback) ++ * ++ * \return best matching GLib's priority ++ */ ++static gint ++conv_prio_libqb2glib(enum qb_loop_priority prio) ++{ ++ gint ret = G_PRIORITY_DEFAULT; ++ switch (prio) { ++ case QB_LOOP_LOW: ++ ret = G_PRIORITY_LOW; ++ break; ++ case QB_LOOP_HIGH: ++ ret = G_PRIORITY_HIGH; ++ break; ++ default: ++ crm_trace("Invalid libqb's loop priority %d, assuming QB_LOOP_MED", ++ prio); ++ /* fall-through */ ++ case QB_LOOP_MED: ++ break; ++ } ++ return ret; ++} ++ ++/*! ++ * \internal ++ * \brief Convert libqb's poll priority to rate limiting spec ++ * ++ * \param[in] prio libqb's poll priority (#QB_LOOP_MED assumed as fallback) ++ * ++ * \return best matching rate limiting spec ++ */ ++static enum qb_ipcs_rate_limit ++conv_libqb_prio2ratelimit(enum qb_loop_priority prio) ++{ ++ /* this is an inversion of what libqb's qb_ipcs_request_rate_limit does */ ++ enum qb_ipcs_rate_limit ret = QB_IPCS_RATE_NORMAL; ++ switch (prio) { ++ case QB_LOOP_LOW: ++ ret = QB_IPCS_RATE_SLOW; ++ break; ++ case QB_LOOP_HIGH: ++ ret = QB_IPCS_RATE_FAST; ++ break; ++ default: ++ crm_trace("Invalid libqb's loop priority %d, assuming QB_LOOP_MED", ++ prio); ++ /* fall-through */ ++ case QB_LOOP_MED: ++ break; ++ } ++ return ret; ++} ++ + static int32_t + gio_poll_dispatch_update(enum qb_loop_priority p, int32_t fd, int32_t evts, + void *data, qb_ipcs_dispatch_fn_t fn, int32_t add) +@@ -555,8 +614,8 @@ gio_poll_dispatch_update(enum qb_loop_priority p, int32_t fd, int32_t evts, + adaptor->p = p; + adaptor->is_used++; + adaptor->source = +- g_io_add_watch_full(channel, G_PRIORITY_DEFAULT, evts, gio_read_socket, adaptor, +- gio_poll_destroy); ++ g_io_add_watch_full(channel, conv_prio_libqb2glib(p), evts, ++ gio_read_socket, adaptor, gio_poll_destroy); + + /* Now that mainloop now holds a reference to channel, + * thanks to g_io_add_watch_full(), drop ours from g_io_channel_unix_new(). +@@ -640,7 +699,15 @@ pick_ipc_type(enum qb_ipc_type requested) + + qb_ipcs_service_t * + mainloop_add_ipc_server(const char *name, enum qb_ipc_type type, +- struct qb_ipcs_service_handlers * callbacks) ++ struct qb_ipcs_service_handlers *callbacks) ++{ ++ return mainloop_add_ipc_server_with_prio(name, type, callbacks, QB_LOOP_MED); ++} ++ ++qb_ipcs_service_t * ++mainloop_add_ipc_server_with_prio(const char *name, enum qb_ipc_type type, ++ struct qb_ipcs_service_handlers *callbacks, ++ enum qb_loop_priority prio) + { + int rc = 0; + qb_ipcs_service_t *server = NULL; +@@ -652,6 +719,15 @@ mainloop_add_ipc_server(const char *name, enum qb_ipc_type type, + crm_client_init(); + server = qb_ipcs_create(name, 0, pick_ipc_type(type), callbacks); + ++ if (server == NULL) { ++ crm_err("Could not create %s IPC server: %s (%d)", name, pcmk_strerror(rc), rc); ++ return NULL; ++ } ++ ++ if (prio != QB_LOOP_MED) { ++ qb_ipcs_request_rate_limit(server, conv_libqb_prio2ratelimit(prio)); ++ } ++ + #ifdef HAVE_IPCS_GET_BUFFER_SIZE + /* All clients should use at least ipc_buffer_max as their buffer size */ + qb_ipcs_enforce_buffer_size(server, crm_ipc_default_buffer_size()); +-- +1.8.3.1 + + +From 3401f25994e8cc059898550082f9b75f2d07f103 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= +Date: Wed, 29 Aug 2018 15:50:57 +0200 +Subject: [PATCH 2/2] High: stonith-ng's function cannot be blocked with CIB + updates forever + +In the high-load (or high-rate-config-change) scenarios, +pacemaker-fenced would be unable to provide service when basically DoS'd +with CIB update notifications. Try to reconcile that with elevated +priority of the server's proper listening interface in the mainloop, at +worst, it will try to fence with slightly outdated config, but appears +to be less bad than not carrying the execution at all, for instance. +Other daemons might be considered as well. + +Prerequisites: +- https://github.com/ClusterLabs/libqb/pull/352 + (libqb used to contain a bug due to which one particular step in the + initial-client-connection-accepting-at-the-server procedure that would + be carried out with hard-coded (and hence possibly lower than competing + events') priority, which backfires exactly in this case (once the + pacemaker part is fixed -- by the means of elevating priority for + the API end-point of fenced so that it won't get consistently + overridden with a non-socket-based event source/trigger) + +How to verify: +- mocked/based -N (see commit adding that module to mocked based daemon) +--- + lib/common/utils.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/lib/common/utils.c b/lib/common/utils.c +index 758eb1b..d1c3e26 100644 +--- a/lib/common/utils.c ++++ b/lib/common/utils.c +@@ -1031,7 +1031,8 @@ attrd_ipc_server_init(qb_ipcs_service_t **ipcs, struct qb_ipcs_service_handlers + void + stonith_ipc_server_init(qb_ipcs_service_t **ipcs, struct qb_ipcs_service_handlers *cb) + { +- *ipcs = mainloop_add_ipc_server("stonith-ng", QB_IPC_NATIVE, cb); ++ *ipcs = mainloop_add_ipc_server_with_prio("stonith-ng", QB_IPC_NATIVE, cb, ++ QB_LOOP_HIGH); + + if (*ipcs == NULL) { + crm_err("Failed to create fencer: exiting and inhibiting respawn."); +-- +1.8.3.1 + diff --git a/SOURCES/005-security-code.patch b/SOURCES/005-security-code.patch deleted file mode 100644 index 633a396..0000000 --- a/SOURCES/005-security-code.patch +++ /dev/null @@ -1,28 +0,0 @@ -From d9b0269d59a00329feb19b6e65b10a233a3dd414 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 29 Apr 2019 14:34:32 -0500 -Subject: [PATCH] Low: libcrmcommon: return proper code if testing pid is - denied - -7dda20d avoided a use-of-NULL in an unlikely corner case, but returned the -wrong code in that case. ---- - lib/common/pid.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/lib/common/pid.c b/lib/common/pid.c -index 4fbf2dd..ccee03f 100644 ---- a/lib/common/pid.c -+++ b/lib/common/pid.c -@@ -47,7 +47,7 @@ crm_pid_active(long pid, const char *daemon) - } else if ((rc = kill(pid, 0)) < 0 && errno == ESRCH) { - return 0; /* no such PID detected */ - -- } else if (rc < 0 && have_proc_pid == -1) { -+ } else if (rc < 0 && (daemon == NULL || have_proc_pid == -1)) { - if (last_asked_pid != pid) { - crm_info("Cannot examine PID %ld: %s", pid, strerror(errno)); - last_asked_pid = pid; --- -1.8.3.1 - diff --git a/SOURCES/006-bundle-fixes.patch b/SOURCES/006-bundle-fixes.patch new file mode 100644 index 0000000..1c3ea40 --- /dev/null +++ b/SOURCES/006-bundle-fixes.patch @@ -0,0 +1,233 @@ +From 169d424cf88594f15e7e66baa705df6b727aa807 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 4 Jun 2019 16:24:16 -0500 +Subject: [PATCH 1/4] Log: pacemaker-remoted: use different default log if pid + 1 + +When pacemaker-remoted runs as pid 1 inside a container, there may not be a +/var/log/pacemaker directory. To get around this, use a default log of +/var/log/pcmk-init.log when running as pid 1. + +This was chosen over alternatives (creating the /var/log/pacemaker directory, +or passing the log location as an environment variable when creating the +implicit container resource) because it both avoids forcing a restart of +active bundles due to configuration change (as well as preserving regression +test output) and allows users to configure an explicit log location via the +container image or the bundle's extra arguments. +--- + daemons/execd/pacemaker-execd.c | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/daemons/execd/pacemaker-execd.c b/daemons/execd/pacemaker-execd.c +index e2fdfca..cfa5500 100644 +--- a/daemons/execd/pacemaker-execd.c ++++ b/daemons/execd/pacemaker-execd.c +@@ -429,6 +429,14 @@ static void spawn_pidone(int argc, char **argv, char **envp) + return; + } + ++ /* Containers can be expected to have /var/log, but they may not have ++ * /var/log/pacemaker, so use a different default if no value has been ++ * explicitly configured in the container's environment. ++ */ ++ if (daemon_option("logfile") == NULL) { ++ set_daemon_option("logfile", "/var/log/pcmk-init.log"); ++ } ++ + sigfillset(&set); + sigprocmask(SIG_BLOCK, &set, 0); + +-- +1.8.3.1 + + +From 7e362387a092b5617b36a69961115f7703e4d801 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 17 May 2019 12:39:43 -0500 +Subject: [PATCH 2/4] Refactor: libpe_status: add enum for bundle mount flags + +More readable than 0 or 1 +--- + lib/pengine/bundle.c | 17 +++++++++-------- + lib/pengine/variant.h | 9 ++++++++- + 2 files changed, 17 insertions(+), 9 deletions(-) + +diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c +index 3b32f04..b223f03 100644 +--- a/lib/pengine/bundle.c ++++ b/lib/pengine/bundle.c +@@ -228,7 +228,7 @@ create_docker_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data, + for(GListPtr pIter = data->mounts; pIter != NULL; pIter = pIter->next) { + pe__bundle_mount_t *mount = pIter->data; + +- if(mount->flags) { ++ if (is_set(mount->flags, pe__bundle_mount_subdir)) { + char *source = crm_strdup_printf( + "%s/%s-%d", mount->source, data->prefix, replica->offset); + +@@ -396,7 +396,7 @@ create_podman_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data, + for(GListPtr pIter = data->mounts; pIter != NULL; pIter = pIter->next) { + pe__bundle_mount_t *mount = pIter->data; + +- if(mount->flags) { ++ if (is_set(mount->flags, pe__bundle_mount_subdir)) { + char *source = crm_strdup_printf( + "%s/%s-%d", mount->source, data->prefix, replica->offset); + +@@ -562,7 +562,7 @@ create_rkt_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data, + for(GListPtr pIter = data->mounts; pIter != NULL; pIter = pIter->next) { + pe__bundle_mount_t *mount = pIter->data; + +- if(mount->flags) { ++ if (is_set(mount->flags, pe__bundle_mount_subdir)) { + char *source = crm_strdup_printf( + "%s/%s-%d", mount->source, data->prefix, replica->offset); + +@@ -894,7 +894,7 @@ create_container(pe_resource_t *parent, pe__bundle_variant_data_t *data, + + static void + mount_add(pe__bundle_variant_data_t *bundle_data, const char *source, +- const char *target, const char *options, int flags) ++ const char *target, const char *options, uint32_t flags) + { + pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t)); + +@@ -1142,11 +1142,11 @@ pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set) + const char *source = crm_element_value(xml_child, "source-dir"); + const char *target = crm_element_value(xml_child, "target-dir"); + const char *options = crm_element_value(xml_child, "options"); +- int flags = 0; ++ int flags = pe__bundle_mount_none; + + if (source == NULL) { + source = crm_element_value(xml_child, "source-dir-root"); +- flags = 1; ++ set_bit(flags, pe__bundle_mount_subdir); + } + + if (source && target) { +@@ -1251,9 +1251,10 @@ pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set) + * reasonable. + */ + mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION, +- DEFAULT_REMOTE_KEY_LOCATION, NULL, 0); ++ DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none); + +- mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL, 1); ++ mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL, ++ pe__bundle_mount_subdir); + + port = calloc(1, sizeof(pe__bundle_port_t)); + if(bundle_data->control_port) { +diff --git a/lib/pengine/variant.h b/lib/pengine/variant.h +index f46aa11..7f77eef 100644 +--- a/lib/pengine/variant.h ++++ b/lib/pengine/variant.h +@@ -51,11 +51,18 @@ typedef struct { + pe_resource_t *remote; + } pe__bundle_replica_t; + ++enum pe__bundle_mount_flags { ++ pe__bundle_mount_none = 0x00, ++ ++ // mount instance-specific subdirectory rather than source directly ++ pe__bundle_mount_subdir = 0x01 ++}; ++ + typedef struct { + char *source; + char *target; + char *options; +- int flags; ++ uint32_t flags; // bitmask of pe__bundle_mount_flags + } pe__bundle_mount_t; + + typedef struct { +-- +1.8.3.1 + + +From 87eac95868930ffda4d964c2b6bd9960b6893cc9 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 17 May 2019 14:13:54 -0500 +Subject: [PATCH 3/4] Fix: controller: don't check join status after remote + node appears + +Only cluster nodes have join state +--- + daemons/controld/controld_callbacks.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/daemons/controld/controld_callbacks.c b/daemons/controld/controld_callbacks.c +index 06ffb9d..3ce7470 100644 +--- a/daemons/controld/controld_callbacks.c ++++ b/daemons/controld/controld_callbacks.c +@@ -228,7 +228,7 @@ peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *d + crm_trace("Alive=%d, appeared=%d, down=%d", + alive, appeared, (down? down->id : -1)); + +- if (appeared && (alive > 0)) { ++ if (appeared && (alive > 0) && !is_remote) { + register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL); + } + +-- +1.8.3.1 + + +From 5755b63850a17cd91bca28e83c39119378fe1887 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Sat, 18 May 2019 21:59:00 -0500 +Subject: [PATCH 4/4] Doc: Pacemaker Explained: document effect of SELinux on + bundle storage + +--- + doc/Pacemaker_Explained/en-US/Ch-Advanced-Resources.txt | 15 ++++++++++++--- + 1 file changed, 12 insertions(+), 3 deletions(-) + +diff --git a/doc/Pacemaker_Explained/en-US/Ch-Advanced-Resources.txt b/doc/Pacemaker_Explained/en-US/Ch-Advanced-Resources.txt +index e431626..4a181df 100644 +--- a/doc/Pacemaker_Explained/en-US/Ch-Advanced-Resources.txt ++++ b/doc/Pacemaker_Explained/en-US/Ch-Advanced-Resources.txt +@@ -999,11 +999,11 @@ association with Docker, Inc. is implied.] + ++ options="rw,Z"/> + ++ options="rw,Z"/> + + + +@@ -1293,7 +1293,8 @@ indexterm:[bundle,storage,storage-mapping] + + |options + | +-|File system mount options to use when mapping the storage ++|A comma-separated list of file system mount options to use when mapping the ++ storage + indexterm:[options,storage-mapping] + indexterm:[storage-mapping,Property,options] + +@@ -1322,6 +1323,14 @@ The +PCMK_authkey_location+ environment variable must not be set to anything + other than the default of `/etc/pacemaker/authkey` on any node in the cluster. + ==== + ++[IMPORTANT] ++==== ++If SELinux is used in enforcing mode on the host, you must ensure the container ++is allowed to use any storage you mount into it. For Docker and podman bundles, ++adding "Z" to the mount options will create a container-specific label for the ++mount that allows the container access. ++==== ++ + === Bundle Primitive === + + A bundle may optionally contain one ++ resource +-- +1.8.3.1 + diff --git a/SOURCES/006-migration.patch b/SOURCES/006-migration.patch deleted file mode 100644 index 9a87584..0000000 --- a/SOURCES/006-migration.patch +++ /dev/null @@ -1,55 +0,0 @@ -From 0dca216a84720e5652b1d87aac8c7e0aa8b4a31c Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 27 Feb 2019 14:21:26 -0600 -Subject: [PATCH] High: libcrmcommon: complete interrupted live migrations - correctly - -8435966 (2.0.0) got the sense of compare_version() wrong when attempting to -remove the old behavior for recording partial live migrations, instead -re-introducing it. - -This causes a regression in behavior when a transition containing a live -migration is aborted after the migrate_to has been initiated but before the -migrate_from has been initiated. In that case, it would schedule a full start -on the destination node rather than a migrate_from. ---- - lib/common/operations.c | 16 ++-------------- - 1 file changed, 2 insertions(+), 14 deletions(-) - -diff --git a/lib/common/operations.c b/lib/common/operations.c -index 40d4c18..88c4fa5 100644 ---- a/lib/common/operations.c -+++ b/lib/common/operations.c -@@ -396,8 +396,8 @@ create_operation_update(xmlNode * parent, lrmd_event_data_t * op, const char * c - - task = op->op_type; - -- /* Remap the task name under various scenarios, to make life easier for the -- * PE when determining the current state. -+ /* Record a successful reload as a start, and a failed reload as a monitor, -+ * to make life easier for the scheduler when determining the current state. - */ - if (crm_str_eq(task, "reload", TRUE)) { - if (op->op_status == PCMK_LRM_OP_DONE) { -@@ -405,18 +405,6 @@ create_operation_update(xmlNode * parent, lrmd_event_data_t * op, const char * c - } else { - task = CRMD_ACTION_STATUS; - } -- -- } else if (crm_str_eq(task, CRMD_ACTION_MIGRATE, TRUE)) { -- /* if the migrate_from fails it will have enough info to do the right thing */ -- if (op->op_status == PCMK_LRM_OP_DONE) { -- task = CRMD_ACTION_STOP; -- } else { -- task = CRMD_ACTION_STATUS; -- } -- -- } else if ((op->op_status == PCMK_LRM_OP_DONE) -- && crm_str_eq(task, CRMD_ACTION_MIGRATED, TRUE)) { -- task = CRMD_ACTION_START; - } - - key = generate_op_key(op->rsc_id, task, op->interval_ms); --- -1.8.3.1 - diff --git a/SOURCES/007-fork-controld_fencing.patch b/SOURCES/007-fork-controld_fencing.patch new file mode 100644 index 0000000..903f01a --- /dev/null +++ b/SOURCES/007-fork-controld_fencing.patch @@ -0,0 +1,2158 @@ +From edd133ade2bd9b003d3437280271a9c9dbab3ed6 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 23 May 2019 16:36:12 -0500 +Subject: [PATCH] Refactor: controller: separate fencing-related functionality + into own source file + +Before: + 748 daemons/controld/controld_te_actions.c + 942 daemons/controld/controld_te_callbacks.c + 725 daemons/controld/controld_te_utils.c + 84 daemons/controld/controld_transition.h + 110 daemons/controld/controld_utils.h + +After: + 838 daemons/controld/controld_fencing.c + 37 daemons/controld/controld_fencing.h + 631 daemons/controld/controld_te_actions.c + 701 daemons/controld/controld_te_callbacks.c + 298 daemons/controld/controld_te_utils.c + 65 daemons/controld/controld_transition.h + 106 daemons/controld/controld_utils.h +--- + daemons/controld/Makefile.am | 5 +- + daemons/controld/controld_callbacks.c | 3 +- + daemons/controld/controld_control.c | 2 +- + daemons/controld/controld_election.c | 3 +- + daemons/controld/controld_fencing.c | 838 +++++++++++++++++++++++++++++++ + daemons/controld/controld_fencing.h | 37 ++ + daemons/controld/controld_fsa.c | 1 + + daemons/controld/controld_messages.c | 1 + + daemons/controld/controld_te_actions.c | 121 +---- + daemons/controld/controld_te_callbacks.c | 243 +-------- + daemons/controld/controld_te_utils.c | 429 +--------------- + daemons/controld/controld_transition.c | 1 - + daemons/controld/controld_transition.h | 21 +- + daemons/controld/controld_utils.h | 4 - + 14 files changed, 891 insertions(+), 818 deletions(-) + create mode 100644 daemons/controld/controld_fencing.c + create mode 100644 daemons/controld/controld_fencing.h + +diff --git a/daemons/controld/Makefile.am b/daemons/controld/Makefile.am +index 17c3342..858e1bb 100644 +--- a/daemons/controld/Makefile.am ++++ b/daemons/controld/Makefile.am +@@ -1,5 +1,7 @@ + # +-# Copyright 2004-2018 Andrew Beekhof ++# Copyright 2018-2019 the Pacemaker project contributors ++# ++# The version control history for this file may have further details. + # + # This source code is licensed under the GNU General Public License version 2 + # or later (GPLv2+) WITHOUT ANY WARRANTY. +@@ -46,6 +48,7 @@ pacemaker_controld_SOURCES = pacemaker-controld.c \ + controld_election.c \ + controld_execd.c \ + controld_execd_state.c \ ++ controld_fencing.c \ + controld_fsa.c \ + controld_join_client.c \ + controld_join_dc.c \ +diff --git a/daemons/controld/controld_callbacks.c b/daemons/controld/controld_callbacks.c +index a188263..06ffb9d 100644 +--- a/daemons/controld/controld_callbacks.c ++++ b/daemons/controld/controld_callbacks.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2018 Andrew Beekhof ++ * Copyright 2004-2019 the Pacemaker project contributors + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. +@@ -22,6 +22,7 @@ + #include + #include + #include ++#include + #include + #include + +diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c +index 6d9f335..7f918c0 100644 +--- a/daemons/controld/controld_control.c ++++ b/daemons/controld/controld_control.c +@@ -25,6 +25,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -147,7 +148,6 @@ extern char *max_generation_from; + extern xmlNode *max_generation_xml; + extern GHashTable *resource_history; + extern GHashTable *voted; +-extern char *te_client_id; + + void + crmd_fast_exit(crm_exit_t exit_code) +diff --git a/daemons/controld/controld_election.c b/daemons/controld/controld_election.c +index 5d6858c..9e49c7b 100644 +--- a/daemons/controld/controld_election.c ++++ b/daemons/controld/controld_election.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2019 Andrew Beekhof ++ * Copyright 2004-2019 the Pacemaker project contributors + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. +@@ -18,6 +18,7 @@ + #include + #include + #include ++#include + #include + #include + #include +diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c +new file mode 100644 +index 0000000..cde57b5 +--- /dev/null ++++ b/daemons/controld/controld_fencing.c +@@ -0,0 +1,838 @@ ++/* ++ * Copyright 2004-2019 the Pacemaker project contributors ++ * ++ * This source code is licensed under the GNU General Public License version 2 ++ * or later (GPLv2+) WITHOUT ANY WARRANTY. ++ */ ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++ ++#ifdef HAVE_SYS_REBOOT_H ++# include ++# include ++#endif ++ ++/* ++ * stonith failure counting ++ * ++ * We don't want to get stuck in a permanent fencing loop. Keep track of the ++ * number of fencing failures for each target node, and the most we'll restart a ++ * transition for. ++ */ ++ ++struct st_fail_rec { ++ int count; ++}; ++ ++static unsigned long int stonith_max_attempts = 10; ++static GHashTable *stonith_failures = NULL; ++ ++void ++update_stonith_max_attempts(const char *value) ++{ ++ if (safe_str_eq(value, CRM_INFINITY_S)) { ++ stonith_max_attempts = CRM_SCORE_INFINITY; ++ } else { ++ stonith_max_attempts = crm_int_helper(value, NULL); ++ } ++} ++ ++static gboolean ++too_many_st_failures(const char *target) ++{ ++ GHashTableIter iter; ++ const char *key = NULL; ++ struct st_fail_rec *value = NULL; ++ ++ if (stonith_failures == NULL) { ++ return FALSE; ++ } ++ ++ if (target == NULL) { ++ g_hash_table_iter_init(&iter, stonith_failures); ++ while (g_hash_table_iter_next(&iter, (gpointer *) &key, ++ (gpointer *) &value)) { ++ ++ if (value->count >= stonith_max_attempts) { ++ target = (const char*)key; ++ goto too_many; ++ } ++ } ++ } else { ++ value = g_hash_table_lookup(stonith_failures, target); ++ if ((value != NULL) && (value->count >= stonith_max_attempts)) { ++ goto too_many; ++ } ++ } ++ return FALSE; ++ ++too_many: ++ crm_warn("Too many failures (%d) to fence %s, giving up", ++ value->count, target); ++ return TRUE; ++} ++ ++/*! ++ * \internal ++ * \brief Reset a stonith fail count ++ * ++ * \param[in] target Name of node to reset, or NULL for all ++ */ ++void ++st_fail_count_reset(const char *target) ++{ ++ if (stonith_failures == NULL) { ++ return; ++ } ++ ++ if (target) { ++ struct st_fail_rec *rec = NULL; ++ ++ rec = g_hash_table_lookup(stonith_failures, target); ++ if (rec) { ++ rec->count = 0; ++ } ++ } else { ++ GHashTableIter iter; ++ const char *key = NULL; ++ struct st_fail_rec *rec = NULL; ++ ++ g_hash_table_iter_init(&iter, stonith_failures); ++ while (g_hash_table_iter_next(&iter, (gpointer *) &key, ++ (gpointer *) &rec)) { ++ rec->count = 0; ++ } ++ } ++} ++ ++static void ++st_fail_count_increment(const char *target) ++{ ++ struct st_fail_rec *rec = NULL; ++ ++ if (stonith_failures == NULL) { ++ stonith_failures = crm_str_table_new(); ++ } ++ ++ rec = g_hash_table_lookup(stonith_failures, target); ++ if (rec) { ++ rec->count++; ++ } else { ++ rec = malloc(sizeof(struct st_fail_rec)); ++ if(rec == NULL) { ++ return; ++ } ++ ++ rec->count = 1; ++ g_hash_table_insert(stonith_failures, strdup(target), rec); ++ } ++} ++ ++/* end stonith fail count functions */ ++ ++ ++static void ++cib_fencing_updated(xmlNode *msg, int call_id, int rc, xmlNode *output, ++ void *user_data) ++{ ++ if (rc < pcmk_ok) { ++ crm_err("Fencing update %d for %s: failed - %s (%d)", ++ call_id, (char *)user_data, pcmk_strerror(rc), rc); ++ crm_log_xml_warn(msg, "Failed update"); ++ abort_transition(INFINITY, tg_shutdown, "CIB update failed", NULL); ++ ++ } else { ++ crm_info("Fencing update %d for %s: complete", call_id, (char *)user_data); ++ } ++} ++ ++static void ++send_stonith_update(crm_action_t *action, const char *target, const char *uuid) ++{ ++ int rc = pcmk_ok; ++ crm_node_t *peer = NULL; ++ ++ /* We (usually) rely on the membership layer to do node_update_cluster, ++ * and the peer status callback to do node_update_peer, because the node ++ * might have already rejoined before we get the stonith result here. ++ */ ++ int flags = node_update_join | node_update_expected; ++ ++ /* zero out the node-status & remove all LRM status info */ ++ xmlNode *node_state = NULL; ++ ++ CRM_CHECK(target != NULL, return); ++ CRM_CHECK(uuid != NULL, return); ++ ++ /* Make sure the membership and join caches are accurate */ ++ peer = crm_get_peer_full(0, target, CRM_GET_PEER_ANY); ++ ++ CRM_CHECK(peer != NULL, return); ++ ++ if (peer->state == NULL) { ++ /* Usually, we rely on the membership layer to update the cluster state ++ * in the CIB. However, if the node has never been seen, do it here, so ++ * the node is not considered unclean. ++ */ ++ flags |= node_update_cluster; ++ } ++ ++ if (peer->uuid == NULL) { ++ crm_info("Recording uuid '%s' for node '%s'", uuid, target); ++ peer->uuid = strdup(uuid); ++ } ++ ++ crmd_peer_down(peer, TRUE); ++ ++ /* Generate a node state update for the CIB */ ++ node_state = create_node_state_update(peer, flags, NULL, __FUNCTION__); ++ ++ /* we have to mark whether or not remote nodes have already been fenced */ ++ if (peer->flags & crm_remote_node) { ++ time_t now = time(NULL); ++ char *now_s = crm_itoa(now); ++ crm_xml_add(node_state, XML_NODE_IS_FENCED, now_s); ++ free(now_s); ++ } ++ ++ /* Force our known ID */ ++ crm_xml_add(node_state, XML_ATTR_UUID, uuid); ++ ++ rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, node_state, ++ cib_quorum_override | cib_scope_local | cib_can_create); ++ ++ /* Delay processing the trigger until the update completes */ ++ crm_debug("Sending fencing update %d for %s", rc, target); ++ fsa_register_cib_callback(rc, FALSE, strdup(target), cib_fencing_updated); ++ ++ /* Make sure it sticks */ ++ /* fsa_cib_conn->cmds->bump_epoch(fsa_cib_conn, cib_quorum_override|cib_scope_local); */ ++ ++ erase_status_tag(peer->uname, XML_CIB_TAG_LRM, cib_scope_local); ++ erase_status_tag(peer->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local); ++ ++ free_xml(node_state); ++ return; ++} ++ ++/*! ++ * \internal ++ * \brief Abort transition due to stonith failure ++ * ++ * \param[in] abort_action Whether to restart or stop transition ++ * \param[in] target Don't restart if this (NULL for any) has too many failures ++ * \param[in] reason Log this stonith action XML as abort reason (or NULL) ++ */ ++static void ++abort_for_stonith_failure(enum transition_action abort_action, ++ const char *target, xmlNode *reason) ++{ ++ /* If stonith repeatedly fails, we eventually give up on starting a new ++ * transition for that reason. ++ */ ++ if ((abort_action != tg_stop) && too_many_st_failures(target)) { ++ abort_action = tg_stop; ++ } ++ abort_transition(INFINITY, abort_action, "Stonith failed", reason); ++} ++ ++ ++/* ++ * stonith cleanup list ++ * ++ * If the DC is shot, proper notifications might not go out. ++ * The stonith cleanup list allows the cluster to (re-)send ++ * notifications once a new DC is elected. ++ */ ++ ++static GListPtr stonith_cleanup_list = NULL; ++ ++/*! ++ * \internal ++ * \brief Add a node to the stonith cleanup list ++ * ++ * \param[in] target Name of node to add ++ */ ++void ++add_stonith_cleanup(const char *target) { ++ stonith_cleanup_list = g_list_append(stonith_cleanup_list, strdup(target)); ++} ++ ++/*! ++ * \internal ++ * \brief Remove a node from the stonith cleanup list ++ * ++ * \param[in] Name of node to remove ++ */ ++void ++remove_stonith_cleanup(const char *target) ++{ ++ GListPtr iter = stonith_cleanup_list; ++ ++ while (iter != NULL) { ++ GListPtr tmp = iter; ++ char *iter_name = tmp->data; ++ ++ iter = iter->next; ++ if (safe_str_eq(target, iter_name)) { ++ crm_trace("Removing %s from the cleanup list", iter_name); ++ stonith_cleanup_list = g_list_delete_link(stonith_cleanup_list, tmp); ++ free(iter_name); ++ } ++ } ++} ++ ++/*! ++ * \internal ++ * \brief Purge all entries from the stonith cleanup list ++ */ ++void ++purge_stonith_cleanup() ++{ ++ if (stonith_cleanup_list) { ++ GListPtr iter = NULL; ++ ++ for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) { ++ char *target = iter->data; ++ ++ crm_info("Purging %s from stonith cleanup list", target); ++ free(target); ++ } ++ g_list_free(stonith_cleanup_list); ++ stonith_cleanup_list = NULL; ++ } ++} ++ ++/*! ++ * \internal ++ * \brief Send stonith updates for all entries in cleanup list, then purge it ++ */ ++void ++execute_stonith_cleanup() ++{ ++ GListPtr iter; ++ ++ for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) { ++ char *target = iter->data; ++ crm_node_t *target_node = crm_get_peer(0, target); ++ const char *uuid = crm_peer_uuid(target_node); ++ ++ crm_notice("Marking %s, target of a previous stonith action, as clean", target); ++ send_stonith_update(NULL, target, uuid); ++ free(target); ++ } ++ g_list_free(stonith_cleanup_list); ++ stonith_cleanup_list = NULL; ++} ++ ++/* end stonith cleanup list functions */ ++ ++ ++/* stonith API client ++ * ++ * Functions that need to interact directly with the fencer via its API ++ */ ++ ++stonith_t *stonith_api = NULL; ++crm_trigger_t *stonith_reconnect = NULL; ++char *te_client_id = NULL; ++ ++static gboolean ++fail_incompletable_stonith(crm_graph_t *graph) ++{ ++ GListPtr lpc = NULL; ++ const char *task = NULL; ++ xmlNode *last_action = NULL; ++ ++ if (graph == NULL) { ++ return FALSE; ++ } ++ ++ for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) { ++ GListPtr lpc2 = NULL; ++ synapse_t *synapse = (synapse_t *) lpc->data; ++ ++ if (synapse->confirmed) { ++ continue; ++ } ++ ++ for (lpc2 = synapse->actions; lpc2 != NULL; lpc2 = lpc2->next) { ++ crm_action_t *action = (crm_action_t *) lpc2->data; ++ ++ if (action->type != action_type_crm || action->confirmed) { ++ continue; ++ } ++ ++ task = crm_element_value(action->xml, XML_LRM_ATTR_TASK); ++ if (task && safe_str_eq(task, CRM_OP_FENCE)) { ++ action->failed = TRUE; ++ last_action = action->xml; ++ update_graph(graph, action); ++ crm_notice("Failing action %d (%s): fencer terminated", ++ action->id, ID(action->xml)); ++ } ++ } ++ } ++ ++ if (last_action != NULL) { ++ crm_warn("Fencer failure resulted in unrunnable actions"); ++ abort_for_stonith_failure(tg_restart, NULL, last_action); ++ return TRUE; ++ } ++ ++ return FALSE; ++} ++ ++static void ++tengine_stonith_connection_destroy(stonith_t *st, stonith_event_t *e) ++{ ++ if (is_set(fsa_input_register, R_ST_REQUIRED)) { ++ crm_crit("Fencing daemon connection failed"); ++ mainloop_set_trigger(stonith_reconnect); ++ ++ } else { ++ crm_info("Fencing daemon disconnected"); ++ } ++ ++ if (stonith_api) { ++ stonith_api->state = stonith_disconnected; ++ } ++ ++ if (AM_I_DC) { ++ fail_incompletable_stonith(transition_graph); ++ trigger_graph(); ++ } ++} ++ ++static void ++tengine_stonith_notify(stonith_t *st, stonith_event_t *st_event) ++{ ++ if (te_client_id == NULL) { ++ te_client_id = crm_strdup_printf("%s.%lu", crm_system_name, ++ (unsigned long) getpid()); ++ } ++ ++ if (st_event == NULL) { ++ crm_err("Notify data not found"); ++ return; ++ } ++ ++ crmd_alert_fencing_op(st_event); ++ ++ if ((st_event->result == pcmk_ok) && safe_str_eq("on", st_event->action)) { ++ crm_notice("%s was successfully unfenced by %s (at the request of %s)", ++ st_event->target, ++ st_event->executioner? st_event->executioner : "", ++ st_event->origin); ++ /* TODO: Hook up st_event->device */ ++ return; ++ ++ } else if (safe_str_eq("on", st_event->action)) { ++ crm_err("Unfencing of %s by %s failed: %s (%d)", ++ st_event->target, ++ st_event->executioner? st_event->executioner : "", ++ pcmk_strerror(st_event->result), st_event->result); ++ return; ++ ++ } else if ((st_event->result == pcmk_ok) ++ && crm_str_eq(st_event->target, fsa_our_uname, TRUE)) { ++ ++ crm_crit("We were allegedly just fenced by %s for %s!", ++ st_event->executioner? st_event->executioner : "", ++ st_event->origin); /* Dumps blackbox if enabled */ ++ ++ qb_log_fini(); /* Try to get the above log message to disk - somehow */ ++ ++ /* Get out ASAP and do not come back up. ++ * ++ * Triggering a reboot is also not the worst idea either since ++ * the rest of the cluster thinks we're safely down ++ */ ++ ++#ifdef RB_HALT_SYSTEM ++ reboot(RB_HALT_SYSTEM); ++#endif ++ ++ /* ++ * If reboot() fails or is not supported, coming back up will ++ * probably lead to a situation where the other nodes set our ++ * status to 'lost' because of the fencing callback and will ++ * discard subsequent election votes with: ++ * ++ * Election 87 (current: 5171, owner: 103): Processed vote from east-03 (Peer is not part of our cluster) ++ * ++ * So just stay dead, something is seriously messed up anyway. ++ * ++ */ ++ exit(CRM_EX_FATAL); // None of our wrappers since we already called qb_log_fini() ++ return; ++ } ++ ++ /* Update the count of stonith failures for this target, in case we become ++ * DC later. The current DC has already updated its fail count in ++ * tengine_stonith_callback(). ++ */ ++ if (!AM_I_DC && safe_str_eq(st_event->operation, T_STONITH_NOTIFY_FENCE)) { ++ if (st_event->result == pcmk_ok) { ++ st_fail_count_reset(st_event->target); ++ } else { ++ st_fail_count_increment(st_event->target); ++ } ++ } ++ ++ crm_notice("Peer %s was%s terminated (%s) by %s on behalf of %s: %s " ++ CRM_XS " initiator=%s ref=%s", ++ st_event->target, st_event->result == pcmk_ok ? "" : " not", ++ st_event->action, ++ st_event->executioner ? st_event->executioner : "", ++ (st_event->client_origin? st_event->client_origin : ""), ++ pcmk_strerror(st_event->result), ++ st_event->origin, st_event->id); ++ ++ if (st_event->result == pcmk_ok) { ++ crm_node_t *peer = crm_find_known_peer_full(0, st_event->target, CRM_GET_PEER_ANY); ++ const char *uuid = NULL; ++ gboolean we_are_executioner = safe_str_eq(st_event->executioner, fsa_our_uname); ++ ++ if (peer == NULL) { ++ return; ++ } ++ ++ uuid = crm_peer_uuid(peer); ++ ++ crm_trace("target=%s dc=%s", st_event->target, fsa_our_dc); ++ if(AM_I_DC) { ++ /* The DC always sends updates */ ++ send_stonith_update(NULL, st_event->target, uuid); ++ ++ /* @TODO Ideally, at this point, we'd check whether the fenced node ++ * hosted any guest nodes, and call remote_node_down() for them. ++ * Unfortunately, the controller doesn't have a simple, reliable way ++ * to map hosts to guests. It might be possible to track this in the ++ * peer cache via crm_remote_peer_cache_refresh(). For now, we rely ++ * on the PE creating fence pseudo-events for the guests. ++ */ ++ ++ if (st_event->client_origin ++ && safe_str_neq(st_event->client_origin, te_client_id)) { ++ ++ /* Abort the current transition graph if it wasn't us ++ * that invoked stonith to fence someone ++ */ ++ crm_info("External fencing operation from %s fenced %s", st_event->client_origin, st_event->target); ++ abort_transition(INFINITY, tg_restart, "External Fencing Operation", NULL); ++ } ++ ++ /* Assume it was our leader if we don't currently have one */ ++ } else if (((fsa_our_dc == NULL) || safe_str_eq(fsa_our_dc, st_event->target)) ++ && is_not_set(peer->flags, crm_remote_node)) { ++ ++ crm_notice("Target %s our leader %s (recorded: %s)", ++ fsa_our_dc ? "was" : "may have been", st_event->target, ++ fsa_our_dc ? fsa_our_dc : ""); ++ ++ /* Given the CIB resyncing that occurs around elections, ++ * have one node update the CIB now and, if the new DC is different, ++ * have them do so too after the election ++ */ ++ if (we_are_executioner) { ++ send_stonith_update(NULL, st_event->target, uuid); ++ } ++ add_stonith_cleanup(st_event->target); ++ } ++ ++ /* If the target is a remote node, and we host its connection, ++ * immediately fail all monitors so it can be recovered quickly. ++ * The connection won't necessarily drop when a remote node is fenced, ++ * so the failure might not otherwise be detected until the next poke. ++ */ ++ if (is_set(peer->flags, crm_remote_node)) { ++ remote_ra_fail(st_event->target); ++ } ++ ++ crmd_peer_down(peer, TRUE); ++ } ++} ++ ++/*! ++ * \brief Connect to fencer ++ * ++ * \param[in] user_data If NULL, retry failures now, otherwise retry in main loop ++ * ++ * \return TRUE ++ * \note If user_data is NULL, this will wait 2s between attempts, for up to ++ * 30 attempts, meaning the controller could be blocked as long as 58s. ++ */ ++gboolean ++te_connect_stonith(gpointer user_data) ++{ ++ int rc = pcmk_ok; ++ ++ if (stonith_api == NULL) { ++ stonith_api = stonith_api_new(); ++ } ++ ++ if (stonith_api->state != stonith_disconnected) { ++ crm_trace("Already connected to fencer, no need to retry"); ++ return TRUE; ++ } ++ ++ if (user_data == NULL) { ++ // Blocking (retry failures now until successful) ++ rc = stonith_api_connect_retry(stonith_api, crm_system_name, 30); ++ if (rc != pcmk_ok) { ++ crm_err("Could not connect to fencer in 30 attempts: %s " ++ CRM_XS " rc=%d", pcmk_strerror(rc), rc); ++ } ++ } else { ++ // Non-blocking (retry failures later in main loop) ++ rc = stonith_api->cmds->connect(stonith_api, crm_system_name, NULL); ++ if (rc != pcmk_ok) { ++ if (is_set(fsa_input_register, R_ST_REQUIRED)) { ++ crm_err("Fencer connection failed (will retry): %s " ++ CRM_XS " rc=%d", pcmk_strerror(rc), rc); ++ mainloop_set_trigger(stonith_reconnect); ++ } else { ++ crm_info("Fencer connection failed (ignoring because no longer required): %s " ++ CRM_XS " rc=%d", pcmk_strerror(rc), rc); ++ } ++ return TRUE; ++ } ++ } ++ ++ if (rc == pcmk_ok) { ++ stonith_api->cmds->register_notification(stonith_api, ++ T_STONITH_NOTIFY_DISCONNECT, ++ tengine_stonith_connection_destroy); ++ stonith_api->cmds->register_notification(stonith_api, ++ T_STONITH_NOTIFY_FENCE, ++ tengine_stonith_notify); ++ } ++ return TRUE; ++} ++ ++static gboolean ++do_stonith_history_sync(gpointer user_data) ++{ ++ if (stonith_api && (stonith_api->state != stonith_disconnected)) { ++ stonith_history_t *history = NULL; ++ ++ stonith_api->cmds->history(stonith_api, ++ st_opt_sync_call | st_opt_broadcast, ++ NULL, &history, 5); ++ stonith_history_free(history); ++ return TRUE; ++ } else { ++ crm_info("Skip triggering stonith history-sync as stonith is disconnected"); ++ return FALSE; ++ } ++} ++ ++static void ++tengine_stonith_callback(stonith_t *stonith, stonith_callback_data_t *data) ++{ ++ char *uuid = NULL; ++ int stonith_id = -1; ++ int transition_id = -1; ++ crm_action_t *action = NULL; ++ int call_id = data->call_id; ++ int rc = data->rc; ++ char *userdata = data->userdata; ++ ++ CRM_CHECK(userdata != NULL, return); ++ crm_notice("Stonith operation %d/%s: %s (%d)", call_id, (char *)userdata, ++ pcmk_strerror(rc), rc); ++ ++ if (AM_I_DC == FALSE) { ++ return; ++ } ++ ++ /* crm_info("call=%d, optype=%d, node_name=%s, result=%d, node_list=%s, action=%s", */ ++ /* op->call_id, op->optype, op->node_name, op->op_result, */ ++ /* (char *)op->node_list, op->private_data); */ ++ ++ /* filter out old STONITH actions */ ++ CRM_CHECK(decode_transition_key(userdata, &uuid, &transition_id, &stonith_id, NULL), ++ goto bail); ++ ++ if (transition_graph->complete || stonith_id < 0 || safe_str_neq(uuid, te_uuid) ++ || transition_graph->id != transition_id) { ++ crm_info("Ignoring STONITH action initiated outside of the current transition"); ++ goto bail; ++ } ++ ++ action = get_action(stonith_id, FALSE); ++ if (action == NULL) { ++ crm_err("Stonith action not matched"); ++ goto bail; ++ } ++ ++ stop_te_timer(action->timer); ++ if (rc == pcmk_ok) { ++ const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET); ++ const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); ++ const char *op = crm_meta_value(action->params, "stonith_action"); ++ ++ crm_info("Stonith operation %d for %s passed", call_id, target); ++ if (action->confirmed == FALSE) { ++ te_action_confirmed(action); ++ if (safe_str_eq("on", op)) { ++ const char *value = NULL; ++ char *now = crm_itoa(time(NULL)); ++ ++ update_attrd(target, CRM_ATTR_UNFENCED, now, NULL, FALSE); ++ free(now); ++ ++ value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_ALL); ++ update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL, FALSE); ++ ++ value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_SECURE); ++ update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL, FALSE); ++ ++ } else if (action->sent_update == FALSE) { ++ send_stonith_update(action, target, uuid); ++ action->sent_update = TRUE; ++ } ++ } ++ st_fail_count_reset(target); ++ ++ } else { ++ const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET); ++ enum transition_action abort_action = tg_restart; ++ ++ action->failed = TRUE; ++ crm_notice("Stonith operation %d for %s failed (%s): aborting transition.", ++ call_id, target, pcmk_strerror(rc)); ++ ++ /* If no fence devices were available, there's no use in immediately ++ * checking again, so don't start a new transition in that case. ++ */ ++ if (rc == -ENODEV) { ++ crm_warn("No devices found in cluster to fence %s, giving up", ++ target); ++ abort_action = tg_stop; ++ } ++ ++ /* Increment the fail count now, so abort_for_stonith_failure() can ++ * check it. Non-DC nodes will increment it in tengine_stonith_notify(). ++ */ ++ st_fail_count_increment(target); ++ abort_for_stonith_failure(abort_action, target, NULL); ++ } ++ ++ update_graph(transition_graph, action); ++ trigger_graph(); ++ ++ bail: ++ free(userdata); ++ free(uuid); ++ return; ++} ++ ++gboolean ++te_fence_node(crm_graph_t *graph, crm_action_t *action) ++{ ++ int rc = 0; ++ const char *id = NULL; ++ const char *uuid = NULL; ++ const char *target = NULL; ++ const char *type = NULL; ++ gboolean invalid_action = FALSE; ++ enum stonith_call_options options = st_opt_none; ++ ++ id = ID(action->xml); ++ target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET); ++ uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); ++ type = crm_meta_value(action->params, "stonith_action"); ++ ++ CRM_CHECK(id != NULL, invalid_action = TRUE); ++ CRM_CHECK(uuid != NULL, invalid_action = TRUE); ++ CRM_CHECK(type != NULL, invalid_action = TRUE); ++ CRM_CHECK(target != NULL, invalid_action = TRUE); ++ ++ if (invalid_action) { ++ crm_log_xml_warn(action->xml, "BadAction"); ++ return FALSE; ++ } ++ ++ crm_notice("Requesting fencing (%s) of node %s " ++ CRM_XS " action=%s timeout=%d", ++ type, target, id, transition_graph->stonith_timeout); ++ ++ /* Passing NULL means block until we can connect... */ ++ te_connect_stonith(NULL); ++ ++ if (crmd_join_phase_count(crm_join_confirmed) == 1) { ++ options |= st_opt_allow_suicide; ++ } ++ ++ rc = stonith_api->cmds->fence(stonith_api, options, target, type, ++ transition_graph->stonith_timeout / 1000, 0); ++ ++ stonith_api->cmds->register_callback(stonith_api, rc, transition_graph->stonith_timeout / 1000, ++ st_opt_timeout_updates, ++ generate_transition_key(transition_graph->id, action->id, ++ 0, te_uuid), ++ "tengine_stonith_callback", tengine_stonith_callback); ++ ++ return TRUE; ++} ++ ++/* end stonith API client functions */ ++ ++ ++/* ++ * stonith history synchronization ++ * ++ * Each node's fencer keeps track of a cluster-wide fencing history. When a node ++ * joins or leaves, we need to synchronize the history across all nodes. ++ */ ++ ++static crm_trigger_t *stonith_history_sync_trigger = NULL; ++static mainloop_timer_t *stonith_history_sync_timer = NULL; ++ ++static gboolean ++stonith_history_sync_set_trigger(gpointer user_data) ++{ ++ mainloop_set_trigger(stonith_history_sync_trigger); ++ return FALSE; ++} ++ ++void ++te_trigger_stonith_history_sync(void) ++{ ++ /* trigger a sync in 5s to give more nodes the ++ * chance to show up so that we don't create ++ * unnecessary stonith-history-sync traffic ++ */ ++ ++ /* as we are finally checking the stonith-connection ++ * in do_stonith_history_sync we should be fine ++ * leaving stonith_history_sync_time & stonith_history_sync_trigger ++ * around ++ */ ++ if (stonith_history_sync_trigger == NULL) { ++ stonith_history_sync_trigger = ++ mainloop_add_trigger(G_PRIORITY_LOW, ++ do_stonith_history_sync, NULL); ++ } ++ ++ if(stonith_history_sync_timer == NULL) { ++ stonith_history_sync_timer = ++ mainloop_timer_add("history_sync", 5000, ++ FALSE, stonith_history_sync_set_trigger, ++ NULL); ++ } ++ crm_info("Fence history will be synchronized cluster-wide within 5 seconds"); ++ mainloop_timer_start(stonith_history_sync_timer); ++} ++ ++/* end stonith history synchronization functions */ +diff --git a/daemons/controld/controld_fencing.h b/daemons/controld/controld_fencing.h +new file mode 100644 +index 0000000..b80a6c9 +--- /dev/null ++++ b/daemons/controld/controld_fencing.h +@@ -0,0 +1,37 @@ ++/* ++ * Copyright 2004-2019 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. ++ * ++ * This source code is licensed under the GNU Lesser General Public License ++ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. ++ */ ++ ++#ifndef CONTROLD_FENCING__H ++# define CONTROLD_FENCING__H ++ ++#include // bool ++#include // crm_graph_t, crm_action_t ++ ++extern crm_trigger_t *stonith_reconnect; ++extern char *te_client_id; ++extern stonith_t *stonith_api; ++ ++// stonith fail counts ++void st_fail_count_reset(const char * target); ++void update_stonith_max_attempts(const char* value); ++ ++// stonith API client ++gboolean te_connect_stonith(gpointer user_data); ++gboolean te_fence_node(crm_graph_t *graph, crm_action_t *action); ++ ++// stonith cleanup list ++void add_stonith_cleanup(const char *target); ++void remove_stonith_cleanup(const char *target); ++void purge_stonith_cleanup(void); ++void execute_stonith_cleanup(void); ++ ++// stonith history synchronization ++void te_trigger_stonith_history_sync(void); ++ ++#endif +diff --git a/daemons/controld/controld_fsa.c b/daemons/controld/controld_fsa.c +index 9eca530..dc1937f 100644 +--- a/daemons/controld/controld_fsa.c ++++ b/daemons/controld/controld_fsa.c +@@ -26,6 +26,7 @@ + #include + #include + #include ++#include + #include + #include + +diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c +index 2ebc203..8f37cbf 100644 +--- a/daemons/controld/controld_messages.c ++++ b/daemons/controld/controld_messages.c +@@ -25,6 +25,7 @@ + #include + #include + #include ++#include + #include + #include + +diff --git a/daemons/controld/controld_te_actions.c b/daemons/controld/controld_te_actions.c +index c95c6c7..2f61556 100644 +--- a/daemons/controld/controld_te_actions.c ++++ b/daemons/controld/controld_te_actions.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2018 Andrew Beekhof ++ * Copyright 2004-2019 the Pacemaker project contributors + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. +@@ -17,6 +17,7 @@ + + #include + #include ++#include + #include + #include + #include +@@ -76,124 +77,6 @@ te_pseudo_action(crm_graph_t * graph, crm_action_t * pseudo) + return TRUE; + } + +-void +-send_stonith_update(crm_action_t * action, const char *target, const char *uuid) +-{ +- int rc = pcmk_ok; +- crm_node_t *peer = NULL; +- +- /* We (usually) rely on the membership layer to do node_update_cluster, +- * and the peer status callback to do node_update_peer, because the node +- * might have already rejoined before we get the stonith result here. +- */ +- int flags = node_update_join | node_update_expected; +- +- /* zero out the node-status & remove all LRM status info */ +- xmlNode *node_state = NULL; +- +- CRM_CHECK(target != NULL, return); +- CRM_CHECK(uuid != NULL, return); +- +- /* Make sure the membership and join caches are accurate */ +- peer = crm_get_peer_full(0, target, CRM_GET_PEER_ANY); +- +- CRM_CHECK(peer != NULL, return); +- +- if (peer->state == NULL) { +- /* Usually, we rely on the membership layer to update the cluster state +- * in the CIB. However, if the node has never been seen, do it here, so +- * the node is not considered unclean. +- */ +- flags |= node_update_cluster; +- } +- +- if (peer->uuid == NULL) { +- crm_info("Recording uuid '%s' for node '%s'", uuid, target); +- peer->uuid = strdup(uuid); +- } +- +- crmd_peer_down(peer, TRUE); +- +- /* Generate a node state update for the CIB */ +- node_state = create_node_state_update(peer, flags, NULL, __FUNCTION__); +- +- /* we have to mark whether or not remote nodes have already been fenced */ +- if (peer->flags & crm_remote_node) { +- time_t now = time(NULL); +- char *now_s = crm_itoa(now); +- crm_xml_add(node_state, XML_NODE_IS_FENCED, now_s); +- free(now_s); +- } +- +- /* Force our known ID */ +- crm_xml_add(node_state, XML_ATTR_UUID, uuid); +- +- rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, node_state, +- cib_quorum_override | cib_scope_local | cib_can_create); +- +- /* Delay processing the trigger until the update completes */ +- crm_debug("Sending fencing update %d for %s", rc, target); +- fsa_register_cib_callback(rc, FALSE, strdup(target), cib_fencing_updated); +- +- /* Make sure it sticks */ +- /* fsa_cib_conn->cmds->bump_epoch(fsa_cib_conn, cib_quorum_override|cib_scope_local); */ +- +- erase_status_tag(peer->uname, XML_CIB_TAG_LRM, cib_scope_local); +- erase_status_tag(peer->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local); +- +- free_xml(node_state); +- return; +-} +- +-static gboolean +-te_fence_node(crm_graph_t * graph, crm_action_t * action) +-{ +- int rc = 0; +- const char *id = NULL; +- const char *uuid = NULL; +- const char *target = NULL; +- const char *type = NULL; +- gboolean invalid_action = FALSE; +- enum stonith_call_options options = st_opt_none; +- +- id = ID(action->xml); +- target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET); +- uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); +- type = crm_meta_value(action->params, "stonith_action"); +- +- CRM_CHECK(id != NULL, invalid_action = TRUE); +- CRM_CHECK(uuid != NULL, invalid_action = TRUE); +- CRM_CHECK(type != NULL, invalid_action = TRUE); +- CRM_CHECK(target != NULL, invalid_action = TRUE); +- +- if (invalid_action) { +- crm_log_xml_warn(action->xml, "BadAction"); +- return FALSE; +- } +- +- crm_notice("Requesting fencing (%s) of node %s " +- CRM_XS " action=%s timeout=%d", +- type, target, id, transition_graph->stonith_timeout); +- +- /* Passing NULL means block until we can connect... */ +- te_connect_stonith(NULL); +- +- if (crmd_join_phase_count(crm_join_confirmed) == 1) { +- options |= st_opt_allow_suicide; +- } +- +- rc = stonith_api->cmds->fence(stonith_api, options, target, type, +- transition_graph->stonith_timeout / 1000, 0); +- +- stonith_api->cmds->register_callback(stonith_api, rc, transition_graph->stonith_timeout / 1000, +- st_opt_timeout_updates, +- generate_transition_key(transition_graph->id, action->id, +- 0, te_uuid), +- "tengine_stonith_callback", tengine_stonith_callback); +- +- return TRUE; +-} +- + static int + get_target_rc(crm_action_t * action) + { +diff --git a/daemons/controld/controld_te_callbacks.c b/daemons/controld/controld_te_callbacks.c +index 22b5f4b..1ab703f 100644 +--- a/daemons/controld/controld_te_callbacks.c ++++ b/daemons/controld/controld_te_callbacks.c +@@ -17,6 +17,7 @@ + + #include + #include ++#include + + #include /* For ONLINESTATUS etc */ + +@@ -27,21 +28,9 @@ gboolean shuttingdown = FALSE; + crm_graph_t *transition_graph; + crm_trigger_t *transition_trigger = NULL; + +-static unsigned long int stonith_max_attempts = 10; +- + /* #define RSC_OP_TEMPLATE "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_CIB_TAG_STATE"[@uname='%s']"//"XML_LRM_TAG_RSC_OP"[@id='%s]" */ + #define RSC_OP_TEMPLATE "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_LRM_TAG_RSC_OP"[@id='%s']" + +-void +-update_stonith_max_attempts(const char* value) +-{ +- if (safe_str_eq(value, CRM_INFINITY_S)) { +- stonith_max_attempts = CRM_SCORE_INFINITY; +- } +- else { +- stonith_max_attempts = crm_int_helper(value, NULL); +- } +-} + static void + te_update_diff_v1(const char *event, xmlNode *diff) + { +@@ -646,236 +635,6 @@ process_te_message(xmlNode * msg, xmlNode * xml_data) + return TRUE; + } + +-GHashTable *stonith_failures = NULL; +-struct st_fail_rec { +- int count; +-}; +- +-static gboolean +-too_many_st_failures(const char *target) +-{ +- GHashTableIter iter; +- const char *key = NULL; +- struct st_fail_rec *value = NULL; +- +- if (stonith_failures == NULL) { +- return FALSE; +- } +- +- if (target == NULL) { +- g_hash_table_iter_init(&iter, stonith_failures); +- while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) { +- if (value->count >= stonith_max_attempts) { +- target = (const char*)key; +- goto too_many; +- } +- } +- } else { +- value = g_hash_table_lookup(stonith_failures, target); +- if ((value != NULL) && (value->count >= stonith_max_attempts)) { +- goto too_many; +- } +- } +- return FALSE; +- +-too_many: +- crm_warn("Too many failures (%d) to fence %s, giving up", +- value->count, target); +- return TRUE; +-} +- +-/*! +- * \internal +- * \brief Reset a stonith fail count +- * +- * \param[in] target Name of node to reset, or NULL for all +- */ +-void +-st_fail_count_reset(const char *target) +-{ +- if (stonith_failures == NULL) { +- return; +- } +- +- if (target) { +- struct st_fail_rec *rec = NULL; +- +- rec = g_hash_table_lookup(stonith_failures, target); +- if (rec) { +- rec->count = 0; +- } +- } else { +- GHashTableIter iter; +- const char *key = NULL; +- struct st_fail_rec *rec = NULL; +- +- g_hash_table_iter_init(&iter, stonith_failures); +- while (g_hash_table_iter_next(&iter, (gpointer *) &key, +- (gpointer *) &rec)) { +- rec->count = 0; +- } +- } +-} +- +-void +-st_fail_count_increment(const char *target) +-{ +- struct st_fail_rec *rec = NULL; +- +- if (stonith_failures == NULL) { +- stonith_failures = crm_str_table_new(); +- } +- +- rec = g_hash_table_lookup(stonith_failures, target); +- if (rec) { +- rec->count++; +- } else { +- rec = malloc(sizeof(struct st_fail_rec)); +- if(rec == NULL) { +- return; +- } +- +- rec->count = 1; +- g_hash_table_insert(stonith_failures, strdup(target), rec); +- } +-} +- +-/*! +- * \internal +- * \brief Abort transition due to stonith failure +- * +- * \param[in] abort_action Whether to restart or stop transition +- * \param[in] target Don't restart if this (NULL for any) has too many failures +- * \param[in] reason Log this stonith action XML as abort reason (or NULL) +- */ +-void +-abort_for_stonith_failure(enum transition_action abort_action, +- const char *target, xmlNode *reason) +-{ +- /* If stonith repeatedly fails, we eventually give up on starting a new +- * transition for that reason. +- */ +- if ((abort_action != tg_stop) && too_many_st_failures(target)) { +- abort_action = tg_stop; +- } +- abort_transition(INFINITY, abort_action, "Stonith failed", reason); +-} +- +-void +-tengine_stonith_callback(stonith_t * stonith, stonith_callback_data_t * data) +-{ +- char *uuid = NULL; +- int stonith_id = -1; +- int transition_id = -1; +- crm_action_t *action = NULL; +- int call_id = data->call_id; +- int rc = data->rc; +- char *userdata = data->userdata; +- +- CRM_CHECK(userdata != NULL, return); +- crm_notice("Stonith operation %d/%s: %s (%d)", call_id, (char *)userdata, +- pcmk_strerror(rc), rc); +- +- if (AM_I_DC == FALSE) { +- return; +- } +- +- /* crm_info("call=%d, optype=%d, node_name=%s, result=%d, node_list=%s, action=%s", */ +- /* op->call_id, op->optype, op->node_name, op->op_result, */ +- /* (char *)op->node_list, op->private_data); */ +- +- /* filter out old STONITH actions */ +- CRM_CHECK(decode_transition_key(userdata, &uuid, &transition_id, &stonith_id, NULL), +- goto bail); +- +- if (transition_graph->complete || stonith_id < 0 || safe_str_neq(uuid, te_uuid) +- || transition_graph->id != transition_id) { +- crm_info("Ignoring STONITH action initiated outside of the current transition"); +- goto bail; +- } +- +- action = get_action(stonith_id, FALSE); +- if (action == NULL) { +- crm_err("Stonith action not matched"); +- goto bail; +- } +- +- stop_te_timer(action->timer); +- if (rc == pcmk_ok) { +- const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET); +- const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); +- const char *op = crm_meta_value(action->params, "stonith_action"); +- +- crm_info("Stonith operation %d for %s passed", call_id, target); +- if (action->confirmed == FALSE) { +- te_action_confirmed(action); +- if (safe_str_eq("on", op)) { +- const char *value = NULL; +- char *now = crm_itoa(time(NULL)); +- +- update_attrd(target, CRM_ATTR_UNFENCED, now, NULL, FALSE); +- free(now); +- +- value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_ALL); +- update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL, FALSE); +- +- value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_SECURE); +- update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL, FALSE); +- +- } else if (action->sent_update == FALSE) { +- send_stonith_update(action, target, uuid); +- action->sent_update = TRUE; +- } +- } +- st_fail_count_reset(target); +- +- } else { +- const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET); +- enum transition_action abort_action = tg_restart; +- +- action->failed = TRUE; +- crm_notice("Stonith operation %d for %s failed (%s): aborting transition.", +- call_id, target, pcmk_strerror(rc)); +- +- /* If no fence devices were available, there's no use in immediately +- * checking again, so don't start a new transition in that case. +- */ +- if (rc == -ENODEV) { +- crm_warn("No devices found in cluster to fence %s, giving up", +- target); +- abort_action = tg_stop; +- } +- +- /* Increment the fail count now, so abort_for_stonith_failure() can +- * check it. Non-DC nodes will increment it in tengine_stonith_notify(). +- */ +- st_fail_count_increment(target); +- abort_for_stonith_failure(abort_action, target, NULL); +- } +- +- update_graph(transition_graph, action); +- trigger_graph(); +- +- bail: +- free(userdata); +- free(uuid); +- return; +-} +- +-void +-cib_fencing_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) +-{ +- if (rc < pcmk_ok) { +- crm_err("Fencing update %d for %s: failed - %s (%d)", +- call_id, (char *)user_data, pcmk_strerror(rc), rc); +- crm_log_xml_warn(msg, "Failed update"); +- abort_transition(INFINITY, tg_shutdown, "CIB update failed", NULL); +- +- } else { +- crm_info("Fencing update %d for %s: complete", call_id, (char *)user_data); +- } +-} +- + void + cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) + { +diff --git a/daemons/controld/controld_te_utils.c b/daemons/controld/controld_te_utils.c +index 22f83ad..1496244 100644 +--- a/daemons/controld/controld_te_utils.c ++++ b/daemons/controld/controld_te_utils.c +@@ -6,441 +6,14 @@ + */ + + #include +- +-#include + #include +- + #include +- + #include ++ + #include + #include +-#include + #include + #include +-#include +- +-crm_trigger_t *stonith_reconnect = NULL; +-static crm_trigger_t *stonith_history_sync_trigger = NULL; +-static mainloop_timer_t *stonith_history_sync_timer = NULL; +- +-/* +- * stonith cleanup list +- * +- * If the DC is shot, proper notifications might not go out. +- * The stonith cleanup list allows the cluster to (re-)send +- * notifications once a new DC is elected. +- */ +- +-static GListPtr stonith_cleanup_list = NULL; +- +-/*! +- * \internal +- * \brief Add a node to the stonith cleanup list +- * +- * \param[in] target Name of node to add +- */ +-void +-add_stonith_cleanup(const char *target) { +- stonith_cleanup_list = g_list_append(stonith_cleanup_list, strdup(target)); +-} +- +-/*! +- * \internal +- * \brief Remove a node from the stonith cleanup list +- * +- * \param[in] Name of node to remove +- */ +-void +-remove_stonith_cleanup(const char *target) +-{ +- GListPtr iter = stonith_cleanup_list; +- +- while (iter != NULL) { +- GListPtr tmp = iter; +- char *iter_name = tmp->data; +- +- iter = iter->next; +- if (safe_str_eq(target, iter_name)) { +- crm_trace("Removing %s from the cleanup list", iter_name); +- stonith_cleanup_list = g_list_delete_link(stonith_cleanup_list, tmp); +- free(iter_name); +- } +- } +-} +- +-/*! +- * \internal +- * \brief Purge all entries from the stonith cleanup list +- */ +-void +-purge_stonith_cleanup() +-{ +- if (stonith_cleanup_list) { +- GListPtr iter = NULL; +- +- for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) { +- char *target = iter->data; +- +- crm_info("Purging %s from stonith cleanup list", target); +- free(target); +- } +- g_list_free(stonith_cleanup_list); +- stonith_cleanup_list = NULL; +- } +-} +- +-/*! +- * \internal +- * \brief Send stonith updates for all entries in cleanup list, then purge it +- */ +-void +-execute_stonith_cleanup() +-{ +- GListPtr iter; +- +- for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) { +- char *target = iter->data; +- crm_node_t *target_node = crm_get_peer(0, target); +- const char *uuid = crm_peer_uuid(target_node); +- +- crm_notice("Marking %s, target of a previous stonith action, as clean", target); +- send_stonith_update(NULL, target, uuid); +- free(target); +- } +- g_list_free(stonith_cleanup_list); +- stonith_cleanup_list = NULL; +-} +- +-/* end stonith cleanup list functions */ +- +-static gboolean +-fail_incompletable_stonith(crm_graph_t * graph) +-{ +- GListPtr lpc = NULL; +- const char *task = NULL; +- xmlNode *last_action = NULL; +- +- if (graph == NULL) { +- return FALSE; +- } +- +- for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) { +- GListPtr lpc2 = NULL; +- synapse_t *synapse = (synapse_t *) lpc->data; +- +- if (synapse->confirmed) { +- continue; +- } +- +- for (lpc2 = synapse->actions; lpc2 != NULL; lpc2 = lpc2->next) { +- crm_action_t *action = (crm_action_t *) lpc2->data; +- +- if (action->type != action_type_crm || action->confirmed) { +- continue; +- } +- +- task = crm_element_value(action->xml, XML_LRM_ATTR_TASK); +- if (task && safe_str_eq(task, CRM_OP_FENCE)) { +- action->failed = TRUE; +- last_action = action->xml; +- update_graph(graph, action); +- crm_notice("Failing action %d (%s): fencer terminated", +- action->id, ID(action->xml)); +- } +- } +- } +- +- if (last_action != NULL) { +- crm_warn("Fencer failure resulted in unrunnable actions"); +- abort_for_stonith_failure(tg_restart, NULL, last_action); +- return TRUE; +- } +- +- return FALSE; +-} +- +-static void +-tengine_stonith_connection_destroy(stonith_t * st, stonith_event_t * e) +-{ +- if (is_set(fsa_input_register, R_ST_REQUIRED)) { +- crm_crit("Fencing daemon connection failed"); +- mainloop_set_trigger(stonith_reconnect); +- +- } else { +- crm_info("Fencing daemon disconnected"); +- } +- +- /* cbchan will be garbage at this point, arrange for it to be reset */ +- if(stonith_api) { +- stonith_api->state = stonith_disconnected; +- } +- +- if (AM_I_DC) { +- fail_incompletable_stonith(transition_graph); +- trigger_graph(); +- } +-} +- +-char *te_client_id = NULL; +- +-#ifdef HAVE_SYS_REBOOT_H +-# include +-# include +-#endif +- +-static void +-tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event) +-{ +- if(te_client_id == NULL) { +- te_client_id = crm_strdup_printf("%s.%lu", crm_system_name, +- (unsigned long) getpid()); +- } +- +- if (st_event == NULL) { +- crm_err("Notify data not found"); +- return; +- } +- +- crmd_alert_fencing_op(st_event); +- +- if (st_event->result == pcmk_ok && safe_str_eq("on", st_event->action)) { +- crm_notice("%s was successfully unfenced by %s (at the request of %s)", +- st_event->target, st_event->executioner ? st_event->executioner : "", st_event->origin); +- /* TODO: Hook up st_event->device */ +- return; +- +- } else if (safe_str_eq("on", st_event->action)) { +- crm_err("Unfencing of %s by %s failed: %s (%d)", +- st_event->target, st_event->executioner ? st_event->executioner : "", +- pcmk_strerror(st_event->result), st_event->result); +- return; +- +- } else if (st_event->result == pcmk_ok && crm_str_eq(st_event->target, fsa_our_uname, TRUE)) { +- crm_crit("We were allegedly just fenced by %s for %s!", +- st_event->executioner ? st_event->executioner : "", st_event->origin); /* Dumps blackbox if enabled */ +- +- qb_log_fini(); /* Try to get the above log message to disk - somehow */ +- +- /* Get out ASAP and do not come back up. +- * +- * Triggering a reboot is also not the worst idea either since +- * the rest of the cluster thinks we're safely down +- */ +- +-#ifdef RB_HALT_SYSTEM +- reboot(RB_HALT_SYSTEM); +-#endif +- +- /* +- * If reboot() fails or is not supported, coming back up will +- * probably lead to a situation where the other nodes set our +- * status to 'lost' because of the fencing callback and will +- * discard subsequent election votes with: +- * +- * Election 87 (current: 5171, owner: 103): Processed vote from east-03 (Peer is not part of our cluster) +- * +- * So just stay dead, something is seriously messed up anyway. +- * +- */ +- exit(CRM_EX_FATAL); // None of our wrappers since we already called qb_log_fini() +- return; +- } +- +- /* Update the count of stonith failures for this target, in case we become +- * DC later. The current DC has already updated its fail count in +- * tengine_stonith_callback(). +- */ +- if (!AM_I_DC && safe_str_eq(st_event->operation, T_STONITH_NOTIFY_FENCE)) { +- if (st_event->result == pcmk_ok) { +- st_fail_count_reset(st_event->target); +- } else { +- st_fail_count_increment(st_event->target); +- } +- } +- +- crm_notice("Peer %s was%s terminated (%s) by %s on behalf of %s: %s " +- CRM_XS " initiator=%s ref=%s", +- st_event->target, st_event->result == pcmk_ok ? "" : " not", +- st_event->action, +- st_event->executioner ? st_event->executioner : "", +- (st_event->client_origin? st_event->client_origin : ""), +- pcmk_strerror(st_event->result), +- st_event->origin, st_event->id); +- +- if (st_event->result == pcmk_ok) { +- crm_node_t *peer = crm_find_known_peer_full(0, st_event->target, CRM_GET_PEER_ANY); +- const char *uuid = NULL; +- gboolean we_are_executioner = safe_str_eq(st_event->executioner, fsa_our_uname); +- +- if (peer == NULL) { +- return; +- } +- +- uuid = crm_peer_uuid(peer); +- +- crm_trace("target=%s dc=%s", st_event->target, fsa_our_dc); +- if(AM_I_DC) { +- /* The DC always sends updates */ +- send_stonith_update(NULL, st_event->target, uuid); +- +- /* @TODO Ideally, at this point, we'd check whether the fenced node +- * hosted any guest nodes, and call remote_node_down() for them. +- * Unfortunately, the controller doesn't have a simple, reliable way +- * to map hosts to guests. It might be possible to track this in the +- * peer cache via crm_remote_peer_cache_refresh(). For now, we rely +- * on the PE creating fence pseudo-events for the guests. +- */ +- +- if (st_event->client_origin && safe_str_neq(st_event->client_origin, te_client_id)) { +- +- /* Abort the current transition graph if it wasn't us +- * that invoked stonith to fence someone +- */ +- crm_info("External fencing operation from %s fenced %s", st_event->client_origin, st_event->target); +- abort_transition(INFINITY, tg_restart, "External Fencing Operation", NULL); +- } +- +- /* Assume it was our leader if we don't currently have one */ +- } else if (((fsa_our_dc == NULL) || safe_str_eq(fsa_our_dc, st_event->target)) +- && !is_set(peer->flags, crm_remote_node)) { +- +- crm_notice("Target %s our leader %s (recorded: %s)", +- fsa_our_dc ? "was" : "may have been", st_event->target, +- fsa_our_dc ? fsa_our_dc : ""); +- +- /* Given the CIB resyncing that occurs around elections, +- * have one node update the CIB now and, if the new DC is different, +- * have them do so too after the election +- */ +- if (we_are_executioner) { +- send_stonith_update(NULL, st_event->target, uuid); +- } +- add_stonith_cleanup(st_event->target); +- } +- +- /* If the target is a remote node, and we host its connection, +- * immediately fail all monitors so it can be recovered quickly. +- * The connection won't necessarily drop when a remote node is fenced, +- * so the failure might not otherwise be detected until the next poke. +- */ +- if (is_set(peer->flags, crm_remote_node)) { +- remote_ra_fail(st_event->target); +- } +- +- crmd_peer_down(peer, TRUE); +- } +-} +- +-static gboolean +-do_stonith_history_sync(gpointer user_data) +-{ +- if (stonith_api && (stonith_api->state != stonith_disconnected)) { +- stonith_history_t *history = NULL; +- +- stonith_api->cmds->history(stonith_api, +- st_opt_sync_call | st_opt_broadcast, +- NULL, &history, 5); +- stonith_history_free(history); +- return TRUE; +- } else { +- crm_info("Skip triggering stonith history-sync as stonith is disconnected"); +- return FALSE; +- } +-} +- +-static gboolean +-stonith_history_sync_set_trigger(gpointer user_data) +-{ +- mainloop_set_trigger(stonith_history_sync_trigger); +- return FALSE; +-} +- +-void +-te_trigger_stonith_history_sync(void) +-{ +- /* trigger a sync in 5s to give more nodes the +- * chance to show up so that we don't create +- * unnecessary stonith-history-sync traffic +- */ +- +- /* as we are finally checking the stonith-connection +- * in do_stonith_history_sync we should be fine +- * leaving stonith_history_sync_time & stonith_history_sync_trigger +- * around +- */ +- if (stonith_history_sync_trigger == NULL) { +- stonith_history_sync_trigger = +- mainloop_add_trigger(G_PRIORITY_LOW, +- do_stonith_history_sync, NULL); +- } +- +- if(stonith_history_sync_timer == NULL) { +- stonith_history_sync_timer = +- mainloop_timer_add("history_sync", 5000, +- FALSE, stonith_history_sync_set_trigger, +- NULL); +- } +- crm_info("Fence history will be synchronized cluster-wide within 5 seconds"); +- mainloop_timer_start(stonith_history_sync_timer); +-} +- +-/*! +- * \brief Connect to fencer +- * +- * \param[in] user_data If NULL, retry failures now, otherwise retry in main loop +- * +- * \return TRUE +- * \note If user_data is NULL, this will wait 2s between attempts, for up to +- * 30 attempts, meaning the controller could be blocked as long as 58s. +- */ +-gboolean +-te_connect_stonith(gpointer user_data) +-{ +- int rc = pcmk_ok; +- +- if (stonith_api == NULL) { +- stonith_api = stonith_api_new(); +- } +- +- if (stonith_api->state != stonith_disconnected) { +- crm_trace("Already connected to fencer, no need to retry"); +- return TRUE; +- } +- +- if (user_data == NULL) { +- // Blocking (retry failures now until successful) +- rc = stonith_api_connect_retry(stonith_api, crm_system_name, 30); +- if (rc != pcmk_ok) { +- crm_err("Could not connect to fencer in 30 attempts: %s " +- CRM_XS " rc=%d", pcmk_strerror(rc), rc); +- } +- } else { +- // Non-blocking (retry failures later in main loop) +- rc = stonith_api->cmds->connect(stonith_api, crm_system_name, NULL); +- if (rc != pcmk_ok) { +- if (is_set(fsa_input_register, R_ST_REQUIRED)) { +- crm_err("Fencer connection failed (will retry): %s " +- CRM_XS " rc=%d", pcmk_strerror(rc), rc); +- mainloop_set_trigger(stonith_reconnect); +- } else { +- crm_info("Fencer connection failed (ignoring because no longer required): %s " +- CRM_XS " rc=%d", pcmk_strerror(rc), rc); +- } +- return TRUE; +- } +- } +- +- if (rc == pcmk_ok) { +- stonith_api->cmds->register_notification(stonith_api, +- T_STONITH_NOTIFY_DISCONNECT, +- tengine_stonith_connection_destroy); +- stonith_api->cmds->register_notification(stonith_api, +- T_STONITH_NOTIFY_FENCE, +- tengine_stonith_notify); +- } +- return TRUE; +-} + + gboolean + stop_te_timer(crm_action_timer_t * timer) +diff --git a/daemons/controld/controld_transition.c b/daemons/controld/controld_transition.c +index 5f164ab..b942ab4 100644 +--- a/daemons/controld/controld_transition.c ++++ b/daemons/controld/controld_transition.c +@@ -18,7 +18,6 @@ + + + extern crm_graph_functions_t te_graph_fns; +-stonith_t *stonith_api = NULL; + + static void + global_cib_callback(const xmlNode * msg, int callid, int rc, xmlNode * output) +diff --git a/daemons/controld/controld_transition.h b/daemons/controld/controld_transition.h +index a162f99..f31ac2d 100644 +--- a/daemons/controld/controld_transition.h ++++ b/daemons/controld/controld_transition.h +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2018 Andrew Beekhof ++ * Copyright 2004-2019 the Pacemaker project contributors + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. +@@ -12,15 +12,6 @@ + # include + # include + # include +-extern stonith_t *stonith_api; +-extern void send_stonith_update(crm_action_t * stonith_action, const char *target, +- const char *uuid); +- +-/* stonith cleanup list */ +-void add_stonith_cleanup(const char *target); +-void remove_stonith_cleanup(const char *target); +-void purge_stonith_cleanup(void); +-void execute_stonith_cleanup(void); + + /* tengine */ + extern crm_action_t *match_down_event(const char *target); +@@ -46,16 +37,11 @@ extern char *te_uuid; + + extern void notify_crmd(crm_graph_t * graph); + +-void cib_fencing_updated(xmlNode *msg, int call_id, int rc, xmlNode *output, +- void *user_data); + void cib_action_updated(xmlNode *msg, int call_id, int rc, xmlNode *output, + void *user_data); + gboolean action_timer_callback(gpointer data); + gboolean te_graph_trigger(gpointer user_data); + void te_update_diff(const char *event, xmlNode *msg); +-void tengine_stonith_callback(stonith_t *stonith, +- stonith_callback_data_t *data); +-void update_stonith_max_attempts(const char* value); + + extern void trigger_graph_processing(const char *fn, int line); + void abort_after_delay(int abort_priority, enum transition_action abort_action, +@@ -68,12 +54,7 @@ extern void abort_transition_graph(int abort_priority, enum transition_action ab + # define abort_transition(pri, action, text, reason) \ + abort_transition_graph(pri, action, text, reason,__FUNCTION__,__LINE__); + +-extern gboolean te_connect_stonith(gpointer user_data); +- +-extern void te_trigger_stonith_history_sync(void); +- + extern crm_trigger_t *transition_trigger; +-extern crm_trigger_t *stonith_reconnect; + + extern char *failed_stop_offset; + extern char *failed_start_offset; +diff --git a/daemons/controld/controld_utils.h b/daemons/controld/controld_utils.h +index 68992f5..8b80e3c 100644 +--- a/daemons/controld/controld_utils.h ++++ b/daemons/controld/controld_utils.h +@@ -85,10 +85,6 @@ int crmd_join_phase_count(enum crm_join_phase phase); + void crmd_join_phase_log(int level); + + const char *get_timer_desc(fsa_timer_t * timer); +-void st_fail_count_reset(const char * target); +-void st_fail_count_increment(const char *target); +-void abort_for_stonith_failure(enum transition_action abort_action, +- const char *target, xmlNode *reason); + void crmd_peer_down(crm_node_t *peer, bool full); + unsigned int cib_op_timeout(void); + +-- +1.8.3.1 + +From 3002e485651e1ad18da6d44e7672dbe4f0380d3b Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 23 May 2019 18:18:06 -0500 +Subject: [PATCH] Refactor: controller: isolate stonith API handling + +can now make more variables and functions static +--- + daemons/controld/controld_control.c | 28 +++------------------ + daemons/controld/controld_fencing.c | 49 ++++++++++++++++++++++++++++++++++--- + daemons/controld/controld_fencing.h | 7 ++---- + 3 files changed, 50 insertions(+), 34 deletions(-) + +diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c +index 7f918c0..e99d605 100644 +--- a/daemons/controld/controld_control.c ++++ b/daemons/controld/controld_control.c +@@ -113,14 +113,7 @@ do_shutdown(long long action, + { + /* just in case */ + set_bit(fsa_input_register, R_SHUTDOWN); +- +- if (stonith_api) { +- /* Prevent it from coming up again */ +- clear_bit(fsa_input_register, R_ST_REQUIRED); +- +- crm_info("Disconnecting from fencer"); +- stonith_api->cmds->disconnect(stonith_api); +- } ++ controld_disconnect_fencer(FALSE); + } + + /* A_SHUTDOWN_REQ */ +@@ -201,12 +194,7 @@ crmd_exit(crm_exit_t exit_code) + + controld_close_attrd_ipc(); + pe_subsystem_free(); +- +- if(stonith_api) { +- crm_trace("Disconnecting fencing API"); +- clear_bit(fsa_input_register, R_ST_REQUIRED); +- stonith_api->cmds->free(stonith_api); stonith_api = NULL; +- } ++ controld_disconnect_fencer(TRUE); + + if ((exit_code == CRM_EX_OK) && (crmd_mainloop == NULL)) { + crm_debug("No mainloop detected"); +@@ -258,7 +246,6 @@ crmd_exit(crm_exit_t exit_code) + mainloop_destroy_trigger(fsa_source); fsa_source = NULL; + + mainloop_destroy_trigger(config_read); config_read = NULL; +- mainloop_destroy_trigger(stonith_reconnect); stonith_reconnect = NULL; + mainloop_destroy_trigger(transition_trigger); transition_trigger = NULL; + + crm_client_cleanup(); +@@ -288,7 +275,6 @@ crmd_exit(crm_exit_t exit_code) + free(fsa_cluster_name); fsa_cluster_name = NULL; + + free(te_uuid); te_uuid = NULL; +- free(te_client_id); te_client_id = NULL; + free(fsa_pe_ref); fsa_pe_ref = NULL; + free(failed_stop_offset); failed_stop_offset = NULL; + free(failed_start_offset); failed_start_offset = NULL; +@@ -627,15 +613,7 @@ do_started(long long action, + crm_err("Failed to create IPC server: shutting down and inhibiting respawn"); + register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); + } +- +- // Try connecting to fencer (retrying later in mainloop if failed) +- if (stonith_reconnect == NULL) { +- stonith_reconnect = mainloop_add_trigger(G_PRIORITY_LOW, +- te_connect_stonith, +- GINT_TO_POINTER(TRUE)); +- } +- set_bit(fsa_input_register, R_ST_REQUIRED); +- mainloop_set_trigger(stonith_reconnect); ++ controld_trigger_fencer_connect(); + + crm_notice("Pacemaker controller successfully started and accepting connections"); + clear_bit(fsa_input_register, R_STARTING); +diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c +index cde57b5..92336e9 100644 +--- a/daemons/controld/controld_fencing.c ++++ b/daemons/controld/controld_fencing.c +@@ -341,9 +341,9 @@ execute_stonith_cleanup() + * Functions that need to interact directly with the fencer via its API + */ + +-stonith_t *stonith_api = NULL; +-crm_trigger_t *stonith_reconnect = NULL; +-char *te_client_id = NULL; ++static stonith_t *stonith_api = NULL; ++static crm_trigger_t *stonith_reconnect = NULL; ++static char *te_client_id = NULL; + + static gboolean + fail_incompletable_stonith(crm_graph_t *graph) +@@ -571,7 +571,7 @@ tengine_stonith_notify(stonith_t *st, stonith_event_t *st_event) + * \note If user_data is NULL, this will wait 2s between attempts, for up to + * 30 attempts, meaning the controller could be blocked as long as 58s. + */ +-gboolean ++static gboolean + te_connect_stonith(gpointer user_data) + { + int rc = pcmk_ok; +@@ -619,6 +619,47 @@ te_connect_stonith(gpointer user_data) + return TRUE; + } + ++/*! ++ \internal ++ \brief Schedule fencer connection attempt in main loop ++*/ ++void ++controld_trigger_fencer_connect() ++{ ++ if (stonith_reconnect == NULL) { ++ stonith_reconnect = mainloop_add_trigger(G_PRIORITY_LOW, ++ te_connect_stonith, ++ GINT_TO_POINTER(TRUE)); ++ } ++ set_bit(fsa_input_register, R_ST_REQUIRED); ++ mainloop_set_trigger(stonith_reconnect); ++} ++ ++void ++controld_disconnect_fencer(bool destroy) ++{ ++ if (stonith_api) { ++ // Prevent fencer connection from coming up again ++ clear_bit(fsa_input_register, R_ST_REQUIRED); ++ ++ stonith_api->cmds->disconnect(stonith_api); ++ } ++ if (destroy) { ++ if (stonith_api) { ++ stonith_api->cmds->free(stonith_api); ++ stonith_api = NULL; ++ } ++ if (stonith_reconnect) { ++ mainloop_destroy_trigger(stonith_reconnect); ++ stonith_reconnect = NULL; ++ } ++ if (te_client_id) { ++ free(te_client_id); ++ te_client_id = NULL; ++ } ++ } ++} ++ + static gboolean + do_stonith_history_sync(gpointer user_data) + { +diff --git a/daemons/controld/controld_fencing.h b/daemons/controld/controld_fencing.h +index b80a6c9..3ef537f 100644 +--- a/daemons/controld/controld_fencing.h ++++ b/daemons/controld/controld_fencing.h +@@ -13,16 +13,13 @@ + #include // bool + #include // crm_graph_t, crm_action_t + +-extern crm_trigger_t *stonith_reconnect; +-extern char *te_client_id; +-extern stonith_t *stonith_api; +- + // stonith fail counts + void st_fail_count_reset(const char * target); + void update_stonith_max_attempts(const char* value); + + // stonith API client +-gboolean te_connect_stonith(gpointer user_data); ++void controld_trigger_fencer_connect(void); ++void controld_disconnect_fencer(bool destroy); + gboolean te_fence_node(crm_graph_t *graph, crm_action_t *action); + + // stonith cleanup list +-- +1.8.3.1 + diff --git a/SOURCES/007-unfencing.patch b/SOURCES/007-unfencing.patch deleted file mode 100644 index 1ce5c0a..0000000 --- a/SOURCES/007-unfencing.patch +++ /dev/null @@ -1,309 +0,0 @@ -From 28566d6832274c59f27bb7b2f1f54420a3f3d822 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 9 May 2019 20:26:08 -0500 -Subject: [PATCH 1/2] Refactor: libpe_status: functionize unfencing digest code - more - -... for readability, reusability, and avoiding unnecessary function calls or -memory allocation. ---- - lib/pengine/utils.c | 159 ++++++++++++++++++++++++++++++++++++++-------------- - 1 file changed, 118 insertions(+), 41 deletions(-) - -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index 2f4dc1e..f80f8d4 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -2080,57 +2080,134 @@ rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, - return data; - } - -+/*! -+ * \internal -+ * \brief Create an unfencing summary for use in special node attribute -+ * -+ * Create a string combining a fence device's resource ID, agent type, and -+ * parameter digest (whether for all parameters or just non-private parameters). -+ * This can be stored in a special node attribute, allowing us to detect changes -+ * in either the agent type or parameters, to know whether unfencing must be -+ * redone or can be safely skipped when the device's history is cleaned. -+ * -+ * \param[in] rsc_id Fence device resource ID -+ * \param[in] agent_type Fence device agent -+ * \param[in] param_digest Fence device parameter digest -+ * -+ * \return Newly allocated string with unfencing digest -+ * \note The caller is responsible for freeing the result. -+ */ -+static inline char * -+create_unfencing_summary(const char *rsc_id, const char *agent_type, -+ const char *param_digest) -+{ -+ return crm_strdup_printf("%s:%s:%s", rsc_id, agent_type, param_digest); -+} -+ -+/*! -+ * \internal -+ * \brief Check whether a node can skip unfencing -+ * -+ * Check whether a fence device's current definition matches a node's -+ * stored summary of when it was last unfenced by the device. -+ * -+ * \param[in] rsc_id Fence device's resource ID -+ * \param[in] agent Fence device's agent type -+ * \param[in] digest_calc Fence device's current parameter digest -+ * \param[in] node_summary Value of node's special unfencing node attribute -+ * (a comma-separated list of unfencing summaries for -+ * all devices that have unfenced this node) -+ * -+ * \return TRUE if digest matches, FALSE otherwise -+ */ -+static bool -+unfencing_digest_matches(const char *rsc_id, const char *agent, -+ const char *digest_calc, const char *node_summary) -+{ -+ bool matches = FALSE; -+ -+ if (rsc_id && agent && digest_calc && node_summary) { -+ char *search_secure = create_unfencing_summary(rsc_id, agent, -+ digest_calc); -+ -+ /* The digest was calculated including the device ID and agent, -+ * so there is no risk of collision using strstr(). -+ */ -+ matches = (strstr(node_summary, search_secure) != NULL); -+ crm_trace("Calculated unfencing digest '%s' %sfound in '%s'", -+ search_secure, matches? "" : "not ", node_summary); -+ free(search_secure); -+ } -+ return matches; -+} -+ -+/* Magic string to use as action name for digest cache entries used for -+ * unfencing checks. This is not a real action name (i.e. "on"), so -+ * check_action_definition() won't confuse these entries with real actions. -+ */ - #define STONITH_DIGEST_TASK "stonith-on" - -+/*! -+ * \internal -+ * \brief Calculate fence device digests and digest comparison result -+ * -+ * \param[in] rsc Fence device resource -+ * \param[in] agent Fence device's agent type -+ * \param[in] node Node with digest cache to use -+ * \param[in] data_set Cluster working set -+ * -+ * \return Node's digest cache entry -+ */ - static op_digest_cache_t * --fencing_action_digest_cmp(resource_t * rsc, node_t * node, pe_working_set_t * data_set) -+fencing_action_digest_cmp(pe_resource_t *rsc, const char *agent, -+ pe_node_t *node, pe_working_set_t *data_set) - { -- char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); -- op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, node, NULL, data_set); -+ const char *node_summary = NULL; - -- const char *digest_all = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL); -- const char *digest_secure = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE); -+ // Calculate device's current parameter digests -+ char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); -+ op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, -+ node, NULL, data_set); - -- /* No 'reloads' for fencing device changes -- * -- * We use the resource id + agent + digest so that we can detect -- * changes to the agent and/or the parameters used -- */ -- char *search_all = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_all_calc); -- char *search_secure = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_secure_calc); -+ free(key); - -- data->rc = RSC_DIGEST_ALL; -- if (digest_all == NULL) { -- /* it is unknown what the previous op digest was */ -+ // Check whether node has special unfencing summary node attribute -+ node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL); -+ if (node_summary == NULL) { - data->rc = RSC_DIGEST_UNKNOWN; -+ return data; -+ } - -- } else if (strstr(digest_all, search_all)) { -+ // Check whether full parameter digest matches -+ if (unfencing_digest_matches(rsc->id, agent, data->digest_all_calc, -+ node_summary)) { - data->rc = RSC_DIGEST_MATCH; -+ return data; -+ } - -- } else if(digest_secure && data->digest_secure_calc) { -- if(strstr(digest_secure, search_secure)) { -- if (is_set(data_set->flags, pe_flag_stdout)) { -- printf("Only 'private' parameters to %s for unfencing %s changed\n", -- rsc->id, node->details->uname); -- } -- data->rc = RSC_DIGEST_MATCH; -+ // Check whether secure parameter digest matches -+ node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE); -+ if (unfencing_digest_matches(rsc->id, agent, data->digest_secure_calc, -+ node_summary)) { -+ data->rc = RSC_DIGEST_MATCH; -+ if (is_set(data_set->flags, pe_flag_stdout)) { -+ printf("Only 'private' parameters to %s for unfencing %s changed\n", -+ rsc->id, node->details->uname); - } -+ return data; - } - -- if (is_set(data_set->flags, pe_flag_sanitized) -- && is_set(data_set->flags, pe_flag_stdout) -- && (data->rc == RSC_DIGEST_ALL) -+ // Parameters don't match -+ data->rc = RSC_DIGEST_ALL; -+ if (is_set(data_set->flags, (pe_flag_sanitized|pe_flag_stdout)) - && data->digest_secure_calc) { -- printf("Parameters to %s for unfencing %s changed, try '%s:%s:%s'\n", -- rsc->id, node->details->uname, rsc->id, -- (const char *) g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), -- data->digest_secure_calc); -- } -- -- free(key); -- free(search_all); -- free(search_secure); -+ char *digest = create_unfencing_summary(rsc->id, agent, -+ data->digest_secure_calc); - -+ printf("Parameters to %s for unfencing %s changed, try '%s'\n", -+ rsc->id, node->details->uname, digest); -+ free(digest); -+ } - return data; - } - -@@ -2218,9 +2295,6 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe - * - * We may do this for all nodes in the future, but for now - * the check_action_definition() based stuff works fine. -- * -- * Use "stonith-on" to avoid creating cache entries for -- * operations check_action_definition() would look for. - */ - long max = 1024; - long digests_all_offset = 0; -@@ -2232,8 +2306,11 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe - - for (GListPtr gIter = matches; gIter != NULL; gIter = gIter->next) { - resource_t *match = gIter->data; -- op_digest_cache_t *data = fencing_action_digest_cmp(match, node, data_set); -+ const char *agent = g_hash_table_lookup(match->meta, -+ XML_ATTR_TYPE); -+ op_digest_cache_t *data = NULL; - -+ data = fencing_action_digest_cmp(match, agent, node, data_set); - if(data->rc == RSC_DIGEST_ALL) { - optional = FALSE; - crm_notice("Unfencing %s (remote): because the definition of %s changed", node->details->uname, match->id); -@@ -2244,11 +2321,11 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe - - digests_all_offset += snprintf( - digests_all+digests_all_offset, max-digests_all_offset, -- "%s:%s:%s,", match->id, (const char*)g_hash_table_lookup(match->meta, XML_ATTR_TYPE), data->digest_all_calc); -+ "%s:%s:%s,", match->id, agent, data->digest_all_calc); - - digests_secure_offset += snprintf( - digests_secure+digests_secure_offset, max-digests_secure_offset, -- "%s:%s:%s,", match->id, (const char*)g_hash_table_lookup(match->meta, XML_ATTR_TYPE), data->digest_secure_calc); -+ "%s:%s:%s,", match->id, agent, data->digest_secure_calc); - } - g_hash_table_insert(stonith_op->meta, - strdup(XML_OP_ATTR_DIGESTS_ALL), --- -1.8.3.1 - - -From fd6e06ff419c95f4423202163d2d4dca3f03a4c5 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 10 May 2019 11:57:31 -0500 -Subject: [PATCH 2/2] Fix: libpe_status: calculate secure digests for unfencing - ops - -The calculation of digests for detection of when unfencing is needed reused -rsc_action_digest(). However that would only add secure digests when the -pe_flag_sanitized flag was set, which is only set by crm_simulate, so secure -digests would never be added in normal cluster operation. This led to -node attributes like name="#digests-secure" -value="stonith-fence_compute-fence-nova:fence_compute:(null),". - -Now, rsc_action_digest() takes a new argument to select whether secure digests -are added, which is always set to TRUE when calculating unfencing digests. ---- - lib/pengine/utils.c | 27 ++++++++++++++++++++++----- - 1 file changed, 22 insertions(+), 5 deletions(-) - -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index f80f8d4..5b893f7 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -1936,9 +1936,24 @@ append_versioned_params(xmlNode *versioned_params, const char *ra_version, xmlNo - } - #endif - -+/*! -+ * \internal -+ * \brief Calculate action digests and store in node's digest cache -+ * -+ * \param[in] rsc Resource that action was for -+ * \param[in] task Name of action performed -+ * \param[in] key Action's task key -+ * \param[in] node Node action was performed on -+ * \param[in] xml_op XML of operation in CIB status (if available) -+ * \param[in] calc_secure Whether to calculate secure digest -+ * \param[in] data_set Cluster working set -+ * -+ * \return Pointer to node's digest cache entry -+ */ - static op_digest_cache_t * --rsc_action_digest(resource_t * rsc, const char *task, const char *key, -- node_t * node, xmlNode * xml_op, pe_working_set_t * data_set) -+rsc_action_digest(pe_resource_t *rsc, const char *task, const char *key, -+ pe_node_t *node, xmlNode *xml_op, bool calc_secure, -+ pe_working_set_t *data_set) - { - op_digest_cache_t *data = NULL; - -@@ -2007,7 +2022,7 @@ rsc_action_digest(resource_t * rsc, const char *task, const char *key, - - data->digest_all_calc = calculate_operation_digest(data->params_all, op_version); - -- if (is_set(data_set->flags, pe_flag_sanitized)) { -+ if (calc_secure) { - data->params_secure = copy_xml(data->params_all); - if(secure_list) { - filter_parameters(data->params_secure, secure_list, FALSE); -@@ -2053,7 +2068,9 @@ rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, - - interval_ms = crm_parse_ms(interval_ms_s); - key = generate_op_key(rsc->id, task, interval_ms); -- data = rsc_action_digest(rsc, task, key, node, xml_op, data_set); -+ data = rsc_action_digest(rsc, task, key, node, xml_op, -+ is_set(data_set->flags, pe_flag_sanitized), -+ data_set); - - data->rc = RSC_DIGEST_MATCH; - if (digest_restart && data->digest_restart_calc && strcmp(data->digest_restart_calc, digest_restart) != 0) { -@@ -2167,7 +2184,7 @@ fencing_action_digest_cmp(pe_resource_t *rsc, const char *agent, - // Calculate device's current parameter digests - char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); - op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, -- node, NULL, data_set); -+ node, NULL, TRUE, data_set); - - free(key); - --- -1.8.3.1 - diff --git a/SOURCES/008-remote.patch b/SOURCES/008-remote.patch deleted file mode 100644 index 9760137..0000000 --- a/SOURCES/008-remote.patch +++ /dev/null @@ -1,545 +0,0 @@ -From dad337a96dfeca4dbde7bbd97f99f24956440fc2 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Sat, 8 Jun 2019 16:25:04 -0500 -Subject: [PATCH 1/4] Refactor: libpe_status: add function for checking - shutdown attribute - -... to reduce code duplication and allow further reuse ---- - include/crm/pengine/internal.h | 2 ++ - lib/pengine/unpack.c | 8 ++------ - lib/pengine/utils.c | 20 ++++++++++++++++++++ - 3 files changed, 24 insertions(+), 6 deletions(-) - -diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h -index fd55bb9..a2a3d52 100644 ---- a/include/crm/pengine/internal.h -+++ b/include/crm/pengine/internal.h -@@ -359,4 +359,6 @@ void pe__foreach_param_check(pe_working_set_t *data_set, - enum pe_check_parameters, - pe_working_set_t*)); - void pe__free_param_checks(pe_working_set_t *data_set); -+ -+bool pe__shutdown_requested(pe_node_t *node); - #endif -diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c -index 081df07..9d13a57 100644 ---- a/lib/pengine/unpack.c -+++ b/lib/pengine/unpack.c -@@ -909,7 +909,6 @@ unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * - const char *resource_discovery_enabled = NULL; - xmlNode *attrs = NULL; - resource_t *rsc = NULL; -- const char *shutdown = NULL; - - if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) { - return; -@@ -931,8 +930,7 @@ unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * - attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE); - add_node_attrs(attrs, this_node, TRUE, data_set); - -- shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); -- if (shutdown != NULL && safe_str_neq("0", shutdown)) { -+ if (pe__shutdown_requested(this_node)) { - crm_info("Node %s is shutting down", this_node->details->uname); - this_node->details->shutdown = TRUE; - if (rsc) { -@@ -1392,7 +1390,6 @@ gboolean - determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set_t * data_set) - { - gboolean online = FALSE; -- const char *shutdown = NULL; - const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED); - - if (this_node == NULL) { -@@ -1402,9 +1399,8 @@ determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set - - this_node->details->shutdown = FALSE; - this_node->details->expected_up = FALSE; -- shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); - -- if (shutdown != NULL && safe_str_neq("0", shutdown)) { -+ if (pe__shutdown_requested(this_node)) { - this_node->details->shutdown = TRUE; - - } else if (safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) { -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index 5b893f7..c5fd0f7 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -2510,3 +2510,23 @@ void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrit - } - } - } -+ -+/*! -+ * \internal -+ * \brief Check whether shutdown has been requested for a node -+ * -+ * \param[in] node Node to check -+ * -+ * \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise -+ * \note This differs from simply using node->details->shutdown in that it can -+ * be used before that has been determined (and in fact to determine it), -+ * and it can also be used to distinguish requested shutdown from implicit -+ * shutdown of remote nodes by virtue of their connection stopping. -+ */ -+bool -+pe__shutdown_requested(pe_node_t *node) -+{ -+ const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN); -+ -+ return shutdown && strcmp(shutdown, "0"); -+} --- -1.8.3.1 - - -From 1e9903326a59f58d9dd2f2618d709f8aa61e41e9 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 5 Jun 2019 16:37:26 -0500 -Subject: [PATCH 2/4] Fix: scheduler: remote state is failed if node is - shutting down with connection failure - -When determining remote state, if the connection resource is failed and not -being started again, we consider the state to be unknown if the connection has -a reconnect interval, because we won't know whether the connection can be -recovered until the interval expires and we re-attempt connection. - -However, if the node is shutting down at the time, we won't re-attempt -connection, so consider the state failed in that case. (Note that we check the -actual shutdown node attribute, rather than node->details->shutdown, since that -is set for remote nodes whenever the connection is stopping.) - -This avoids a situation where actions that cannot succeed can be scheduled on a -remote node that's shutting down. ---- - daemons/schedulerd/sched_allocate.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/daemons/schedulerd/sched_allocate.c b/daemons/schedulerd/sched_allocate.c -index 3363a72..b7d1b48 100644 ---- a/daemons/schedulerd/sched_allocate.c -+++ b/daemons/schedulerd/sched_allocate.c -@@ -1972,7 +1972,8 @@ get_remote_node_state(pe_node_t *node) - - if ((remote_rsc->next_role == RSC_ROLE_STOPPED) - && remote_rsc->remote_reconnect_ms -- && node->details->remote_was_fenced) { -+ && node->details->remote_was_fenced -+ && !pe__shutdown_requested(node)) { - - /* We won't know whether the connection is recoverable until the - * reconnect interval expires and we reattempt connection. --- -1.8.3.1 - - -From ea70750d04219618b5feeda04443b27616e441a0 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 5 Jun 2019 16:43:19 -0500 -Subject: [PATCH 3/4] Fix: libpe_status: don't order implied stops relative to - a remote connection - -Actions behind a remote connection are ordered relative to any start or stop of -the remote connection. However, if the action is a stop implied due to fencing, -it does not require the remote connection, and the ordering should not be done. - -This avoids a delay in the remote connection recovery if it is failed, e.g. -previously the ordering would look like: - - fence remote node -> implied stop of resource on remote -> stop connection - -Now, the connection stop can proceed simultaneously with the remote node -fencing. ---- - daemons/schedulerd/sched_allocate.c | 11 +++++------ - 1 file changed, 5 insertions(+), 6 deletions(-) - -diff --git a/daemons/schedulerd/sched_allocate.c b/daemons/schedulerd/sched_allocate.c -index b7d1b48..9f82c00 100644 ---- a/daemons/schedulerd/sched_allocate.c -+++ b/daemons/schedulerd/sched_allocate.c -@@ -2065,14 +2065,13 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) - pe_order_implies_first, data_set); - - } else if(state == remote_state_failed) { -- /* We would only be here if the resource is -- * running on the remote node. Since we have no -- * way to stop it, it is necessary to fence the -- * node. -+ /* The resource is active on the node, but since we don't have a -+ * valid connection, the only way to stop the resource is by -+ * fencing the node. There is no need to order the stop relative -+ * to the remote connection, since the stop will become implied -+ * by the fencing. - */ - pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable"); -- order_action_then_stop(action, remote_rsc, -- pe_order_implies_first, data_set); - - } else if(remote_rsc->next_role == RSC_ROLE_STOPPED) { - /* State must be remote_state_unknown or remote_state_stopped. --- -1.8.3.1 - - -From 091c367369b892d26fe0de99d35cf521b6249d10 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Sat, 8 Jun 2019 16:51:20 -0500 -Subject: [PATCH 4/4] Test: cts-scheduler: update regression tests for remote - connection ordering change - -Remote connection stops no longer have to wait for implied stops of resources -behind the connection. - -Unchanged from before, if the remote connection stops are implied themselves, -they can be confirmed immediately without waiting for their host's fencing, -because remote connections have "requires" set to "quorum" rather than -"fencing". ---- - cts/scheduler/order-expired-failure.dot | 1 - - cts/scheduler/order-expired-failure.exp | 6 +----- - cts/scheduler/order-expired-failure.summary | 2 +- - cts/scheduler/remote-connection-unrecoverable.dot | 1 - - cts/scheduler/remote-connection-unrecoverable.exp | 6+----- - cts/scheduler/remote-connection-unrecoverable.summary | 2 +- - cts/scheduler/remote-fence-before-reconnect.dot | 1 - - cts/scheduler/remote-fence-before-reconnect.exp | 6 +----- - cts/scheduler/remote-fence-before-reconnect.summary | 2 +- - cts/scheduler/remote-recover-all.dot | 2 -- - cts/scheduler/remote-recover-all.exp | 12 ++---------- - cts/scheduler/remote-recover-all.summary | 4 ++-- - cts/scheduler/remote-recover-no-resources.dot | 1 - - cts/scheduler/remote-recover-no-resources.exp | 6 +----- - cts/scheduler/remote-recover-no-resources.summary | 2 +- - cts/scheduler/remote-recover-unknown.dot | 1 - - cts/scheduler/remote-recover-unknown.exp | 6 +----- - cts/scheduler/remote-recover-unknown.summary | 2 +- - 18 files changed, 14 insertions(+), 53 deletions(-) - -diff --git a/cts/scheduler/order-expired-failure.dot b/cts/scheduler/order-expired-failure.dot -index 2e9963b..5c21d5d 100644 ---- a/cts/scheduler/order-expired-failure.dot -+++ b/cts/scheduler/order-expired-failure.dot -@@ -4,7 +4,6 @@ digraph "g" { - "compute-unfence-trigger-clone_stop_0" [ style=bold color="green" fontcolor="orange"] - "compute-unfence-trigger-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] - "compute-unfence-trigger_stop_0 overcloud-novacompute-1" -> "compute-unfence-trigger-clone_stopped_0" [ style = bold] --"compute-unfence-trigger_stop_0 overcloud-novacompute-1" -> "overcloud-novacompute-1_stop_0 controller-1" [ style = bold] - "compute-unfence-trigger_stop_0 overcloud-novacompute-1" [ style=bold color="green" fontcolor="orange"] - "ip-10.0.0.110_monitor_10000 controller-1" [ style=bold color="green" fontcolor="black"] - "ip-10.0.0.110_start_0 controller-1" -> "ip-10.0.0.110_monitor_10000 controller-1" [ style = bold] -diff --git a/cts/scheduler/order-expired-failure.exp b/cts/scheduler/order-expired-failure.exp -index c476bc2..4a50493 100644 ---- a/cts/scheduler/order-expired-failure.exp -+++ b/cts/scheduler/order-expired-failure.exp -@@ -9,11 +9,7 @@ - - - -- -- -- -- -- -+ - - - -diff --git a/cts/scheduler/order-expired-failure.summary b/cts/scheduler/order-expired-failure.summary -index c86bb91..2cf43ed 100644 ---- a/cts/scheduler/order-expired-failure.summary -+++ b/cts/scheduler/order-expired-failure.summary -@@ -52,6 +52,7 @@ Transition Summary: - * Stop compute-unfence-trigger:1 ( overcloud-novacompute-1 ) due to node availability - - Executing cluster transition: -+ * Resource action: overcloud-novacompute-1 stop on controller-1 - * Resource action: stonith-fence_compute-fence-nova stop on controller-2 - * Fencing overcloud-novacompute-1 (reboot) - * Cluster action: clear_failcount for overcloud-novacompute-1 on controller-1 -@@ -62,7 +63,6 @@ Executing cluster transition: - * Resource action: ip-10.0.0.110 monitor=10000 on controller-1 - * Pseudo action: compute-unfence-trigger_stop_0 - * Pseudo action: compute-unfence-trigger-clone_stopped_0 -- * Resource action: overcloud-novacompute-1 stop on controller-1 - Using the original execution date of: 2018-04-09 07:55:35Z - - Revised cluster status: -diff --git a/cts/scheduler/remote-connection-unrecoverable.dot b/cts/scheduler/remote-connection-unrecoverable.dot -index c86bb91..2cf43ed 100644 ---- a/cts/scheduler/remote-connection-unrecoverable.dot -+++ b/cts/scheduler/remote-connection-unrecoverable.dot -@@ -12,7 +12,6 @@ - "rsc1_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] - "rsc1_start_0 node2" -> "rsc1_monitor_10000 node2" [ style = bold] - "rsc1_start_0 node2" [ style=bold color="green" fontcolor="black"] --"rsc1_stop_0 remote1" -> "remote1_stop_0 node1" [ style = bold] - "rsc1_stop_0 remote1" -> "rsc1_delete_0 remote1" [ style = dashed] - "rsc1_stop_0 remote1" -> "rsc1_start_0 node2" [ style = bold] - "rsc1_stop_0 remote1" -> "rsc2-master_demote_0" [ style = bold] - -diff --git a/cts/scheduler/remote-connection-unrecoverable.exp b/cts/scheduler/remote-connection-unrecoverable.exp -index c86bb91..2cf43ed 100644 ---- a/cts/scheduler/remote-connection-unrecoverable.exp -+++ b/cts/scheduler/remote-connection-unrecoverable.exp -@@ -5,11 +5,7 @@ - - - -- -- -- -- -- -+ - - - - -diff --git a/cts/scheduler/remote-connection-unrecoverable.summary b/cts/scheduler/remote-connection-unrecoverable.summary -index 23fa9ca..caff564 100644 ---- a/cts/scheduler/remote-connection-unrecoverable.summary -+++ b/cts/scheduler/remote-connection-unrecoverable.summary -@@ -21,6 +21,7 @@ Transition Summary: - * Stop rsc2:0 ( Master node1 ) due to node availability - - Executing cluster transition: -+ * Pseudo action: remote1_stop_0 - * Resource action: killer stop on node2 - * Resource action: rsc1 monitor on node2 - * Fencing node1 (reboot) -@@ -29,7 +30,6 @@ Executing cluster transition: - * Resource action: killer monitor=60000 on node2 - * Pseudo action: rsc1_stop_0 - * Pseudo action: rsc2-master_demote_0 -- * Pseudo action: remote1_stop_0 - * Resource action: rsc1 start on node2 - * Pseudo action: rsc2_demote_0 - * Pseudo action: rsc2-master_demoted_0 -diff --git a/cts/scheduler/remote-fence-before-reconnect.dot b/cts/scheduler/remote-fence-before-reconnect.dot -index 4ced43e..5812b7f 100644 ---- a/cts/scheduler/remote-fence-before-reconnect.dot -+++ b/cts/scheduler/remote-fence-before-reconnect.dot -@@ -3,7 +3,6 @@ - "fake2_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"] - "fake2_start_0 c7auto1" -> "fake2_monitor_10000 c7auto1" [ style = bold] - "fake2_start_0 c7auto1" [ style=bold color="green" fontcolor="black"] --"fake2_stop_0 c7auto4" -> "c7auto4_stop_0 c7auto1" [ style = bold] - "fake2_stop_0 c7auto4" -> "fake2_start_0 c7auto1" [ style = bold] - "fake2_stop_0 c7auto4" [ style=bold color="green" fontcolor="orange"] - "stonith 'reboot' c7auto4" -> "fake2_start_0 c7auto1" [ style = bold] -diff --git a/cts/scheduler/remote-fence-before-reconnect.exp b/cts/scheduler/remote-fence-before-reconnect.exp -index f99d9ef..f506f85 100644 ---- a/cts/scheduler/remote-fence-before-reconnect.exp -+++ b/cts/scheduler/remote-fence-before-reconnect.exp -@@ -9,11 +9,7 @@ - - - -- -- -- -- -- -+ - - - -diff --git a/cts/scheduler/remote-fence-before-reconnect.summary b/cts/scheduler/remote-fence-before-reconnect.summary -index f61e18b..03eac20 100644 ---- a/cts/scheduler/remote-fence-before-reconnect.summary -+++ b/cts/scheduler/remote-fence-before-reconnect.summary -@@ -17,9 +17,9 @@ Transition Summary: - * Move fake2 ( c7auto4 -> c7auto1 ) - - Executing cluster transition: -+ * Resource action: c7auto4 stop on c7auto1 - * Fencing c7auto4 (reboot) - * Pseudo action: fake2_stop_0 -- * Resource action: c7auto4 stop on c7auto1 - * Resource action: fake2 start on c7auto1 - * Resource action: fake2 monitor=10000 on c7auto1 - -diff --git a/cts/scheduler/remote-recover-all.dot b/cts/scheduler/remote-recover-all.dot -index deed802..4128b10 100644 ---- a/cts/scheduler/remote-recover-all.dot -+++ b/cts/scheduler/remote-recover-all.dot -@@ -19,7 +19,6 @@ digraph "g" { - "galera_demote_0 galera-2" -> "galera_stop_0 galera-2" [ style = bold] - "galera_demote_0 galera-2" [ style=bold color="green" fontcolor="orange"] - "galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"] --"galera_stop_0 galera-2" -> "galera-2_stop_0 controller-1" [ style = bold] - "galera_stop_0 galera-2" -> "galera-master_stopped_0" [ style = bold] - "galera_stop_0 galera-2" [ style=bold color="green" fontcolor="orange"] - "haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold] -@@ -60,7 +59,6 @@ digraph "g" { - "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] - "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] - "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] --"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] - "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] - "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] - "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] -diff --git a/cts/scheduler/remote-recover-all.exp b/cts/scheduler/remote-recover-all.exp -index 8137ffb..0cb51f6 100644 ---- a/cts/scheduler/remote-recover-all.exp -+++ b/cts/scheduler/remote-recover-all.exp -@@ -5,11 +5,7 @@ - - - -- -- -- -- -- -+ - - - -@@ -57,11 +53,7 @@ - - - -- -- -- -- -- -+ - - - -diff --git a/cts/scheduler/remote-recover-all.summary b/cts/scheduler/remote-recover-all.summary -index 2ac0c6a..d095fdd 100644 ---- a/cts/scheduler/remote-recover-all.summary -+++ b/cts/scheduler/remote-recover-all.summary -@@ -56,7 +56,9 @@ Transition Summary: - * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) - - Executing cluster transition: -+ * Pseudo action: messaging-1_stop_0 - * Pseudo action: galera-0_stop_0 -+ * Pseudo action: galera-2_stop_0 - * Pseudo action: galera-master_demote_0 - * Pseudo action: redis-master_pre_notify_stop_0 - * Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0 -@@ -94,7 +96,6 @@ Executing cluster transition: - * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 - * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 - * Resource action: galera-0 monitor=20000 on controller-2 -- * Pseudo action: galera-2_stop_0 - * Resource action: rabbitmq notify on messaging-2 - * Resource action: rabbitmq notify on messaging-0 - * Pseudo action: rabbitmq_notified_0 -@@ -107,7 +108,6 @@ Executing cluster transition: - * Resource action: ip-172.17.1.17 start on controller-2 - * Resource action: ip-172.17.4.11 start on controller-2 - * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 -- * Pseudo action: messaging-1_stop_0 - * Pseudo action: redis_notified_0 - * Resource action: ip-172.17.1.14 monitor=10000 on controller-2 - * Resource action: ip-172.17.1.17 monitor=10000 on controller-2 -diff --git a/cts/scheduler/remote-recover-no-resources.dot b/cts/scheduler/remote-recover-no-resources.dot -index ef78aa6..a2f8ce0 100644 ---- a/cts/scheduler/remote-recover-no-resources.dot -+++ b/cts/scheduler/remote-recover-no-resources.dot -@@ -45,7 +45,6 @@ digraph "g" { - "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] - "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] - "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] --"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] - "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] - "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] - "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] -diff --git a/cts/scheduler/remote-recover-no-resources.exp b/cts/scheduler/remote-recover-no-resources.exp -index 8a67c11..90470fb 100644 ---- a/cts/scheduler/remote-recover-no-resources.exp -+++ b/cts/scheduler/remote-recover-no-resources.exp -@@ -5,11 +5,7 @@ - - - -- -- -- -- -- -+ - - - -diff --git a/cts/scheduler/remote-recover-no-resources.summary b/cts/scheduler/remote-recover-no-resources.summary -index 89da784..18a989b 100644 ---- a/cts/scheduler/remote-recover-no-resources.summary -+++ b/cts/scheduler/remote-recover-no-resources.summary -@@ -54,6 +54,7 @@ Transition Summary: - * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) - - Executing cluster transition: -+ * Pseudo action: messaging-1_stop_0 - * Pseudo action: galera-0_stop_0 - * Pseudo action: galera-2_stop_0 - * Pseudo action: redis-master_pre_notify_stop_0 -@@ -92,7 +93,6 @@ Executing cluster transition: - * Pseudo action: ip-172.17.1.17_stop_0 - * Pseudo action: ip-172.17.4.11_stop_0 - * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 -- * Pseudo action: messaging-1_stop_0 - * Resource action: redis notify on controller-0 - * Resource action: redis notify on controller-2 - * Pseudo action: redis-master_confirmed-post_notify_stopped_0 -diff --git a/cts/scheduler/remote-recover-unknown.dot b/cts/scheduler/remote-recover-unknown.dot -index 5cd760b..29ab59f 100644 ---- a/cts/scheduler/remote-recover-unknown.dot -+++ b/cts/scheduler/remote-recover-unknown.dot -@@ -46,7 +46,6 @@ digraph "g" { - "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] - "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] - "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] --"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] - "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] - "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] - "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] -diff --git a/cts/scheduler/remote-recover-unknown.exp b/cts/scheduler/remote-recover-unknown.exp -index ac6f004..82cb65f7 100644 ---- a/cts/scheduler/remote-recover-unknown.exp -+++ b/cts/scheduler/remote-recover-unknown.exp -@@ -5,11 +5,7 @@ - - - -- -- -- -- -- -+ - - - -diff --git a/cts/scheduler/remote-recover-unknown.summary b/cts/scheduler/remote-recover-unknown.summary -index 2c60713..4d7a411 100644 ---- a/cts/scheduler/remote-recover-unknown.summary -+++ b/cts/scheduler/remote-recover-unknown.summary -@@ -55,6 +55,7 @@ Transition Summary: - * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) - - Executing cluster transition: -+ * Pseudo action: messaging-1_stop_0 - * Pseudo action: galera-0_stop_0 - * Pseudo action: galera-2_stop_0 - * Pseudo action: redis-master_pre_notify_stop_0 -@@ -94,7 +95,6 @@ Executing cluster transition: - * Pseudo action: ip-172.17.1.17_stop_0 - * Pseudo action: ip-172.17.4.11_stop_0 - * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 -- * Pseudo action: messaging-1_stop_0 - * Resource action: redis notify on controller-0 - * Resource action: redis notify on controller-2 - * Pseudo action: redis-master_confirmed-post_notify_stopped_0 --- -1.8.3.1 - diff --git a/SOURCES/008-stonith_admin-header-refactoring.patch b/SOURCES/008-stonith_admin-header-refactoring.patch new file mode 100644 index 0000000..97a8b8d --- /dev/null +++ b/SOURCES/008-stonith_admin-header-refactoring.patch @@ -0,0 +1,1715 @@ +From 756a3e522aa444b456e21128a52317226b346005 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 21 May 2019 15:26:20 -0500 +Subject: [PATCH 01/11] Doc: libpacemaker: correct doxygen block for shutdown + op creator + +copy/paste error +--- + lib/pacemaker/pcmk_sched_utils.c | 5 +---- + 1 file changed, 1 insertion(+), 4 deletions(-) + +diff --git a/lib/pacemaker/pcmk_sched_utils.c b/lib/pacemaker/pcmk_sched_utils.c +index 5342e51..7b5cb7d 100644 +--- a/lib/pacemaker/pcmk_sched_utils.c ++++ b/lib/pacemaker/pcmk_sched_utils.c +@@ -446,10 +446,7 @@ pe_cancel_op(pe_resource_t *rsc, const char *task, guint interval_ms, + * \internal + * \brief Create a shutdown op for a scheduler transition + * +- * \param[in] rsc Resource of action to cancel +- * \param[in] task Name of action to cancel +- * \param[in] interval_ms Interval of action to cancel +- * \param[in] node Node of action to cancel ++ * \param[in] node Node being shut down + * \param[in] data_set Working set of cluster + * + * \return Created op +-- +1.8.3.1 + + +From 5249dd9295307c0e22e223ea7d6f5f24a0a3fe25 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 24 May 2019 10:17:15 -0500 +Subject: [PATCH 02/11] Refactor: libpe_status: rename target rc function + +... in line with current naming standards, to avoid confusion with +controller function of the same name +--- + include/crm/pengine/internal.h | 2 +- + lib/pengine/failcounts.c | 4 ++-- + lib/pengine/unpack.c | 4 ++-- + 3 files changed, 5 insertions(+), 5 deletions(-) + +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index 6d22db7..fd55bb9 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -288,7 +288,7 @@ pe_base_name_eq(resource_t *rsc, const char *id) + return FALSE; + } + +-int get_target_rc(xmlNode * xml_op); ++int pe__target_rc_from_xml(xmlNode *xml_op); + + gint sort_node_uname(gconstpointer a, gconstpointer b); + bool is_set_recursive(resource_t * rsc, long long flag, bool any); +diff --git a/lib/pengine/failcounts.c b/lib/pengine/failcounts.c +index 8f01c07..0c8ca5d 100644 +--- a/lib/pengine/failcounts.c ++++ b/lib/pengine/failcounts.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2008-2018 Andrew Beekhof ++ * Copyright 2008-2019 the Pacemaker project contributors + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. +@@ -62,7 +62,7 @@ is_matched_failure(const char *rsc_id, xmlNode *conf_op_xml, + + if (safe_str_eq(expected_op_key, lrm_op_id)) { + int rc = 0; +- int target_rc = get_target_rc(lrm_op_xml); ++ int target_rc = pe__target_rc_from_xml(lrm_op_xml); + + crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_RC, &rc); + if (rc != target_rc) { +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 02cef2c..0e8177b 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -3013,7 +3013,7 @@ static bool check_operation_expiry(resource_t *rsc, node_t *node, int rc, xmlNod + return expired; + } + +-int get_target_rc(xmlNode *xml_op) ++int pe__target_rc_from_xml(xmlNode *xml_op) + { + int target_rc = 0; + const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY); +@@ -3141,7 +3141,7 @@ unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, xmlNode ** last + + int rc = 0; + int status = PCMK_LRM_OP_UNKNOWN; +- int target_rc = get_target_rc(xml_op); ++ int target_rc = pe__target_rc_from_xml(xml_op); + guint interval_ms = 0; + + gboolean expired = FALSE; +-- +1.8.3.1 + + +From 2ccbefc2b623a2671f14824c6aea87c87fc338a0 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 24 May 2019 09:59:35 -0500 +Subject: [PATCH 03/11] Refactor: libpacemaker: make transition.h internal + +transition.h has always been installed even though it was purely internal. +Since libtransitioner is now merged into libpacemaker, move transition.h +to include/pcmki/pcmki_transition.h and make it internal. + +Also, get rid of pcmki_sched_transition.h since it no longer contains +anything transition-related, and move its two function declarations to +pcmki_sched_utils.h. +--- + daemons/controld/controld_execd_state.c | 4 +- + daemons/controld/controld_fencing.h | 2 +- + daemons/controld/controld_join_dc.c | 4 +- + daemons/controld/controld_transition.h | 2 +- + daemons/controld/controld_utils.h | 7 +- + include/crm/Makefile.am | 5 +- + include/crm/transition.h | 153 -------------------------------- + include/pacemaker-internal.h | 2 +- + include/pcmki/Makefile.am | 12 +-- + include/pcmki/pcmki_sched_transition.h | 22 ----- + include/pcmki/pcmki_sched_utils.h | 16 ++++ + include/pcmki/pcmki_transition.h | 143 +++++++++++++++++++++++++++++ + lib/pacemaker/pcmk_sched_transition.c | 3 +- + lib/pacemaker/pcmk_trans_graph.c | 27 ++---- + lib/pacemaker/pcmk_trans_unpack.c | 23 ++--- + lib/pacemaker/pcmk_trans_utils.c | 27 ++---- + tools/crm_simulate.c | 1 - + 17 files changed, 201 insertions(+), 252 deletions(-) + delete mode 100644 include/crm/transition.h + delete mode 100644 include/pcmki/pcmki_sched_transition.h + create mode 100644 include/pcmki/pcmki_transition.h + +diff --git a/daemons/controld/controld_execd_state.c b/daemons/controld/controld_execd_state.c +index 8a1a7f3..4e9f096 100644 +--- a/daemons/controld/controld_execd_state.c ++++ b/daemons/controld/controld_execd_state.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2012-2018 David Vossel ++ * Copyright 2012-2019 the Pacemaker project contributors + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. +@@ -10,6 +10,7 @@ + #include + #include + ++#include + #include + #include + #include +@@ -18,7 +19,6 @@ + #include + #include + #include +-#include + #include + + GHashTable *lrm_state_table = NULL; +diff --git a/daemons/controld/controld_fencing.h b/daemons/controld/controld_fencing.h +index 3ef537f..8f7f19b 100644 +--- a/daemons/controld/controld_fencing.h ++++ b/daemons/controld/controld_fencing.h +@@ -11,7 +11,7 @@ + # define CONTROLD_FENCING__H + + #include // bool +-#include // crm_graph_t, crm_action_t ++#include // crm_graph_t, crm_action_t + + // stonith fail counts + void st_fail_count_reset(const char * target); +diff --git a/daemons/controld/controld_join_dc.c b/daemons/controld/controld_join_dc.c +index ddee895..d790d9a 100644 +--- a/daemons/controld/controld_join_dc.c ++++ b/daemons/controld/controld_join_dc.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2018 Andrew Beekhof ++ * Copyright 2004-2019 the Pacemaker project contributors + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. +@@ -15,7 +15,7 @@ + + #include + #include +-#include "controld_transition.h" ++#include + + char *max_epoch = NULL; + char *max_generation_from = NULL; +diff --git a/daemons/controld/controld_transition.h b/daemons/controld/controld_transition.h +index f31ac2d..192a9e8 100644 +--- a/daemons/controld/controld_transition.h ++++ b/daemons/controld/controld_transition.h +@@ -8,10 +8,10 @@ + #ifndef TENGINE__H + # define TENGINE__H + +-# include + # include + # include + # include ++# include + + /* tengine */ + extern crm_action_t *match_down_event(const char *target); +diff --git a/daemons/controld/controld_utils.h b/daemons/controld/controld_utils.h +index 8b80e3c..1946a82 100644 +--- a/daemons/controld/controld_utils.h ++++ b/daemons/controld/controld_utils.h +@@ -11,11 +11,10 @@ + # define CRMD_UTILS__H + + # include +-# include + # include +-# include /* For CIB_OP_MODIFY */ +-# include "controld_fsa.h" // For fsa_cib_conn +-# include "controld_alerts.h" ++# include // CIB_OP_MODIFY ++# include // fsa_cib_conn ++# include + + # define FAKE_TE_ID "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + +diff --git a/include/crm/Makefile.am b/include/crm/Makefile.am +index 3f5f4bf..bc939eb 100644 +--- a/include/crm/Makefile.am ++++ b/include/crm/Makefile.am +@@ -1,5 +1,5 @@ + # +-# Copyright 2004-2018 the Pacemaker project contributors ++# Copyright 2004-2019 the Pacemaker project contributors + # + # The version control history for this file may have further details. + # +@@ -22,7 +22,6 @@ MAINTAINERCLEANFILES = Makefile.in + headerdir=$(pkgincludedir)/crm + + header_HEADERS = attrd.h cib.h cluster.h compatibility.h crm.h \ +- lrmd.h msg_xml.h services.h stonith-ng.h \ +- transition.h ++ lrmd.h msg_xml.h services.h stonith-ng.h + + SUBDIRS = common pengine cib fencing cluster +diff --git a/include/crm/transition.h b/include/crm/transition.h +deleted file mode 100644 +index 6e9a875..0000000 +--- a/include/crm/transition.h ++++ /dev/null +@@ -1,153 +0,0 @@ +-/* +- * Copyright 2004-2018 the Pacemaker project contributors +- * +- * The version control history for this file may have further details. +- * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of the GNU Lesser General Public +- * License as published by the Free Software Foundation; either +- * version 2 of the License, or (at your option) any later version. +- * +- * This software is distributed in the hope that it will be useful, +- * but WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * General Public License for more details. +- * +- * You should have received a copy of the GNU Lesser General Public +- * License along with this library; if not, write to the Free Software +- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +- */ +-#ifndef CRM_TRANSITION__H +-# define CRM_TRANSITION__H +- +-#ifdef __cplusplus +-extern "C" { +-#endif +- +-#include +-#include +-#include +- +-typedef enum { +- action_type_pseudo, +- action_type_rsc, +- action_type_crm +-} action_type_e; +- +-typedef struct te_timer_s crm_action_timer_t; +-typedef struct crm_graph_s crm_graph_t; +- +-typedef struct synapse_s { +- int id; +- int priority; +- +- gboolean ready; +- gboolean failed; +- gboolean executed; +- gboolean confirmed; +- +- GListPtr actions; /* crm_action_t* */ +- GListPtr inputs; /* crm_action_t* */ +-} synapse_t; +- +-typedef struct crm_action_s { +- int id; +- int timeout; +- guint interval_ms; +- GHashTable *params; +- action_type_e type; +- +- crm_action_timer_t *timer; +- synapse_t *synapse; +- +- gboolean sent_update; /* sent to the CIB */ +- gboolean executed; /* sent to the CRM */ +- gboolean confirmed; +- +- gboolean failed; +- gboolean can_fail; +- +- xmlNode *xml; +- +-} crm_action_t; +- +-struct te_timer_s { +- int source_id; +- int timeout; +- crm_action_t *action; +-}; +- +-/* order matters here */ +-enum transition_action { +- tg_done, +- tg_stop, +- tg_restart, +- tg_shutdown, +-}; +- +-struct crm_graph_s { +- int id; +- char *source; +- int abort_priority; +- +- gboolean complete; +- const char *abort_reason; +- enum transition_action completion_action; +- +- int num_actions; +- int num_synapses; +- +- int batch_limit; +- int network_delay; +- int stonith_timeout; +- int transition_timeout; +- +- int fired; +- int pending; +- int skipped; +- int completed; +- int incomplete; +- +- GListPtr synapses; /* synapse_t* */ +- +- int migration_limit; +-}; +- +-typedef struct crm_graph_functions_s { +- gboolean(*pseudo) (crm_graph_t * graph, crm_action_t * action); +- gboolean(*rsc) (crm_graph_t * graph, crm_action_t * action); +- gboolean(*crmd) (crm_graph_t * graph, crm_action_t * action); +- gboolean(*stonith) (crm_graph_t * graph, crm_action_t * action); +- gboolean(*allowed) (crm_graph_t * graph, crm_action_t * action); +-} crm_graph_functions_t; +- +-enum transition_status { +- transition_active, +- transition_pending, /* active but no actions performed this time */ +- transition_complete, +- transition_stopped, +- transition_terminated, +- transition_action_failed, +- transition_failed, +-}; +- +-void set_default_graph_functions(void); +-void set_graph_functions(crm_graph_functions_t * fns); +-crm_graph_t *unpack_graph(xmlNode * xml_graph, const char *reference); +-int run_graph(crm_graph_t * graph); +-gboolean update_graph(crm_graph_t * graph, crm_action_t * action); +-void destroy_graph(crm_graph_t * graph); +-const char *transition_status(enum transition_status state); +-void print_graph(unsigned int log_level, crm_graph_t * graph); +-void print_action(int log_level, const char *prefix, crm_action_t * action); +-bool update_abort_priority(crm_graph_t * graph, int priority, +- enum transition_action action, const char *abort_reason); +-const char *actiontype2text(action_type_e type); +-lrmd_event_data_t *convert_graph_action(xmlNode * resource, crm_action_t * action, int status, +- int rc); +- +-#ifdef __cplusplus +-} +-#endif +- +-#endif +diff --git a/include/pacemaker-internal.h b/include/pacemaker-internal.h +index 3627ba5..51d7225 100644 +--- a/include/pacemaker-internal.h ++++ b/include/pacemaker-internal.h +@@ -13,8 +13,8 @@ + # include + # include + # include +-# include + # include + # include ++# include + + #endif +diff --git a/include/pcmki/Makefile.am b/include/pcmki/Makefile.am +index b163e89..4cf1cf2 100644 +--- a/include/pcmki/Makefile.am ++++ b/include/pcmki/Makefile.am +@@ -9,11 +9,11 @@ + + MAINTAINERCLEANFILES = Makefile.in + +-noinst_HEADERS = pcmki_error.h \ +- pcmki_sched_allocate.h \ +- pcmki_sched_notif.h \ +- pcmki_sched_transition.h \ +- pcmki_sched_utils.h \ +- pcmki_scheduler.h ++noinst_HEADERS = pcmki_error.h \ ++ pcmki_sched_allocate.h \ ++ pcmki_sched_notif.h \ ++ pcmki_sched_utils.h \ ++ pcmki_scheduler.h \ ++ pcmki_transition.h + + .PHONY: $(ARCHIVE_VERSION) +diff --git a/include/pcmki/pcmki_sched_transition.h b/include/pcmki/pcmki_sched_transition.h +deleted file mode 100644 +index 41f5d61..0000000 +--- a/include/pcmki/pcmki_sched_transition.h ++++ /dev/null +@@ -1,22 +0,0 @@ +-/* +- * Copyright 2014-2019 the Pacemaker project contributors +- * +- * The version control history for this file may have further details. +- * +- * This source code is licensed under the GNU Lesser General Public License +- * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. +- */ +-#ifndef SCHED_TRANSITION__H +-# define SCHED_TRANSITION__H +- +-#include +- +-void modify_configuration( +- pe_working_set_t * data_set, cib_t *cib, +- const char *quorum, const char *watchdog, GListPtr node_up, GListPtr node_down, GListPtr node_fail, +- GListPtr op_inject, GListPtr ticket_grant, GListPtr ticket_revoke, +- GListPtr ticket_standby, GListPtr ticket_activate); +- +-int run_simulation(pe_working_set_t * data_set, cib_t *cib, GListPtr op_fail_list, bool quiet); +- +-#endif +diff --git a/include/pcmki/pcmki_sched_utils.h b/include/pcmki/pcmki_sched_utils.h +index b47a2bb..4361235 100644 +--- a/include/pcmki/pcmki_sched_utils.h ++++ b/include/pcmki/pcmki_sched_utils.h +@@ -10,6 +10,14 @@ + #ifndef PENGINE_AUTILS__H + # define PENGINE_AUTILS__H + ++#include // bool ++#include // GList, GHashTable, gboolean, guint ++#include // GListPtr ++#include // cib_t ++#include ++#include ++#include ++ + /* Constraint helper functions */ + extern rsc_colocation_t *invert_constraint(rsc_colocation_t * constraint); + +@@ -75,4 +83,12 @@ pe_action_t *sched_shutdown_op(pe_node_t *node, pe_working_set_t *data_set); + + # define LOAD_STOPPED "load_stopped" + ++void modify_configuration( ++ pe_working_set_t * data_set, cib_t *cib, ++ const char *quorum, const char *watchdog, GListPtr node_up, GListPtr node_down, GListPtr node_fail, ++ GListPtr op_inject, GListPtr ticket_grant, GListPtr ticket_revoke, ++ GListPtr ticket_standby, GListPtr ticket_activate); ++ ++int run_simulation(pe_working_set_t * data_set, cib_t *cib, GListPtr op_fail_list, bool quiet); ++ + #endif +diff --git a/include/pcmki/pcmki_transition.h b/include/pcmki/pcmki_transition.h +new file mode 100644 +index 0000000..d9a0ff6 +--- /dev/null ++++ b/include/pcmki/pcmki_transition.h +@@ -0,0 +1,143 @@ ++/* ++ * Copyright 2004-2019 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. ++ * ++ * This source code is licensed under the GNU Lesser General Public License ++ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. ++ */ ++ ++#ifndef CRM_TRANSITION__H ++# define CRM_TRANSITION__H ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include ++#include ++#include ++ ++typedef enum { ++ action_type_pseudo, ++ action_type_rsc, ++ action_type_crm ++} action_type_e; ++ ++typedef struct te_timer_s crm_action_timer_t; ++typedef struct crm_graph_s crm_graph_t; ++ ++typedef struct synapse_s { ++ int id; ++ int priority; ++ ++ gboolean ready; ++ gboolean failed; ++ gboolean executed; ++ gboolean confirmed; ++ ++ GListPtr actions; /* crm_action_t* */ ++ GListPtr inputs; /* crm_action_t* */ ++} synapse_t; ++ ++typedef struct crm_action_s { ++ int id; ++ int timeout; ++ guint interval_ms; ++ GHashTable *params; ++ action_type_e type; ++ ++ crm_action_timer_t *timer; ++ synapse_t *synapse; ++ ++ gboolean sent_update; /* sent to the CIB */ ++ gboolean executed; /* sent to the CRM */ ++ gboolean confirmed; ++ ++ gboolean failed; ++ gboolean can_fail; ++ ++ xmlNode *xml; ++ ++} crm_action_t; ++ ++struct te_timer_s { ++ int source_id; ++ int timeout; ++ crm_action_t *action; ++}; ++ ++/* order matters here */ ++enum transition_action { ++ tg_done, ++ tg_stop, ++ tg_restart, ++ tg_shutdown, ++}; ++ ++struct crm_graph_s { ++ int id; ++ char *source; ++ int abort_priority; ++ ++ gboolean complete; ++ const char *abort_reason; ++ enum transition_action completion_action; ++ ++ int num_actions; ++ int num_synapses; ++ ++ int batch_limit; ++ int network_delay; ++ int stonith_timeout; ++ int transition_timeout; ++ ++ int fired; ++ int pending; ++ int skipped; ++ int completed; ++ int incomplete; ++ ++ GListPtr synapses; /* synapse_t* */ ++ ++ int migration_limit; ++}; ++ ++typedef struct crm_graph_functions_s { ++ gboolean(*pseudo) (crm_graph_t * graph, crm_action_t * action); ++ gboolean(*rsc) (crm_graph_t * graph, crm_action_t * action); ++ gboolean(*crmd) (crm_graph_t * graph, crm_action_t * action); ++ gboolean(*stonith) (crm_graph_t * graph, crm_action_t * action); ++ gboolean(*allowed) (crm_graph_t * graph, crm_action_t * action); ++} crm_graph_functions_t; ++ ++enum transition_status { ++ transition_active, ++ transition_pending, /* active but no actions performed this time */ ++ transition_complete, ++ transition_stopped, ++ transition_terminated, ++ transition_action_failed, ++ transition_failed, ++}; ++ ++void set_default_graph_functions(void); ++void set_graph_functions(crm_graph_functions_t * fns); ++crm_graph_t *unpack_graph(xmlNode * xml_graph, const char *reference); ++int run_graph(crm_graph_t * graph); ++gboolean update_graph(crm_graph_t * graph, crm_action_t * action); ++void destroy_graph(crm_graph_t * graph); ++const char *transition_status(enum transition_status state); ++void print_graph(unsigned int log_level, crm_graph_t * graph); ++void print_action(int log_level, const char *prefix, crm_action_t * action); ++bool update_abort_priority(crm_graph_t * graph, int priority, ++ enum transition_action action, const char *abort_reason); ++const char *actiontype2text(action_type_e type); ++lrmd_event_data_t *convert_graph_action(xmlNode * resource, crm_action_t * action, int status, ++ int rc); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/lib/pacemaker/pcmk_sched_transition.c b/lib/pacemaker/pcmk_sched_transition.c +index 0fa5709..8ab8d82 100644 +--- a/lib/pacemaker/pcmk_sched_transition.c ++++ b/lib/pacemaker/pcmk_sched_transition.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2009-2018 Andrew Beekhof ++ * Copyright 2009-2019 the Pacemaker project contributors + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. +@@ -19,7 +19,6 @@ + #include + #include + #include +-#include + #include + #include + #include +diff --git a/lib/pacemaker/pcmk_trans_graph.c b/lib/pacemaker/pcmk_trans_graph.c +index 71568dd..77980e5 100644 +--- a/lib/pacemaker/pcmk_trans_graph.c ++++ b/lib/pacemaker/pcmk_trans_graph.c +@@ -1,19 +1,10 @@ +-/* +- * Copyright (C) 2004 Andrew Beekhof +- * +- * This library is free software; you can redistribute it and/or +- * modify it under the terms of the GNU Lesser General Public +- * License as published by the Free Software Foundation; either +- * version 2.1 of the License, or (at your option) any later version. +- * +- * This library is distributed in the hope that it will be useful, +- * but WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * Lesser General Public License for more details. +- * +- * You should have received a copy of the GNU Lesser General Public +- * License along with this library; if not, write to the Free Software +- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA ++/* ++ * Copyright 2004-2019 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. ++ * ++ * This source code is licensed under the GNU Lesser General Public License ++ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ + + #include +@@ -21,9 +12,7 @@ + #include + #include + #include +-#include +-/* #include */ +-/* */ ++#include + + crm_graph_functions_t *graph_fns = NULL; + +diff --git a/lib/pacemaker/pcmk_trans_unpack.c b/lib/pacemaker/pcmk_trans_unpack.c +index 31e39cb..b8147a9 100644 +--- a/lib/pacemaker/pcmk_trans_unpack.c ++++ b/lib/pacemaker/pcmk_trans_unpack.c +@@ -1,30 +1,21 @@ + /* +- * Copyright (C) 2004 Andrew Beekhof ++ * Copyright 2004-2019 the Pacemaker project contributors + * +- * This library is free software; you can redistribute it and/or +- * modify it under the terms of the GNU Lesser General Public +- * License as published by the Free Software Foundation; either +- * version 2.1 of the License, or (at your option) any later version. ++ * The version control history for this file may have further details. + * +- * This library is distributed in the hope that it will be useful, +- * but WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * Lesser General Public License for more details. +- * +- * You should have received a copy of the GNU Lesser General Public +- * License along with this library; if not, write to the Free Software +- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA ++ * This source code is licensed under the GNU Lesser General Public License ++ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ + + #include + + #include ++#include ++ + #include + #include +- + #include +-#include +-#include ++#include + + CRM_TRACE_INIT_DATA(transitioner); + +diff --git a/lib/pacemaker/pcmk_trans_utils.c b/lib/pacemaker/pcmk_trans_utils.c +index d3199d9..69da7f4 100644 +--- a/lib/pacemaker/pcmk_trans_utils.c ++++ b/lib/pacemaker/pcmk_trans_utils.c +@@ -1,19 +1,10 @@ +-/* +- * Copyright (C) 2004 Andrew Beekhof +- * +- * This library is free software; you can redistribute it and/or +- * modify it under the terms of the GNU Lesser General Public +- * License as published by the Free Software Foundation; either +- * version 2.1 of the License, or (at your option) any later version. +- * +- * This library is distributed in the hope that it will be useful, +- * but WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * Lesser General Public License for more details. +- * +- * You should have received a copy of the GNU Lesser General Public +- * License along with this library; if not, write to the Free Software +- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA ++/* ++ * Copyright 2004-2019 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. ++ * ++ * This source code is licensed under the GNU Lesser General Public License ++ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ + + #include +@@ -21,9 +12,7 @@ + #include + #include + #include +-#include +-/* #include */ +-/* */ ++#include + + extern crm_graph_functions_t *graph_fns; + +diff --git a/tools/crm_simulate.c b/tools/crm_simulate.c +index 1921ee4..d4ab6a3 100644 +--- a/tools/crm_simulate.c ++++ b/tools/crm_simulate.c +@@ -21,7 +21,6 @@ + #include + #include + #include +-#include + #include + #include + #include +-- +1.8.3.1 + + +From 8a0b29d8ed21c97075b4c059fa4b0f5c0d985a73 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 6 Jun 2019 14:18:37 -0500 +Subject: [PATCH 04/11] Test: scheduler: explicitly set concurrent-fencing in + relevant regression tests + +... since concurrent-fencing's default is likely to eventually change, +which would otherwise affect the results of these tests +--- + cts/scheduler/rec-node-14.xml | 1 + + cts/scheduler/remote-connection-unrecoverable.xml | 1 + + cts/scheduler/remote-recover-all.xml | 1 + + cts/scheduler/remote-recover-no-resources.xml | 1 + + cts/scheduler/remote-recover-unknown.xml | 1 + + cts/scheduler/stonith-4.xml | 1 + + cts/scheduler/suicide-needed-inquorate.xml | 1 + + cts/scheduler/ticket-clone-21.xml | 1 + + cts/scheduler/ticket-clone-9.xml | 1 + + 9 files changed, 9 insertions(+) + +diff --git a/cts/scheduler/rec-node-14.xml b/cts/scheduler/rec-node-14.xml +index 60307ba..aefa410 100644 +--- a/cts/scheduler/rec-node-14.xml ++++ b/cts/scheduler/rec-node-14.xml +@@ -4,6 +4,7 @@ + + + ++ + + + +diff --git a/cts/scheduler/remote-connection-unrecoverable.xml b/cts/scheduler/remote-connection-unrecoverable.xml +index df9fee2..efec646 100644 +--- a/cts/scheduler/remote-connection-unrecoverable.xml ++++ b/cts/scheduler/remote-connection-unrecoverable.xml +@@ -7,6 +7,7 @@ + + + ++ + + + +diff --git a/cts/scheduler/remote-recover-all.xml b/cts/scheduler/remote-recover-all.xml +index 0ade7cd..1680166 100644 +--- a/cts/scheduler/remote-recover-all.xml ++++ b/cts/scheduler/remote-recover-all.xml +@@ -10,6 +10,7 @@ + + + ++ + + + +diff --git a/cts/scheduler/remote-recover-no-resources.xml b/cts/scheduler/remote-recover-no-resources.xml +index 37708bb..602ed2b 100644 +--- a/cts/scheduler/remote-recover-no-resources.xml ++++ b/cts/scheduler/remote-recover-no-resources.xml +@@ -10,6 +10,7 @@ + + + ++ + + + +diff --git a/cts/scheduler/remote-recover-unknown.xml b/cts/scheduler/remote-recover-unknown.xml +index f070f11..f47a841 100644 +--- a/cts/scheduler/remote-recover-unknown.xml ++++ b/cts/scheduler/remote-recover-unknown.xml +@@ -10,6 +10,7 @@ + + + ++ + + + +diff --git a/cts/scheduler/stonith-4.xml b/cts/scheduler/stonith-4.xml +index 7979462..dd7af8d 100644 +--- a/cts/scheduler/stonith-4.xml ++++ b/cts/scheduler/stonith-4.xml +@@ -4,6 +4,7 @@ + + + ++ + + + +diff --git a/cts/scheduler/suicide-needed-inquorate.xml b/cts/scheduler/suicide-needed-inquorate.xml +index e626ea6..f87422b 100644 +--- a/cts/scheduler/suicide-needed-inquorate.xml ++++ b/cts/scheduler/suicide-needed-inquorate.xml +@@ -6,6 +6,7 @@ + + + ++ + + + +diff --git a/cts/scheduler/ticket-clone-21.xml b/cts/scheduler/ticket-clone-21.xml +index bb1f044..efd5294 100644 +--- a/cts/scheduler/ticket-clone-21.xml ++++ b/cts/scheduler/ticket-clone-21.xml +@@ -4,6 +4,7 @@ + + + ++ + + + +diff --git a/cts/scheduler/ticket-clone-9.xml b/cts/scheduler/ticket-clone-9.xml +index e77210d..c6d5809 100644 +--- a/cts/scheduler/ticket-clone-9.xml ++++ b/cts/scheduler/ticket-clone-9.xml +@@ -4,6 +4,7 @@ + + + ++ + + + +-- +1.8.3.1 + + +From 359f0e6089ef618361acc2437d779ecad1edb8d3 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 6 Jun 2019 11:22:58 -0500 +Subject: [PATCH 05/11] Build: doc: define variable properly + +broke when moved from GNUmakefile to Makefile.am +--- + doc/Makefile.am | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/doc/Makefile.am b/doc/Makefile.am +index f3b79bb..18747cf 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -25,6 +25,9 @@ RSYNC_DEST ?= root@www.clusterlabs.org:/var/www/html + # don't cross filesystems, sparse, show progress + RSYNC_OPTS = -rlptvzxS --progress + ++LAST_RELEASE ?= Pacemaker-$(VERSION) ++TAG ?= $(shell git log --pretty=format:%H -n 1 HEAD) ++ + publican_docs = + generated_docs = + generated_mans = +@@ -364,9 +367,6 @@ doxygen-upload: doxygen + + # ABI compatibility report as HTML + +-LAST_RELEASE ?= Pacemaker-$(VERSION) +-TAG ?= $(git log --pretty=format:%H -n 1 HEAD) +- + abi: abi-check + ./abi-check $(PACKAGE) $(LAST_RELEASE) $(TAG) + +-- +1.8.3.1 + + +From c0e1ff4cc8578b78b085b98effff11747f81a397 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 10 Jun 2019 11:38:51 -0500 +Subject: [PATCH 06/11] Doc: doxygen: avoid full paths in output graphics + +broke when doxygen target moved from toplevel to doc subdirectory +--- + doc/Doxyfile.in | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in +index 81615fb..bc449df 100644 +--- a/doc/Doxyfile.in ++++ b/doc/Doxyfile.in +@@ -142,7 +142,7 @@ FULL_PATH_NAMES = YES + # will be relative from the directory where doxygen is started. + # This tag requires that the tag FULL_PATH_NAMES is set to YES. + +-STRIP_FROM_PATH = ++STRIP_FROM_PATH = .. + + # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the + # path mentioned in the documentation of a class, which tells the reader which +@@ -151,7 +151,7 @@ STRIP_FROM_PATH = + # specify the list of include paths that are normally passed to the compiler + # using the -I flag. + +-STRIP_FROM_INC_PATH = ++STRIP_FROM_INC_PATH = .. + + # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but + # less readable) file names. This can be useful is your file systems doesn't +-- +1.8.3.1 + + +From a0ab603c5c416148132a91f5bf22d55e65f8ba4e Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 11 Jun 2019 14:26:28 -0500 +Subject: [PATCH 07/11] Low: xml: add API schema for list_item() output + +--- + xml/Makefile.am | 9 +++++---- + xml/api/item-1.1.rng | 19 +++++++++++++++++++ + 2 files changed, 24 insertions(+), 4 deletions(-) + create mode 100644 xml/api/item-1.1.rng + +diff --git a/xml/Makefile.am b/xml/Makefile.am +index 88edbda..49542a3 100644 +--- a/xml/Makefile.am ++++ b/xml/Makefile.am +@@ -1,5 +1,5 @@ + # +-# Copyright 2004-2018 the Pacemaker project contributors ++# Copyright 2004-2019 the Pacemaker project contributors + # + # The version control history for this file may have further details. + # +@@ -63,8 +63,9 @@ RNG_max ?= $(lastword $(RNG_numeric_versions)) + + # A sorted list of all API and RNG versions (numeric and "next") + API_versions = next $(API_numeric_versions) +-API_base = command-output stonith_admin +-API_files = $(foreach base,$(API_base),$(wildcard api/$(base)*.rng)) ++API_request_base = command-output stonith_admin ++API_base = $(API_request_base) item ++API_files = $(foreach base,$(API_base),$(wildcard api/$(base)*.rng)) + + RNG_versions = next $(RNG_numeric_versions) + RNG_version_pairs = $(call version_pairs,${RNG_numeric_versions}) +@@ -139,7 +140,7 @@ api/api-result-%.rng: $(API_files) Makefile.am + echo ' ' >> $@ + echo ' ' >> $@ + echo ' ' >> $@ +- for rng in $(API_base); do $(top_srcdir)/xml/best-match.sh api/$$rng $(*) $(@) " " || :; done ++ for rng in $(API_request_base); do $(top_srcdir)/xml/best-match.sh api/$$rng $(*) $(@) " " || :; done + echo ' ' >> $@ + echo ' ' >> $@ + echo ' ' >> $@ +diff --git a/xml/api/item-1.1.rng b/xml/api/item-1.1.rng +new file mode 100644 +index 0000000..1a065ca +--- /dev/null ++++ b/xml/api/item-1.1.rng +@@ -0,0 +1,19 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +-- +1.8.3.1 + + +From 311d8629241d227dded598225d8f413c9ebb4a9b Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 11 Jun 2019 18:47:10 -0500 +Subject: [PATCH 08/11] Refactor: fencing: expose function for parsing targets + from string + +... as internal API, so it can be reused elsewhere. Also, refactor it for +simplicity and versatility. +--- + daemons/fenced/fenced_commands.c | 91 +--------------------------- + include/crm/fencing/internal.h | 3 + + lib/fencing/st_client.c | 128 +++++++++++++++++++++++++++++++++++++++ + 3 files changed, 133 insertions(+), 89 deletions(-) + +diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c +index 1be2508..54575f9 100644 +--- a/daemons/fenced/fenced_commands.c ++++ b/daemons/fenced/fenced_commands.c +@@ -617,93 +617,6 @@ build_port_aliases(const char *hostmap, GListPtr * targets) + return aliases; + } + +-static void +-parse_host_line(const char *line, int max, GListPtr * output) +-{ +- int lpc = 0; +- int last = 0; +- +- if (max <= 0) { +- return; +- } +- +- /* Check for any complaints about additional parameters that the device doesn't understand */ +- if (strstr(line, "invalid") || strstr(line, "variable")) { +- crm_debug("Skipping: %s", line); +- return; +- } +- +- crm_trace("Processing %d bytes: [%s]", max, line); +- /* Skip initial whitespace */ +- for (lpc = 0; lpc <= max && isspace(line[lpc]); lpc++) { +- last = lpc + 1; +- } +- +- /* Now the actual content */ +- for (lpc = 0; lpc <= max; lpc++) { +- gboolean a_space = isspace(line[lpc]); +- +- if (a_space && lpc < max && isspace(line[lpc + 1])) { +- /* fast-forward to the end of the spaces */ +- +- } else if (a_space || line[lpc] == ',' || line[lpc] == ';' || line[lpc] == 0) { +- int rc = 1; +- char *entry = NULL; +- +- if (lpc != last) { +- entry = calloc(1, 1 + lpc - last); +- rc = sscanf(line + last, "%[a-zA-Z0-9_-.]", entry); +- } +- +- if (entry == NULL) { +- /* Skip */ +- } else if (rc != 1) { +- crm_warn("Could not parse (%d %d): %s", last, lpc, line + last); +- } else if (safe_str_neq(entry, "on") && safe_str_neq(entry, "off")) { +- crm_trace("Adding '%s'", entry); +- *output = g_list_append(*output, entry); +- entry = NULL; +- } +- +- free(entry); +- last = lpc + 1; +- } +- } +-} +- +-static GListPtr +-parse_host_list(const char *hosts) +-{ +- int lpc = 0; +- int max = 0; +- int last = 0; +- GListPtr output = NULL; +- +- if (hosts == NULL) { +- return output; +- } +- +- max = strlen(hosts); +- for (lpc = 0; lpc <= max; lpc++) { +- if (hosts[lpc] == '\n' || hosts[lpc] == 0) { +- int len = lpc - last; +- +- if(len > 1) { +- char *line = strndup(hosts + last, len); +- +- line[len] = 0; /* Because it might be '\n' */ +- parse_host_line(line, len, &output); +- free(line); +- } +- +- last = lpc + 1; +- } +- } +- +- crm_trace("Parsed %d entries from '%s'", g_list_length(output), hosts); +- return output; +-} +- + GHashTable *metadata_cache = NULL; + + void +@@ -937,7 +850,7 @@ build_device_from_xml(xmlNode * msg) + + value = g_hash_table_lookup(device->params, STONITH_ATTR_HOSTLIST); + if (value) { +- device->targets = parse_host_list(value); ++ device->targets = stonith__parse_targets(value); + } + + value = g_hash_table_lookup(device->params, STONITH_ATTR_HOSTMAP); +@@ -1108,7 +1021,7 @@ dynamic_list_search_cb(GPid pid, int rc, const char *output, gpointer user_data) + } else if (!rc) { + crm_info("Refreshing port list for %s", dev->id); + g_list_free_full(dev->targets, free); +- dev->targets = parse_host_list(output); ++ dev->targets = stonith__parse_targets(output); + dev->targets_age = time(NULL); + } + +diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h +index 0c0ac70..f3d38a7 100644 +--- a/include/crm/fencing/internal.h ++++ b/include/crm/fencing/internal.h +@@ -10,6 +10,7 @@ + #ifndef STONITH_NG_INTERNAL__H + # define STONITH_NG_INTERNAL__H + ++# include + # include + # include + # include +@@ -49,6 +50,8 @@ xmlNode *create_device_registration_xml(const char *id, + + void stonith__register_messages(pcmk__output_t *out); + ++GList *stonith__parse_targets(const char *hosts); ++ + # define ST_LEVEL_MAX 10 + + # define F_STONITH_CLIENTID "st_clientid" +diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c +index d949fe1..629887a 100644 +--- a/lib/fencing/st_client.c ++++ b/lib/fencing/st_client.c +@@ -9,6 +9,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -2304,3 +2305,130 @@ stonith_action_str(const char *action) + return action; + } + } ++ ++/*! ++ * \internal ++ * \brief Parse a target name from one line of a target list string ++ * ++ * \param[in] line One line of a target list string ++ * \parma[in] len String length of line ++ * \param[in,out] output List to add newly allocated target name to ++ */ ++static void ++parse_list_line(const char *line, int len, GList **output) ++{ ++ size_t i = 0; ++ size_t entry_start = 0; ++ ++ /* Skip complaints about additional parameters device doesn't understand ++ * ++ * @TODO Document or eliminate the implied restriction of target names ++ */ ++ if (strstr(line, "invalid") || strstr(line, "variable")) { ++ crm_debug("Skipping list output line: %s", line); ++ return; ++ } ++ ++ // Process line content, character by character ++ for (i = 0; i <= len; i++) { ++ ++ if (isspace(line[i]) || (line[i] == ',') || (line[i] == ';') ++ || (line[i] == '\0')) { ++ // We've found a separator (i.e. the end of an entry) ++ ++ int rc = 0; ++ char *entry = NULL; ++ ++ if (i == entry_start) { ++ // Skip leading and sequential separators ++ entry_start = i + 1; ++ continue; ++ } ++ ++ entry = calloc(i - entry_start + 1, sizeof(char)); ++ CRM_ASSERT(entry != NULL); ++ ++ /* Read entry, stopping at first separator ++ * ++ * @TODO Document or eliminate these character restrictions ++ */ ++ rc = sscanf(line + entry_start, "%[a-zA-Z0-9_-.]", entry); ++ if (rc != 1) { ++ crm_warn("Could not parse list output entry: %s " ++ CRM_XS " entry_start=%d position=%d", ++ line + entry_start, entry_start, i); ++ free(entry); ++ ++ } else if (safe_str_eq(entry, "on") || safe_str_eq(entry, "off")) { ++ /* Some agents print the target status in the list output, ++ * though none are known now (the separate list-status command ++ * is used for this, but it can also print "UNKNOWN"). To handle ++ * this possibility, skip such entries. ++ * ++ * @TODO Document or eliminate the implied restriction of target ++ * names. ++ */ ++ free(entry); ++ ++ } else { ++ // We have a valid entry ++ *output = g_list_append(*output, entry); ++ } ++ entry_start = i + 1; ++ } ++ } ++} ++ ++/*! ++ * \internal ++ * \brief Parse a list of targets from a string ++ * ++ * \param[in] list_output Target list as a string ++ * ++ * \return List of target names ++ * \note The target list string format is flexible, to allow for user-specified ++ * lists such pcmk_host_list and the output of an agent's list action ++ * (whether direct or via the API, which escapes newlines). There may be ++ * multiple lines, separated by either a newline or an escaped newline ++ * (backslash n). Each line may have one or more target names, separated ++ * by any combination of whitespace, commas, and semi-colons. Lines ++ * containing "invalid" or "variable" will be ignored entirely. Target ++ * names "on" or "off" (case-insensitive) will be ignored. Target names ++ * may contain only alphanumeric characters, underbars (_), dashes (-), ++ * and dots (.) (if any other character occurs in the name, it and all ++ * subsequent characters in the name will be ignored). ++ * \note The caller is responsible for freeing the result with ++ * g_list_free_full(result, free). ++ */ ++GList * ++stonith__parse_targets(const char *target_spec) ++{ ++ GList *targets = NULL; ++ ++ if (target_spec != NULL) { ++ size_t out_len = strlen(target_spec); ++ size_t line_start = 0; // Starting index of line being processed ++ ++ for (size_t i = 0; i <= out_len; ++i) { ++ if ((target_spec[i] == '\n') || (target_spec[i] == '\0') ++ || ((target_spec[i] == '\\') && (target_spec[i + 1] == 'n'))) { ++ // We've reached the end of one line of output ++ ++ int len = i - line_start; ++ ++ if (len > 0) { ++ char *line = strndup(target_spec + line_start, len); ++ ++ line[len] = '\0'; // Because it might be a newline ++ parse_list_line(line, len, &targets); ++ free(line); ++ } ++ if (target_spec[i] == '\\') { ++ ++i; // backslash-n takes up two positions ++ } ++ line_start = i + 1; ++ } ++ } ++ } ++ return targets; ++} +-- +1.8.3.1 + + +From 60ad7730fbf34c1f67700bace39a083c0e3d1c31 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 10 Jun 2019 16:13:29 -0500 +Subject: [PATCH 09/11] Fix: tools: stonith_admin --list-targets should show + what fencer would use + +This fixes both a regression in 2.0.2 and a pre-existing issue before that. + +Before 2.0.2's 52f614aa, stonith_admin --list-targets would print each line of +output from the fence agent's list action, with any commas and semi-colons +removed. After that, it would only print lines that had three values separated +by spaces. + +In practice, fence agents have some variety in list action output, which is not +yet standardized by the fence agent API. Only fence_xvm is known to use the +three space-separated values. Most agents output the target name and an alias +separated by a comma. + +The earlier behavior would actually be awkward in the comma-separated case, +since the target name and alias would be run together with no separator. + +Neither behaviors matched what was actually used by the fencer. + +This commit refactors to use the new stonith__parse_targets() function, to show +the same list the fencer would use. It also fixes a few related issues: + +* Memory was not properly freed + +* No list wrapper would be printed if the list were empty + +* stonith_admin's XML output did not match its schema (the tool would output a + element, while the schema had ). Now, we abandon the + custom element and use the generic schema instead. While technically + this could be considered backward-incompatible, it's not really, because the + schema didn't match to begin with. Also, the API XML schema is still + considered experimental. + +* Not really a problem, but since we now have the generic schema, + stonith_admin uses this in place of its former dedicated schema. + The only difference is the former allows arbitrary strings while the + latter required NCName, but the reuse is more useful than type validation. +--- + lib/fencing/st_output.c | 28 ---------------- + tools/stonith_admin.c | 52 +++++------------------------- + xml/api/stonith_admin-1.1.rng | 75 +++++++++++++++++++++++++++++++++++++++++++ + 3 files changed, 83 insertions(+), 72 deletions(-) + create mode 100644 xml/api/stonith_admin-1.1.rng + +diff --git a/lib/fencing/st_output.c b/lib/fencing/st_output.c +index a8d0a60..0ceb699 100644 +--- a/lib/fencing/st_output.c ++++ b/lib/fencing/st_output.c +@@ -17,32 +17,6 @@ + #include + + static int +-fence_target_text(pcmk__output_t *out, va_list args) { +- const char *hostname = va_arg(args, const char *); +- const char *uuid = va_arg(args, const char *); +- const char *status = va_arg(args, const char *); +- +- pcmk__indented_printf(out, "%s\t%s\t%s\n", hostname, uuid, status); +- return 0; +-} +- +-static int +-fence_target_xml(pcmk__output_t *out, va_list args) { +- xmlNodePtr node = NULL; +- const char *hostname = va_arg(args, const char *); +- const char *uuid = va_arg(args, const char *); +- const char *status = va_arg(args, const char *); +- +- node = xmlNewNode(NULL, (pcmkXmlStr) "target"); +- xmlSetProp(node, (pcmkXmlStr) "hostname", (pcmkXmlStr) hostname); +- xmlSetProp(node, (pcmkXmlStr) "uuid", (pcmkXmlStr) uuid); +- xmlSetProp(node, (pcmkXmlStr) "status", (pcmkXmlStr) status); +- +- pcmk__xml_add_node(out, node); +- return 0; +-} +- +-static int + last_fenced_text(pcmk__output_t *out, va_list args) { + const char *target = va_arg(args, const char *); + time_t when = va_arg(args, time_t); +@@ -216,8 +190,6 @@ validate_agent_xml(pcmk__output_t *out, va_list args) { + } + + static pcmk__message_entry_t fmt_functions[] = { +- { "fence-target", "text", fence_target_text }, +- { "fence-target", "xml", fence_target_xml }, + { "last-fenced", "text", last_fenced_text }, + { "last-fenced", "xml", last_fenced_xml }, + { "stonith-event", "text", stonith_event_text }, +diff --git a/tools/stonith_admin.c b/tools/stonith_admin.c +index 6be66c6..a7551fd 100644 +--- a/tools/stonith_admin.c ++++ b/tools/stonith_admin.c +@@ -635,53 +635,17 @@ main(int argc, char **argv) + break; + case 's': + rc = st->cmds->list(st, st_opts, device, &lists, timeout); +- if (rc == 0 && lists) { +- char *head = lists; +- char *eol = NULL; ++ if (rc == 0) { ++ GList *targets = stonith__parse_targets(lists); + + out->begin_list(out, "Fence targets", "fence target", "fence targets"); +- +- do { +- char *line = NULL; +- char *elem = NULL; +- +- char *hostname = NULL; +- char *uuid = NULL; +- char *status = NULL; +- +- eol = strstr(head, "\\n"); +- line = strndup(head, eol-head); +- +- while ((elem = strsep(&line, " ")) != NULL) { +- if (strcmp(elem, "") == 0) { +- continue; +- } +- +- if (hostname == NULL) { +- hostname = elem; +- } else if (uuid == NULL) { +- uuid = elem; +- } else if (status == NULL) { +- char *end = NULL; +- status = elem; +- +- end = strchr(status, '\n'); +- if (end != NULL) { +- *end = '\0'; +- } +- } +- } +- +- if (hostname != NULL && uuid != NULL && status != NULL) { +- out->message(out, "fence-target", hostname, uuid, status); +- } +- +- free(line); +- +- head = eol+2; +- } while (eol != NULL); +- ++ while (targets != NULL) { ++ out->list_item(out, NULL, (const char *) targets->data); ++ targets = targets->next; ++ } + out->end_list(out); ++ free(lists); ++ + } else if (rc != 0) { + fprintf(stderr, "List command returned error. rc : %d\n", rc); + } +diff --git a/xml/api/stonith_admin-1.1.rng b/xml/api/stonith_admin-1.1.rng +new file mode 100644 +index 0000000..997670f +--- /dev/null ++++ b/xml/api/stonith_admin-1.1.rng +@@ -0,0 +1,75 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ failed ++ success ++ pending ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +-- +1.8.3.1 + + +From 8cc030c045e0a0b43a2a0dcfec5541e5e07faed3 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 10 Jun 2019 17:41:32 -0500 +Subject: [PATCH 10/11] Fix: libcrmcommon: add stderr source correctly when + outputting XML + +--- + lib/common/output_xml.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/lib/common/output_xml.c b/lib/common/output_xml.c +index c93d66c..994af43 100644 +--- a/lib/common/output_xml.c ++++ b/lib/common/output_xml.c +@@ -130,7 +130,7 @@ xml_subprocess_output(pcmk__output_t *out, int exit_status, + if (proc_stderr != NULL) { + child_node = xmlNewTextChild(node, NULL, (pcmkXmlStr) "output", + (pcmkXmlStr) proc_stderr); +- xmlSetProp(node, (pcmkXmlStr) "source", (pcmkXmlStr) "stderr"); ++ xmlSetProp(child_node, (pcmkXmlStr) "source", (pcmkXmlStr) "stderr"); + } + + pcmk__xml_add_node(out, node); +-- +1.8.3.1 + + +From 4ce8272fde2605099b9d4bb1e211bc66c4f79f90 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 12 Jun 2019 19:56:09 -0500 +Subject: [PATCH 11/11] Test: CTS: update pattern for changed log message + +changed in 0a884f32 +--- + cts/CM_common.py | 4 ++-- + cts/patterns.py | 2 +- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/cts/CM_common.py b/cts/CM_common.py +index 0112fec..b7ff223 100755 +--- a/cts/CM_common.py ++++ b/cts/CM_common.py +@@ -12,7 +12,7 @@ from __future__ import print_function, unicode_literals, absolute_import, divisi + __copyright__ = """Original Author: Huang Zhen + Copyright 2004 International Business Machines + +-with later changes copyright 2004-2018 the Pacemaker project contributors. ++with later changes copyright 2004-2019 the Pacemaker project contributors. + The version control history for this file may have further details. + """ + __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" +@@ -318,7 +318,7 @@ class crm_common(ClusterManager): + + stonith_ignore = [ + r"Updating failcount for child_DoFencing", +- r"(ERROR|error).*: Sign-in failed: triggered a retry", ++ r"error.*: Fencer connection failed \(will retry\)", + "pacemaker-execd.*(ERROR|error): stonithd_receive_ops_result failed.", + ] + +diff --git a/cts/patterns.py b/cts/patterns.py +index 1bdace0..1b86ee7 100644 +--- a/cts/patterns.py ++++ b/cts/patterns.py +@@ -308,7 +308,7 @@ class crm_corosync(BasePatterns): + self.components["pacemaker-fenced-ignore"] = [ + r"error:.*Connection to (fencer|stonith-ng).* (closed|failed|lost)", + r"crit:.*Fencing daemon connection failed", +- r"error:.*Sign-in failed: triggered a retry", ++ r"error:.*Fencer connection failed \(will retry\)", + r"Connection to (fencer|stonith-ng) failed, finalizing .* pending operations", + r"pacemaker-controld.*:\s+Result of .* operation for Fencing.*Error", + ] +-- +1.8.3.1 + diff --git a/SOURCES/009-improve-pacemaker_remote-handling.patch b/SOURCES/009-improve-pacemaker_remote-handling.patch new file mode 100644 index 0000000..0a31c27 --- /dev/null +++ b/SOURCES/009-improve-pacemaker_remote-handling.patch @@ -0,0 +1,1466 @@ +From 28566d6832274c59f27bb7b2f1f54420a3f3d822 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 9 May 2019 20:26:08 -0500 +Subject: [PATCH 01/13] Refactor: libpe_status: functionize unfencing digest + code more + +... for readability, reusability, and avoiding unnecessary function calls or +memory allocation. +--- + lib/pengine/utils.c | 159 ++++++++++++++++++++++++++++++++++++++-------------- + 1 file changed, 118 insertions(+), 41 deletions(-) + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 2f4dc1e..f80f8d4 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -2080,57 +2080,134 @@ rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, + return data; + } + ++/*! ++ * \internal ++ * \brief Create an unfencing summary for use in special node attribute ++ * ++ * Create a string combining a fence device's resource ID, agent type, and ++ * parameter digest (whether for all parameters or just non-private parameters). ++ * This can be stored in a special node attribute, allowing us to detect changes ++ * in either the agent type or parameters, to know whether unfencing must be ++ * redone or can be safely skipped when the device's history is cleaned. ++ * ++ * \param[in] rsc_id Fence device resource ID ++ * \param[in] agent_type Fence device agent ++ * \param[in] param_digest Fence device parameter digest ++ * ++ * \return Newly allocated string with unfencing digest ++ * \note The caller is responsible for freeing the result. ++ */ ++static inline char * ++create_unfencing_summary(const char *rsc_id, const char *agent_type, ++ const char *param_digest) ++{ ++ return crm_strdup_printf("%s:%s:%s", rsc_id, agent_type, param_digest); ++} ++ ++/*! ++ * \internal ++ * \brief Check whether a node can skip unfencing ++ * ++ * Check whether a fence device's current definition matches a node's ++ * stored summary of when it was last unfenced by the device. ++ * ++ * \param[in] rsc_id Fence device's resource ID ++ * \param[in] agent Fence device's agent type ++ * \param[in] digest_calc Fence device's current parameter digest ++ * \param[in] node_summary Value of node's special unfencing node attribute ++ * (a comma-separated list of unfencing summaries for ++ * all devices that have unfenced this node) ++ * ++ * \return TRUE if digest matches, FALSE otherwise ++ */ ++static bool ++unfencing_digest_matches(const char *rsc_id, const char *agent, ++ const char *digest_calc, const char *node_summary) ++{ ++ bool matches = FALSE; ++ ++ if (rsc_id && agent && digest_calc && node_summary) { ++ char *search_secure = create_unfencing_summary(rsc_id, agent, ++ digest_calc); ++ ++ /* The digest was calculated including the device ID and agent, ++ * so there is no risk of collision using strstr(). ++ */ ++ matches = (strstr(node_summary, search_secure) != NULL); ++ crm_trace("Calculated unfencing digest '%s' %sfound in '%s'", ++ search_secure, matches? "" : "not ", node_summary); ++ free(search_secure); ++ } ++ return matches; ++} ++ ++/* Magic string to use as action name for digest cache entries used for ++ * unfencing checks. This is not a real action name (i.e. "on"), so ++ * check_action_definition() won't confuse these entries with real actions. ++ */ + #define STONITH_DIGEST_TASK "stonith-on" + ++/*! ++ * \internal ++ * \brief Calculate fence device digests and digest comparison result ++ * ++ * \param[in] rsc Fence device resource ++ * \param[in] agent Fence device's agent type ++ * \param[in] node Node with digest cache to use ++ * \param[in] data_set Cluster working set ++ * ++ * \return Node's digest cache entry ++ */ + static op_digest_cache_t * +-fencing_action_digest_cmp(resource_t * rsc, node_t * node, pe_working_set_t * data_set) ++fencing_action_digest_cmp(pe_resource_t *rsc, const char *agent, ++ pe_node_t *node, pe_working_set_t *data_set) + { +- char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); +- op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, node, NULL, data_set); ++ const char *node_summary = NULL; + +- const char *digest_all = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL); +- const char *digest_secure = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE); ++ // Calculate device's current parameter digests ++ char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); ++ op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, ++ node, NULL, data_set); + +- /* No 'reloads' for fencing device changes +- * +- * We use the resource id + agent + digest so that we can detect +- * changes to the agent and/or the parameters used +- */ +- char *search_all = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_all_calc); +- char *search_secure = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_secure_calc); ++ free(key); + +- data->rc = RSC_DIGEST_ALL; +- if (digest_all == NULL) { +- /* it is unknown what the previous op digest was */ ++ // Check whether node has special unfencing summary node attribute ++ node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL); ++ if (node_summary == NULL) { + data->rc = RSC_DIGEST_UNKNOWN; ++ return data; ++ } + +- } else if (strstr(digest_all, search_all)) { ++ // Check whether full parameter digest matches ++ if (unfencing_digest_matches(rsc->id, agent, data->digest_all_calc, ++ node_summary)) { + data->rc = RSC_DIGEST_MATCH; ++ return data; ++ } + +- } else if(digest_secure && data->digest_secure_calc) { +- if(strstr(digest_secure, search_secure)) { +- if (is_set(data_set->flags, pe_flag_stdout)) { +- printf("Only 'private' parameters to %s for unfencing %s changed\n", +- rsc->id, node->details->uname); +- } +- data->rc = RSC_DIGEST_MATCH; ++ // Check whether secure parameter digest matches ++ node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE); ++ if (unfencing_digest_matches(rsc->id, agent, data->digest_secure_calc, ++ node_summary)) { ++ data->rc = RSC_DIGEST_MATCH; ++ if (is_set(data_set->flags, pe_flag_stdout)) { ++ printf("Only 'private' parameters to %s for unfencing %s changed\n", ++ rsc->id, node->details->uname); + } ++ return data; + } + +- if (is_set(data_set->flags, pe_flag_sanitized) +- && is_set(data_set->flags, pe_flag_stdout) +- && (data->rc == RSC_DIGEST_ALL) ++ // Parameters don't match ++ data->rc = RSC_DIGEST_ALL; ++ if (is_set(data_set->flags, (pe_flag_sanitized|pe_flag_stdout)) + && data->digest_secure_calc) { +- printf("Parameters to %s for unfencing %s changed, try '%s:%s:%s'\n", +- rsc->id, node->details->uname, rsc->id, +- (const char *) g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), +- data->digest_secure_calc); +- } +- +- free(key); +- free(search_all); +- free(search_secure); ++ char *digest = create_unfencing_summary(rsc->id, agent, ++ data->digest_secure_calc); + ++ printf("Parameters to %s for unfencing %s changed, try '%s'\n", ++ rsc->id, node->details->uname, digest); ++ free(digest); ++ } + return data; + } + +@@ -2218,9 +2295,6 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe + * + * We may do this for all nodes in the future, but for now + * the check_action_definition() based stuff works fine. +- * +- * Use "stonith-on" to avoid creating cache entries for +- * operations check_action_definition() would look for. + */ + long max = 1024; + long digests_all_offset = 0; +@@ -2232,8 +2306,11 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe + + for (GListPtr gIter = matches; gIter != NULL; gIter = gIter->next) { + resource_t *match = gIter->data; +- op_digest_cache_t *data = fencing_action_digest_cmp(match, node, data_set); ++ const char *agent = g_hash_table_lookup(match->meta, ++ XML_ATTR_TYPE); ++ op_digest_cache_t *data = NULL; + ++ data = fencing_action_digest_cmp(match, agent, node, data_set); + if(data->rc == RSC_DIGEST_ALL) { + optional = FALSE; + crm_notice("Unfencing %s (remote): because the definition of %s changed", node->details->uname, match->id); +@@ -2244,11 +2321,11 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe + + digests_all_offset += snprintf( + digests_all+digests_all_offset, max-digests_all_offset, +- "%s:%s:%s,", match->id, (const char*)g_hash_table_lookup(match->meta, XML_ATTR_TYPE), data->digest_all_calc); ++ "%s:%s:%s,", match->id, agent, data->digest_all_calc); + + digests_secure_offset += snprintf( + digests_secure+digests_secure_offset, max-digests_secure_offset, +- "%s:%s:%s,", match->id, (const char*)g_hash_table_lookup(match->meta, XML_ATTR_TYPE), data->digest_secure_calc); ++ "%s:%s:%s,", match->id, agent, data->digest_secure_calc); + } + g_hash_table_insert(stonith_op->meta, + strdup(XML_OP_ATTR_DIGESTS_ALL), +-- +1.8.3.1 + + +From fd6e06ff419c95f4423202163d2d4dca3f03a4c5 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 10 May 2019 11:57:31 -0500 +Subject: [PATCH 02/13] Fix: libpe_status: calculate secure digests for + unfencing ops + +The calculation of digests for detection of when unfencing is needed reused +rsc_action_digest(). However that would only add secure digests when the +pe_flag_sanitized flag was set, which is only set by crm_simulate, so secure +digests would never be added in normal cluster operation. This led to +node attributes like name="#digests-secure" +value="stonith-fence_compute-fence-nova:fence_compute:(null),". + +Now, rsc_action_digest() takes a new argument to select whether secure digests +are added, which is always set to TRUE when calculating unfencing digests. +--- + lib/pengine/utils.c | 27 ++++++++++++++++++++++----- + 1 file changed, 22 insertions(+), 5 deletions(-) + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index f80f8d4..5b893f7 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -1936,9 +1936,24 @@ append_versioned_params(xmlNode *versioned_params, const char *ra_version, xmlNo + } + #endif + ++/*! ++ * \internal ++ * \brief Calculate action digests and store in node's digest cache ++ * ++ * \param[in] rsc Resource that action was for ++ * \param[in] task Name of action performed ++ * \param[in] key Action's task key ++ * \param[in] node Node action was performed on ++ * \param[in] xml_op XML of operation in CIB status (if available) ++ * \param[in] calc_secure Whether to calculate secure digest ++ * \param[in] data_set Cluster working set ++ * ++ * \return Pointer to node's digest cache entry ++ */ + static op_digest_cache_t * +-rsc_action_digest(resource_t * rsc, const char *task, const char *key, +- node_t * node, xmlNode * xml_op, pe_working_set_t * data_set) ++rsc_action_digest(pe_resource_t *rsc, const char *task, const char *key, ++ pe_node_t *node, xmlNode *xml_op, bool calc_secure, ++ pe_working_set_t *data_set) + { + op_digest_cache_t *data = NULL; + +@@ -2007,7 +2022,7 @@ rsc_action_digest(resource_t * rsc, const char *task, const char *key, + + data->digest_all_calc = calculate_operation_digest(data->params_all, op_version); + +- if (is_set(data_set->flags, pe_flag_sanitized)) { ++ if (calc_secure) { + data->params_secure = copy_xml(data->params_all); + if(secure_list) { + filter_parameters(data->params_secure, secure_list, FALSE); +@@ -2053,7 +2068,9 @@ rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, + + interval_ms = crm_parse_ms(interval_ms_s); + key = generate_op_key(rsc->id, task, interval_ms); +- data = rsc_action_digest(rsc, task, key, node, xml_op, data_set); ++ data = rsc_action_digest(rsc, task, key, node, xml_op, ++ is_set(data_set->flags, pe_flag_sanitized), ++ data_set); + + data->rc = RSC_DIGEST_MATCH; + if (digest_restart && data->digest_restart_calc && strcmp(data->digest_restart_calc, digest_restart) != 0) { +@@ -2167,7 +2184,7 @@ fencing_action_digest_cmp(pe_resource_t *rsc, const char *agent, + // Calculate device's current parameter digests + char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); + op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, +- node, NULL, data_set); ++ node, NULL, TRUE, data_set); + + free(key); + +-- +1.8.3.1 + + +From 7886c8ec4dd209078cdc76274ed9d2804ea09b6a Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 5 Jun 2019 12:54:34 -0500 +Subject: [PATCH 03/13] Refactor: controller: pass desired op status when + synthesizing failure + +so we can use new status codes later +--- + daemons/controld/controld_execd.c | 27 +++++++++++++++++++++------ + 1 file changed, 21 insertions(+), 6 deletions(-) + +diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c +index 8e89216..fed9419 100644 +--- a/daemons/controld/controld_execd.c ++++ b/daemons/controld/controld_execd.c +@@ -1424,8 +1424,22 @@ force_reprobe(lrm_state_t *lrm_state, const char *from_sys, + update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node); + } + ++/*! ++ * \internal ++ * \brief Fail a requested action without actually executing it ++ * ++ * For an action that can't be executed, process it similarly to an actual ++ * execution result, with specified error status (except for notify actions, ++ * which will always be treated as successful). ++ * ++ * \param[in] lrm_state Executor connection that action is for ++ * \param[in] action Action XML from request ++ * \param[in] rc Desired return code to use ++ * \param[in] op_status Desired operation status to use ++ */ + static void +-synthesize_lrmd_failure(lrm_state_t *lrm_state, xmlNode *action, int rc) ++synthesize_lrmd_failure(lrm_state_t *lrm_state, xmlNode *action, ++ int op_status, enum ocf_exitcode rc) + { + lrmd_event_data_t *op = NULL; + const char *operation = crm_element_value(action, XML_LRM_ATTR_TASK); +@@ -1451,7 +1465,7 @@ synthesize_lrmd_failure(lrm_state_t *lrm_state, xmlNode *action, int rc) + if (safe_str_eq(operation, RSC_NOTIFY)) { // Notifications can't fail + fake_op_status(lrm_state, op, PCMK_LRM_OP_DONE, PCMK_OCF_OK); + } else { +- fake_op_status(lrm_state, op, PCMK_LRM_OP_ERROR, rc); ++ fake_op_status(lrm_state, op, op_status, rc); + } + + crm_info("Faking " CRM_OP_FMT " result (%d) on %s", +@@ -1744,7 +1758,8 @@ do_lrm_invoke(long long action, + if ((lrm_state == NULL) && is_remote_node) { + crm_err("Failing action because local node has never had connection to remote node %s", + target_node); +- synthesize_lrmd_failure(NULL, input->xml, PCMK_OCF_CONNECTION_DIED); ++ synthesize_lrmd_failure(NULL, input->xml, PCMK_LRM_OP_ERROR, ++ PCMK_OCF_CONNECTION_DIED); + return; + } + CRM_ASSERT(lrm_state != NULL); +@@ -1800,7 +1815,7 @@ do_lrm_invoke(long long action, + + rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc); + if (rc == -ENOTCONN) { +- synthesize_lrmd_failure(lrm_state, input->xml, ++ synthesize_lrmd_failure(lrm_state, input->xml, PCMK_LRM_OP_ERROR, + PCMK_OCF_CONNECTION_DIED); + return; + +@@ -1822,7 +1837,7 @@ do_lrm_invoke(long long action, + // Resource operation on malformed resource + crm_err("Invalid resource definition for %s", ID(xml_rsc)); + crm_log_xml_warn(input->msg, "invalid resource"); +- synthesize_lrmd_failure(lrm_state, input->xml, ++ synthesize_lrmd_failure(lrm_state, input->xml, PCMK_LRM_OP_ERROR, + PCMK_OCF_NOT_CONFIGURED); // fatal error + return; + +@@ -1832,7 +1847,7 @@ do_lrm_invoke(long long action, + CRM_XS " rc=%d", + ID(xml_rsc), pcmk_strerror(rc), rc); + crm_log_xml_warn(input->msg, "failed registration"); +- synthesize_lrmd_failure(lrm_state, input->xml, ++ synthesize_lrmd_failure(lrm_state, input->xml, PCMK_LRM_OP_ERROR, + PCMK_OCF_INVALID_PARAM); // hard error + return; + } +-- +1.8.3.1 + + +From ddc3942d7131db9c9874031ca4b3b4a531221573 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 5 Jun 2019 13:08:15 -0500 +Subject: [PATCH 04/13] Fix: controller: use op status, not rc, for executor + disconnection + +Previously, if an action were requested for an executor (local or remote) that +the controller does not have a connection to, the action's rc would be set to +PCMK_OCF_CONNECTION_DIED and its op status to PCMK_LRM_OP_ERROR. + +This was undesirable for a couple reasons: PCMK_OCF_CONNECTION_DIED is a +nonstandard extension to the OCF return codes, which can confuse users +trying to look up the meaning or interpret cluster status output; and it really +is an operation execution status and not an operation result. + +This changes the result to PCMK_OCF_UNKNOWN_ERROR with a new op status +PCMK_LRM_OP_NOT_CONNECTED. The new codes are mapped to the old ones for older +DCs that don't understand them. +--- + cts/CTStests.py | 2 +- + daemons/controld/controld_execd.c | 21 +++++++++++++++++---- + daemons/controld/controld_execd_state.c | 6 ++++-- + include/crm/services.h | 4 +++- + lib/common/operations.c | 1 + + lib/pengine/unpack.c | 3 ++- + 6 files changed, 28 insertions(+), 9 deletions(-) + +diff --git a/cts/CTStests.py b/cts/CTStests.py +index 32945cb..be7fd7f 100644 +--- a/cts/CTStests.py ++++ b/cts/CTStests.py +@@ -3068,7 +3068,7 @@ class RemoteStonithd(RemoteDriver): + r"schedulerd.*:\s+Recover remote-.*\s*\(.*\)", + r"Calculated [Tt]ransition .*pe-error", + r"error.*: Resource .*ocf::.* is active on 2 nodes attempting recovery", +- r"error: Result of monitor operation for .* on remote-.*: Error", ++ r"error: Result of monitor operation for .* on remote-.*: No executor connection", + ] + + ignore_pats.extend(RemoteDriver.errorstoignore(self)) +diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c +index fed9419..ac215b6 100644 +--- a/daemons/controld/controld_execd.c ++++ b/daemons/controld/controld_execd.c +@@ -1758,8 +1758,8 @@ do_lrm_invoke(long long action, + if ((lrm_state == NULL) && is_remote_node) { + crm_err("Failing action because local node has never had connection to remote node %s", + target_node); +- synthesize_lrmd_failure(NULL, input->xml, PCMK_LRM_OP_ERROR, +- PCMK_OCF_CONNECTION_DIED); ++ synthesize_lrmd_failure(NULL, input->xml, PCMK_LRM_OP_NOT_CONNECTED, ++ PCMK_OCF_UNKNOWN_ERROR); + return; + } + CRM_ASSERT(lrm_state != NULL); +@@ -1815,8 +1815,9 @@ do_lrm_invoke(long long action, + + rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc); + if (rc == -ENOTCONN) { +- synthesize_lrmd_failure(lrm_state, input->xml, PCMK_LRM_OP_ERROR, +- PCMK_OCF_CONNECTION_DIED); ++ synthesize_lrmd_failure(lrm_state, input->xml, ++ PCMK_LRM_OP_NOT_CONNECTED, ++ PCMK_OCF_UNKNOWN_ERROR); + return; + + } else if ((rc < 0) && !create_rsc) { +@@ -2532,6 +2533,18 @@ process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, + CRM_CHECK(op != NULL, return); + CRM_CHECK(op->rsc_id != NULL, return); + ++ // Remap new status codes for older DCs ++ if (compare_version(fsa_our_dc_version, "3.2.0") < 0) { ++ switch (op->op_status) { ++ case PCMK_LRM_OP_NOT_CONNECTED: ++ op->op_status = PCMK_LRM_OP_ERROR; ++ op->rc = PCMK_OCF_CONNECTION_DIED; ++ break; ++ default: ++ break; ++ } ++ } ++ + op_id = make_stop_id(op->rsc_id, op->call_id); + op_key = generate_op_key(op->rsc_id, op->op_type, op->interval_ms); + +diff --git a/daemons/controld/controld_execd_state.c b/daemons/controld/controld_execd_state.c +index 4e9f096..63e6b33 100644 +--- a/daemons/controld/controld_execd_state.c ++++ b/daemons/controld/controld_execd_state.c +@@ -1,6 +1,8 @@ + /* + * Copyright 2012-2019 the Pacemaker project contributors + * ++ * The version control history for this file may have further details. ++ * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. + */ +@@ -76,8 +78,8 @@ fail_pending_op(gpointer key, gpointer value, gpointer user_data) + event.user_data = op->user_data; + event.timeout = 0; + event.interval_ms = op->interval_ms; +- event.rc = PCMK_OCF_CONNECTION_DIED; +- event.op_status = PCMK_LRM_OP_ERROR; ++ event.rc = PCMK_OCF_UNKNOWN_ERROR; ++ event.op_status = PCMK_LRM_OP_NOT_CONNECTED; + event.t_run = op->start_time; + event.t_rcchange = op->start_time; + +diff --git a/include/crm/services.h b/include/crm/services.h +index 4bdd21a..ca9470b 100644 +--- a/include/crm/services.h ++++ b/include/crm/services.h +@@ -100,7 +100,7 @@ enum ocf_exitcode { + + + /* 150-199 reserved for application use */ +- PCMK_OCF_CONNECTION_DIED = 189, /* Operation failure implied by disconnection of the LRM API to a local or remote node */ ++ PCMK_OCF_CONNECTION_DIED = 189, // Deprecated (see PCMK_LRM_OP_NOT_CONNECTED) + + PCMK_OCF_DEGRADED = 190, /* Active resource that is no longer 100% functional */ + PCMK_OCF_DEGRADED_MASTER = 191, /* Promoted resource that is no longer 100% functional */ +@@ -126,6 +126,7 @@ enum op_status { + PCMK_LRM_OP_ERROR_HARD, + PCMK_LRM_OP_ERROR_FATAL, + PCMK_LRM_OP_NOT_INSTALLED, ++ PCMK_LRM_OP_NOT_CONNECTED, + }; + + enum nagios_exitcode { +@@ -337,6 +338,7 @@ gboolean services_alert_async(svc_action_t *action, + case PCMK_LRM_OP_NOTSUPPORTED:return "NOT SUPPORTED"; + case PCMK_LRM_OP_ERROR:return "Error"; + case PCMK_LRM_OP_NOT_INSTALLED:return "Not installed"; ++ case PCMK_LRM_OP_NOT_CONNECTED:return "No executor connection"; + default:return "UNKNOWN!"; + } + } +diff --git a/lib/common/operations.c b/lib/common/operations.c +index 2144cc6..c6b16cb 100644 +--- a/lib/common/operations.c ++++ b/lib/common/operations.c +@@ -395,6 +395,7 @@ did_rsc_op_fail(lrmd_event_data_t * op, int target_rc) + case PCMK_LRM_OP_NOTSUPPORTED: + case PCMK_LRM_OP_TIMEOUT: + case PCMK_LRM_OP_ERROR: ++ case PCMK_LRM_OP_NOT_CONNECTED: + return TRUE; + break; + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 0e8177b..671f0c4 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -3163,7 +3163,7 @@ unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, xmlNode ** last + crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms); + + CRM_CHECK(task != NULL, return FALSE); +- CRM_CHECK(status <= PCMK_LRM_OP_NOT_INSTALLED, return FALSE); ++ CRM_CHECK(status <= PCMK_LRM_OP_NOT_CONNECTED, return FALSE); + CRM_CHECK(status >= PCMK_LRM_OP_PENDING, return FALSE); + + if (safe_str_eq(task, CRMD_ACTION_NOTIFY) || +@@ -3304,6 +3304,7 @@ unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, xmlNode ** last + case PCMK_LRM_OP_ERROR_FATAL: + case PCMK_LRM_OP_TIMEOUT: + case PCMK_LRM_OP_NOTSUPPORTED: ++ case PCMK_LRM_OP_NOT_CONNECTED: + + failure_strategy = get_action_on_fail(rsc, task_key, task, data_set); + if ((failure_strategy == action_fail_ignore) +-- +1.8.3.1 + + +From fc135cb441fb7c66a44fbffe74dcae26c112be3f Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 5 Jun 2019 13:43:08 -0500 +Subject: [PATCH 05/13] Fix: controller: use op status, not rc, for execution + in invalid state + +Previously, if an action were requested while the controller cannot execute actions +(i.e. shutdown), the action's rc would be set to CRM_DIRECT_NACK_RC and its op +status to PCMK_LRM_OP_ERROR. + +This was undesirable for a couple reasons: rc should only be OCF return codes, +and it really is an operation execution status and not an operation result. + +This changes the result to PCMK_OCF_UNKNOWN_ERROR with a new op status +PCMK_LRM_OP_INVALID. The new codes are mapped to the old ones for older +DCs that don't understand them. +--- + daemons/controld/controld_execd.c | 8 ++++++-- + daemons/controld/controld_fsa.h | 6 +----- + daemons/controld/controld_te_events.c | 13 ++++++------- + include/crm/services.h | 2 ++ + lib/common/operations.c | 1 + + lib/pengine/unpack.c | 3 ++- + 6 files changed, 18 insertions(+), 15 deletions(-) + +diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c +index ac215b6..a20f96a 100644 +--- a/daemons/controld/controld_execd.c ++++ b/daemons/controld/controld_execd.c +@@ -2254,8 +2254,8 @@ do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *operat + operation, rsc->id, fsa_state2string(fsa_state), + is_set(fsa_input_register, R_SHUTDOWN)?"true":"false"); + +- op->rc = CRM_DIRECT_NACK_RC; +- op->op_status = PCMK_LRM_OP_ERROR; ++ op->rc = PCMK_OCF_UNKNOWN_ERROR; ++ op->op_status = PCMK_LRM_OP_INVALID; + send_direct_ack(NULL, NULL, rsc, op, rsc->id); + lrmd_free_event(op); + free(op_id); +@@ -2540,6 +2540,10 @@ process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, + op->op_status = PCMK_LRM_OP_ERROR; + op->rc = PCMK_OCF_CONNECTION_DIED; + break; ++ case PCMK_LRM_OP_INVALID: ++ op->op_status = PCMK_LRM_OP_ERROR; ++ op->rc = CRM_DIRECT_NACK_RC; ++ break; + default: + break; + } +diff --git a/daemons/controld/controld_fsa.h b/daemons/controld/controld_fsa.h +index 397a9cd..7527ed9 100644 +--- a/daemons/controld/controld_fsa.h ++++ b/daemons/controld/controld_fsa.h +@@ -426,11 +426,7 @@ enum crmd_fsa_input { + + # define R_IN_RECOVERY 0x80000000ULL + +-/* +- * Magic RC used within the controller to indicate direct nacks +- * (operation is invalid in current state) +- */ +-#define CRM_DIRECT_NACK_RC (99) ++#define CRM_DIRECT_NACK_RC (99) // Deprecated (see PCMK_LRM_OP_INVALID) + + enum crmd_fsa_cause { + C_UNKNOWN = 0, +diff --git a/daemons/controld/controld_te_events.c b/daemons/controld/controld_te_events.c +index b7b48a4..d297241 100644 +--- a/daemons/controld/controld_te_events.c ++++ b/daemons/controld/controld_te_events.c +@@ -123,10 +123,8 @@ update_failcount(xmlNode * event, const char *event_node_uuid, int rc, + const char *on_uname = crm_peer_uname(event_node_uuid); + const char *origin = crm_element_value(event, XML_ATTR_ORIGIN); + +- /* Nothing needs to be done for success, lrm status refresh, +- * or direct nack (internal code for "busy, try again") +- */ +- if ((rc == CRM_DIRECT_NACK_RC) || (rc == target_rc)) { ++ // Nothing needs to be done for success or status refresh ++ if (rc == target_rc) { + return FALSE; + } else if (safe_str_eq(origin, "build_active_RAs")) { + crm_debug("No update for %s (rc=%d) on %s: Old failure from lrm status refresh", +@@ -225,7 +223,7 @@ status_from_rc(crm_action_t * action, int orig_status, int rc, int target_rc) + return PCMK_LRM_OP_DONE; + } + +- if (rc != CRM_DIRECT_NACK_RC) { ++ if (orig_status != PCMK_LRM_OP_INVALID) { + const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY); + const char *uname = crm_element_value(action->xml, XML_LRM_ATTR_TARGET); + +@@ -541,8 +539,9 @@ process_graph_event(xmlNode *event, const char *event_node) + if (action && (rc == target_rc)) { + crm_trace("Processed update to %s: %s", id, magic); + } else { +- if (update_failcount(event, event_node, rc, target_rc, +- (transition_num == -1), ignore_failures)) { ++ if ((status != PCMK_LRM_OP_INVALID) ++ && update_failcount(event, event_node, rc, target_rc, ++ (transition_num == -1), ignore_failures)) { + desc = "failed"; + } + crm_info("Detected action (%d.%d) %s.%d=%s: %s", transition_num, +diff --git a/include/crm/services.h b/include/crm/services.h +index ca9470b..0771241 100644 +--- a/include/crm/services.h ++++ b/include/crm/services.h +@@ -127,6 +127,7 @@ enum op_status { + PCMK_LRM_OP_ERROR_FATAL, + PCMK_LRM_OP_NOT_INSTALLED, + PCMK_LRM_OP_NOT_CONNECTED, ++ PCMK_LRM_OP_INVALID, + }; + + enum nagios_exitcode { +@@ -339,6 +340,7 @@ gboolean services_alert_async(svc_action_t *action, + case PCMK_LRM_OP_ERROR:return "Error"; + case PCMK_LRM_OP_NOT_INSTALLED:return "Not installed"; + case PCMK_LRM_OP_NOT_CONNECTED:return "No executor connection"; ++ case PCMK_LRM_OP_INVALID:return "Cannot execute now"; + default:return "UNKNOWN!"; + } + } +diff --git a/lib/common/operations.c b/lib/common/operations.c +index c6b16cb..480bddc 100644 +--- a/lib/common/operations.c ++++ b/lib/common/operations.c +@@ -396,6 +396,7 @@ did_rsc_op_fail(lrmd_event_data_t * op, int target_rc) + case PCMK_LRM_OP_TIMEOUT: + case PCMK_LRM_OP_ERROR: + case PCMK_LRM_OP_NOT_CONNECTED: ++ case PCMK_LRM_OP_INVALID: + return TRUE; + break; + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 671f0c4..fb1ab60 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -3163,7 +3163,7 @@ unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, xmlNode ** last + crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms); + + CRM_CHECK(task != NULL, return FALSE); +- CRM_CHECK(status <= PCMK_LRM_OP_NOT_CONNECTED, return FALSE); ++ CRM_CHECK(status <= PCMK_LRM_OP_INVALID, return FALSE); + CRM_CHECK(status >= PCMK_LRM_OP_PENDING, return FALSE); + + if (safe_str_eq(task, CRMD_ACTION_NOTIFY) || +@@ -3305,6 +3305,7 @@ unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, xmlNode ** last + case PCMK_LRM_OP_TIMEOUT: + case PCMK_LRM_OP_NOTSUPPORTED: + case PCMK_LRM_OP_NOT_CONNECTED: ++ case PCMK_LRM_OP_INVALID: + + failure_strategy = get_action_on_fail(rsc, task_key, task, data_set); + if ((failure_strategy == action_fail_ignore) +-- +1.8.3.1 + + +From f5ea526b211e95ece16acb0f72bfbbbda60ec437 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 12 Jun 2019 20:48:59 -0500 +Subject: [PATCH 06/13] Doc: libcrmcommon: document CRM_FEATURE_SET in API docs + +--- + include/crm/crm.h | 23 ++++++++++++++++++++++- + 1 file changed, 22 insertions(+), 1 deletion(-) + +diff --git a/include/crm/crm.h b/include/crm/crm.h +index 5f323e8..56a2048 100644 +--- a/include/crm/crm.h ++++ b/include/crm/crm.h +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2018 the Pacemaker project contributors ++ * Copyright 2004-2019 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -29,6 +29,27 @@ extern "C" { + + # include + ++/*! ++ * The CRM feature set assists with compatibility in mixed-version clusters. ++ * The major version number increases when nodes with different versions ++ * would not work (rolling upgrades are not allowed). The minor version ++ * number increases when mixed-version clusters are allowed only during ++ * rolling upgrades (a node with the oldest feature set will be elected DC). The ++ * minor-minor version number is ignored, but allows resource agents to detect ++ * cluster support for various features. ++ * ++ * The feature set also affects the processing of old saved CIBs (such as for ++ * many scheduler regression tests). ++ * ++ * Particular feature points currently used by pacemaker: ++ * ++ * >2.1: Operation updates include timing data ++ * >=3.0.5: XML v2 digests are created ++ * >=3.0.8: Peers do not need acks for cancellations ++ * >=3.0.9: DC will send its own shutdown request to all peers ++ * XML v2 patchsets are created by default ++ * >=3.0.13: Fail counts include operation name and interval ++ */ + # define CRM_FEATURE_SET "3.1.0" + + # define EOS '\0' +-- +1.8.3.1 + + +From 1ff54a448b1178a34f2dd4f615221087e08468de Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 12 Jun 2019 20:51:21 -0500 +Subject: [PATCH 07/13] Feature: libcrmcommon: bump CRM feature set + +... for the new LRM op status codes +--- + include/crm/crm.h | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/include/crm/crm.h b/include/crm/crm.h +index 56a2048..cbf72d3 100644 +--- a/include/crm/crm.h ++++ b/include/crm/crm.h +@@ -49,8 +49,9 @@ extern "C" { + * >=3.0.9: DC will send its own shutdown request to all peers + * XML v2 patchsets are created by default + * >=3.0.13: Fail counts include operation name and interval ++ * >=3.2.0: DC supports PCMK_LRM_OP_INVALID and PCMK_LRM_OP_NOT_CONNECTED + */ +-# define CRM_FEATURE_SET "3.1.0" ++# define CRM_FEATURE_SET "3.2.0" + + # define EOS '\0' + # define DIMOF(a) ((int) (sizeof(a)/sizeof(a[0])) ) +-- +1.8.3.1 + + +From efc639cc835fba27fa5af4a0539e995d95660520 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 5 Jun 2019 15:12:20 -0500 +Subject: [PATCH 08/13] Low: libpe_status: fail connection resource if remote + action gets "not connected" + +--- + lib/pengine/unpack.c | 15 ++++++++++++++- + 1 file changed, 14 insertions(+), 1 deletion(-) + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index fb1ab60..081df07 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -3299,12 +3299,25 @@ unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, xmlNode ** last + unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set); + break; + ++ case PCMK_LRM_OP_NOT_CONNECTED: ++ if (pe__is_guest_or_remote_node(node) ++ && is_set(node->details->remote_rsc->flags, pe_rsc_managed)) { ++ /* We should never get into a situation where a managed remote ++ * connection resource is considered OK but a resource action ++ * behind the connection gets a "not connected" status. But as a ++ * fail-safe in case a bug or unusual circumstances do lead to ++ * that, ensure the remote connection is considered failed. ++ */ ++ set_bit(node->details->remote_rsc->flags, pe_rsc_failed); ++ } ++ ++ // fall through ++ + case PCMK_LRM_OP_ERROR: + case PCMK_LRM_OP_ERROR_HARD: + case PCMK_LRM_OP_ERROR_FATAL: + case PCMK_LRM_OP_TIMEOUT: + case PCMK_LRM_OP_NOTSUPPORTED: +- case PCMK_LRM_OP_NOT_CONNECTED: + case PCMK_LRM_OP_INVALID: + + failure_strategy = get_action_on_fail(rsc, task_key, task, data_set); +-- +1.8.3.1 + + +From dad337a96dfeca4dbde7bbd97f99f24956440fc2 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Sat, 8 Jun 2019 16:25:04 -0500 +Subject: [PATCH 09/13] Refactor: libpe_status: add function for checking + shutdown attribute + +... to reduce code duplication and allow further reuse +--- + include/crm/pengine/internal.h | 2 ++ + lib/pengine/unpack.c | 8 ++------ + lib/pengine/utils.c | 20 ++++++++++++++++++++ + 3 files changed, 24 insertions(+), 6 deletions(-) + +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index fd55bb9..a2a3d52 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -359,4 +359,6 @@ void pe__foreach_param_check(pe_working_set_t *data_set, + enum pe_check_parameters, + pe_working_set_t*)); + void pe__free_param_checks(pe_working_set_t *data_set); ++ ++bool pe__shutdown_requested(pe_node_t *node); + #endif +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 081df07..9d13a57 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -909,7 +909,6 @@ unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * + const char *resource_discovery_enabled = NULL; + xmlNode *attrs = NULL; + resource_t *rsc = NULL; +- const char *shutdown = NULL; + + if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) { + return; +@@ -931,8 +930,7 @@ unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * + attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE); + add_node_attrs(attrs, this_node, TRUE, data_set); + +- shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); +- if (shutdown != NULL && safe_str_neq("0", shutdown)) { ++ if (pe__shutdown_requested(this_node)) { + crm_info("Node %s is shutting down", this_node->details->uname); + this_node->details->shutdown = TRUE; + if (rsc) { +@@ -1392,7 +1390,6 @@ gboolean + determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set_t * data_set) + { + gboolean online = FALSE; +- const char *shutdown = NULL; + const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED); + + if (this_node == NULL) { +@@ -1402,9 +1399,8 @@ determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set + + this_node->details->shutdown = FALSE; + this_node->details->expected_up = FALSE; +- shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); + +- if (shutdown != NULL && safe_str_neq("0", shutdown)) { ++ if (pe__shutdown_requested(this_node)) { + this_node->details->shutdown = TRUE; + + } else if (safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) { +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 5b893f7..c5fd0f7 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -2510,3 +2510,23 @@ void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrit + } + } + } ++ ++/*! ++ * \internal ++ * \brief Check whether shutdown has been requested for a node ++ * ++ * \param[in] node Node to check ++ * ++ * \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise ++ * \note This differs from simply using node->details->shutdown in that it can ++ * be used before that has been determined (and in fact to determine it), ++ * and it can also be used to distinguish requested shutdown from implicit ++ * shutdown of remote nodes by virtue of their connection stopping. ++ */ ++bool ++pe__shutdown_requested(pe_node_t *node) ++{ ++ const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN); ++ ++ return shutdown && strcmp(shutdown, "0"); ++} +-- +1.8.3.1 + + +From 1e9903326a59f58d9dd2f2618d709f8aa61e41e9 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 5 Jun 2019 16:37:26 -0500 +Subject: [PATCH 10/13] Fix: scheduler: remote state is failed if node is + shutting down with connection failure + +When determining remote state, if the connection resource is failed and not +being started again, we consider the state to be unknown if the connection has +a reconnect interval, because we won't know whether the connection can be +recovered until the interval expires and we re-attempt connection. + +However, if the node is shutting down at the time, we won't re-attempt +connection, so consider the state failed in that case. (Note that we check the +actual shutdown node attribute, rather than node->details->shutdown, since that +is set for remote nodes whenever the connection is stopping.) + +This avoids a situation where actions that cannot succeed can be scheduled on a +remote node that's shutting down. +--- + lib/pacemaker/pcmk_sched_allocate.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/lib/pacemaker/pcmk_sched_allocate.c b/lib/pacemaker/pcmk_sched_allocate.c +index 3363a72..b7d1b48 100644 +--- a/lib/pacemaker/pcmk_sched_allocate.c ++++ b/lib/pacemaker/pcmk_sched_allocate.c +@@ -1972,7 +1972,8 @@ get_remote_node_state(pe_node_t *node) + + if ((remote_rsc->next_role == RSC_ROLE_STOPPED) + && remote_rsc->remote_reconnect_ms +- && node->details->remote_was_fenced) { ++ && node->details->remote_was_fenced ++ && !pe__shutdown_requested(node)) { + + /* We won't know whether the connection is recoverable until the + * reconnect interval expires and we reattempt connection. +-- +1.8.3.1 + + +From ea70750d04219618b5feeda04443b27616e441a0 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 5 Jun 2019 16:43:19 -0500 +Subject: [PATCH 11/13] Fix: libpe_status: don't order implied stops relative + to a remote connection + +Actions behind a remote connection are ordered relative to any start or stop of +the remote connection. However, if the action is a stop implied due to fencing, +it does not require the remote connection, and the ordering should not be done. + +This avoids a delay in the remote connection recovery if it is failed, e.g. +previously the ordering would look like: + + fence remote node -> implied stop of resource on remote -> stop connection + +Now, the connection stop can proceed simultaneously with the remote node +fencing. +--- + lib/pacemaker/pcmk_sched_allocate.c | 11 +++++------ + 1 file changed, 5 insertions(+), 6 deletions(-) + +diff --git a/lib/pacemaker/pcmk_sched_allocate.c b/lib/pacemaker/pcmk_sched_allocate.c +index b7d1b48..9f82c00 100644 +--- a/lib/pacemaker/pcmk_sched_allocate.c ++++ b/lib/pacemaker/pcmk_sched_allocate.c +@@ -2065,14 +2065,13 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) + pe_order_implies_first, data_set); + + } else if(state == remote_state_failed) { +- /* We would only be here if the resource is +- * running on the remote node. Since we have no +- * way to stop it, it is necessary to fence the +- * node. ++ /* The resource is active on the node, but since we don't have a ++ * valid connection, the only way to stop the resource is by ++ * fencing the node. There is no need to order the stop relative ++ * to the remote connection, since the stop will become implied ++ * by the fencing. + */ + pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable"); +- order_action_then_stop(action, remote_rsc, +- pe_order_implies_first, data_set); + + } else if(remote_rsc->next_role == RSC_ROLE_STOPPED) { + /* State must be remote_state_unknown or remote_state_stopped. +-- +1.8.3.1 + + +From 091c367369b892d26fe0de99d35cf521b6249d10 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Sat, 8 Jun 2019 16:51:20 -0500 +Subject: [PATCH 12/13] Test: cts-scheduler: update regression tests for remote + connection ordering change + +Remote connection stops no longer have to wait for implied stops of resources +behind the connection. + +Unchanged from before, if the remote connection stops are implied themselves, +they can be confirmed immediately without waiting for their host's fencing, +because remote connections have "requires" set to "quorum" rather than +"fencing". +--- + cts/scheduler/order-expired-failure.dot | 1 - + cts/scheduler/order-expired-failure.exp | 6 +----- + cts/scheduler/order-expired-failure.summary | 2 +- + cts/scheduler/remote-connection-unrecoverable.dot | 2 -- + cts/scheduler/remote-connection-unrecoverable.exp | 9 +-------- + cts/scheduler/remote-connection-unrecoverable.summary | 2 +- + cts/scheduler/remote-fence-before-reconnect.dot | 1 - + cts/scheduler/remote-fence-before-reconnect.exp | 6 +----- + cts/scheduler/remote-fence-before-reconnect.summary | 2 +- + cts/scheduler/remote-recover-all.dot | 2 -- + cts/scheduler/remote-recover-all.exp | 12 ++---------- + cts/scheduler/remote-recover-all.summary | 4 ++-- + cts/scheduler/remote-recover-no-resources.dot | 1 - + cts/scheduler/remote-recover-no-resources.exp | 6 +----- + cts/scheduler/remote-recover-no-resources.summary | 2 +- + cts/scheduler/remote-recover-unknown.dot | 1 - + cts/scheduler/remote-recover-unknown.exp | 6 +----- + cts/scheduler/remote-recover-unknown.summary | 2 +- + 18 files changed, 14 insertions(+), 53 deletions(-) + +diff --git a/cts/scheduler/order-expired-failure.dot b/cts/scheduler/order-expired-failure.dot +index 2e9963b..5c21d5d 100644 +--- a/cts/scheduler/order-expired-failure.dot ++++ b/cts/scheduler/order-expired-failure.dot +@@ -4,7 +4,6 @@ digraph "g" { + "compute-unfence-trigger-clone_stop_0" [ style=bold color="green" fontcolor="orange"] + "compute-unfence-trigger-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] + "compute-unfence-trigger_stop_0 overcloud-novacompute-1" -> "compute-unfence-trigger-clone_stopped_0" [ style = bold] +-"compute-unfence-trigger_stop_0 overcloud-novacompute-1" -> "overcloud-novacompute-1_stop_0 controller-1" [ style = bold] + "compute-unfence-trigger_stop_0 overcloud-novacompute-1" [ style=bold color="green" fontcolor="orange"] + "ip-10.0.0.110_monitor_10000 controller-1" [ style=bold color="green" fontcolor="black"] + "ip-10.0.0.110_start_0 controller-1" -> "ip-10.0.0.110_monitor_10000 controller-1" [ style = bold] +diff --git a/cts/scheduler/order-expired-failure.exp b/cts/scheduler/order-expired-failure.exp +index c476bc2..4a50493 100644 +--- a/cts/scheduler/order-expired-failure.exp ++++ b/cts/scheduler/order-expired-failure.exp +@@ -9,11 +9,7 @@ + + + +- +- +- +- +- ++ + + + +diff --git a/cts/scheduler/order-expired-failure.summary b/cts/scheduler/order-expired-failure.summary +index c86bb91..2cf43ed 100644 +--- a/cts/scheduler/order-expired-failure.summary ++++ b/cts/scheduler/order-expired-failure.summary +@@ -52,6 +52,7 @@ Transition Summary: + * Stop compute-unfence-trigger:1 ( overcloud-novacompute-1 ) due to node availability + + Executing cluster transition: ++ * Resource action: overcloud-novacompute-1 stop on controller-1 + * Resource action: stonith-fence_compute-fence-nova stop on controller-2 + * Fencing overcloud-novacompute-1 (reboot) + * Cluster action: clear_failcount for overcloud-novacompute-1 on controller-1 +@@ -62,7 +63,6 @@ Executing cluster transition: + * Resource action: ip-10.0.0.110 monitor=10000 on controller-1 + * Pseudo action: compute-unfence-trigger_stop_0 + * Pseudo action: compute-unfence-trigger-clone_stopped_0 +- * Resource action: overcloud-novacompute-1 stop on controller-1 + Using the original execution date of: 2018-04-09 07:55:35Z + + Revised cluster status: +diff --git a/cts/scheduler/remote-connection-unrecoverable.dot b/cts/scheduler/remote-connection-unrecoverable.dot +index 7728425..1017d2b 100644 +--- a/cts/scheduler/remote-connection-unrecoverable.dot ++++ b/cts/scheduler/remote-connection-unrecoverable.dot +@@ -7,14 +7,12 @@ digraph "g" { + "remote1_stop_0 node1" [ style=bold color="green" fontcolor="orange"] + "rsc1_delete_0 remote1" -> "rsc1_start_0 node2" [ style = dashed] + "rsc1_delete_0 remote1" [ style=dashed color="red" fontcolor="black"] +-"rsc1_monitor_0 node2" -> "remote1_stop_0 node1" [ style = bold] + "rsc1_monitor_0 node2" -> "rsc1_start_0 node2" [ style = bold] + "rsc1_monitor_0 node2" -> "rsc2-master_demote_0" [ style = bold] + "rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black"] + "rsc1_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] + "rsc1_start_0 node2" -> "rsc1_monitor_10000 node2" [ style = bold] + "rsc1_start_0 node2" [ style=bold color="green" fontcolor="black"] +-"rsc1_stop_0 remote1" -> "remote1_stop_0 node1" [ style = bold] + "rsc1_stop_0 remote1" -> "rsc1_delete_0 remote1" [ style = dashed] + "rsc1_stop_0 remote1" -> "rsc1_start_0 node2" [ style = bold] + "rsc1_stop_0 remote1" -> "rsc2-master_demote_0" [ style = bold] +diff --git a/cts/scheduler/remote-connection-unrecoverable.exp b/cts/scheduler/remote-connection-unrecoverable.exp +index 2c9357b..d57c106 100644 +--- a/cts/scheduler/remote-connection-unrecoverable.exp ++++ b/cts/scheduler/remote-connection-unrecoverable.exp +@@ -5,14 +5,7 @@ + + + +- +- +- +- +- +- +- +- ++ + + + +diff --git a/cts/scheduler/remote-connection-unrecoverable.summary b/cts/scheduler/remote-connection-unrecoverable.summary +index 23fa9ca..caff564 100644 +--- a/cts/scheduler/remote-connection-unrecoverable.summary ++++ b/cts/scheduler/remote-connection-unrecoverable.summary +@@ -21,6 +21,7 @@ Transition Summary: + * Stop rsc2:0 ( Master node1 ) due to node availability + + Executing cluster transition: ++ * Pseudo action: remote1_stop_0 + * Resource action: killer stop on node2 + * Resource action: rsc1 monitor on node2 + * Fencing node1 (reboot) +@@ -29,7 +30,6 @@ Executing cluster transition: + * Resource action: killer monitor=60000 on node2 + * Pseudo action: rsc1_stop_0 + * Pseudo action: rsc2-master_demote_0 +- * Pseudo action: remote1_stop_0 + * Resource action: rsc1 start on node2 + * Pseudo action: rsc2_demote_0 + * Pseudo action: rsc2-master_demoted_0 +diff --git a/cts/scheduler/remote-fence-before-reconnect.dot b/cts/scheduler/remote-fence-before-reconnect.dot +index 4ced43e..5812b7f 100644 +--- a/cts/scheduler/remote-fence-before-reconnect.dot ++++ b/cts/scheduler/remote-fence-before-reconnect.dot +@@ -3,7 +3,6 @@ + "fake2_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"] + "fake2_start_0 c7auto1" -> "fake2_monitor_10000 c7auto1" [ style = bold] + "fake2_start_0 c7auto1" [ style=bold color="green" fontcolor="black"] +-"fake2_stop_0 c7auto4" -> "c7auto4_stop_0 c7auto1" [ style = bold] + "fake2_stop_0 c7auto4" -> "fake2_start_0 c7auto1" [ style = bold] + "fake2_stop_0 c7auto4" [ style=bold color="green" fontcolor="orange"] + "stonith 'reboot' c7auto4" -> "fake2_start_0 c7auto1" [ style = bold] +diff --git a/cts/scheduler/remote-fence-before-reconnect.exp b/cts/scheduler/remote-fence-before-reconnect.exp +index f99d9ef..f506f85 100644 +--- a/cts/scheduler/remote-fence-before-reconnect.exp ++++ b/cts/scheduler/remote-fence-before-reconnect.exp +@@ -9,11 +9,7 @@ + + + +- +- +- +- +- ++ + + + +diff --git a/cts/scheduler/remote-fence-before-reconnect.summary b/cts/scheduler/remote-fence-before-reconnect.summary +index f61e18b..03eac20 100644 +--- a/cts/scheduler/remote-fence-before-reconnect.summary ++++ b/cts/scheduler/remote-fence-before-reconnect.summary +@@ -17,9 +17,9 @@ Transition Summary: + * Move fake2 ( c7auto4 -> c7auto1 ) + + Executing cluster transition: ++ * Resource action: c7auto4 stop on c7auto1 + * Fencing c7auto4 (reboot) + * Pseudo action: fake2_stop_0 +- * Resource action: c7auto4 stop on c7auto1 + * Resource action: fake2 start on c7auto1 + * Resource action: fake2 monitor=10000 on c7auto1 + +diff --git a/cts/scheduler/remote-recover-all.dot b/cts/scheduler/remote-recover-all.dot +index deed802..4128b10 100644 +--- a/cts/scheduler/remote-recover-all.dot ++++ b/cts/scheduler/remote-recover-all.dot +@@ -19,7 +19,6 @@ digraph "g" { + "galera_demote_0 galera-2" -> "galera_stop_0 galera-2" [ style = bold] + "galera_demote_0 galera-2" [ style=bold color="green" fontcolor="orange"] + "galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"] +-"galera_stop_0 galera-2" -> "galera-2_stop_0 controller-1" [ style = bold] + "galera_stop_0 galera-2" -> "galera-master_stopped_0" [ style = bold] + "galera_stop_0 galera-2" [ style=bold color="green" fontcolor="orange"] + "haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold] +@@ -60,7 +59,6 @@ digraph "g" { + "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] + "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] + "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] +-"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] + "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] + "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] + "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] +diff --git a/cts/scheduler/remote-recover-all.exp b/cts/scheduler/remote-recover-all.exp +index 8137ffb..0cb51f6 100644 +--- a/cts/scheduler/remote-recover-all.exp ++++ b/cts/scheduler/remote-recover-all.exp +@@ -5,11 +5,7 @@ + + + +- +- +- +- +- ++ + + + +@@ -57,11 +53,7 @@ + + + +- +- +- +- +- ++ + + + +diff --git a/cts/scheduler/remote-recover-all.summary b/cts/scheduler/remote-recover-all.summary +index 2ac0c6a..d095fdd 100644 +--- a/cts/scheduler/remote-recover-all.summary ++++ b/cts/scheduler/remote-recover-all.summary +@@ -56,7 +56,9 @@ Transition Summary: + * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) + + Executing cluster transition: ++ * Pseudo action: messaging-1_stop_0 + * Pseudo action: galera-0_stop_0 ++ * Pseudo action: galera-2_stop_0 + * Pseudo action: galera-master_demote_0 + * Pseudo action: redis-master_pre_notify_stop_0 + * Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0 +@@ -94,7 +96,6 @@ Executing cluster transition: + * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 + * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 + * Resource action: galera-0 monitor=20000 on controller-2 +- * Pseudo action: galera-2_stop_0 + * Resource action: rabbitmq notify on messaging-2 + * Resource action: rabbitmq notify on messaging-0 + * Pseudo action: rabbitmq_notified_0 +@@ -107,7 +108,6 @@ Executing cluster transition: + * Resource action: ip-172.17.1.17 start on controller-2 + * Resource action: ip-172.17.4.11 start on controller-2 + * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 +- * Pseudo action: messaging-1_stop_0 + * Pseudo action: redis_notified_0 + * Resource action: ip-172.17.1.14 monitor=10000 on controller-2 + * Resource action: ip-172.17.1.17 monitor=10000 on controller-2 +diff --git a/cts/scheduler/remote-recover-no-resources.dot b/cts/scheduler/remote-recover-no-resources.dot +index ef78aa6..a2f8ce0 100644 +--- a/cts/scheduler/remote-recover-no-resources.dot ++++ b/cts/scheduler/remote-recover-no-resources.dot +@@ -45,7 +45,6 @@ digraph "g" { + "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] + "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] + "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] +-"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] + "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] + "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] + "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] +diff --git a/cts/scheduler/remote-recover-no-resources.exp b/cts/scheduler/remote-recover-no-resources.exp +index 8a67c11..90470fb 100644 +--- a/cts/scheduler/remote-recover-no-resources.exp ++++ b/cts/scheduler/remote-recover-no-resources.exp +@@ -5,11 +5,7 @@ + + + +- +- +- +- +- ++ + + + +diff --git a/cts/scheduler/remote-recover-no-resources.summary b/cts/scheduler/remote-recover-no-resources.summary +index 89da784..18a989b 100644 +--- a/cts/scheduler/remote-recover-no-resources.summary ++++ b/cts/scheduler/remote-recover-no-resources.summary +@@ -54,6 +54,7 @@ Transition Summary: + * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) + + Executing cluster transition: ++ * Pseudo action: messaging-1_stop_0 + * Pseudo action: galera-0_stop_0 + * Pseudo action: galera-2_stop_0 + * Pseudo action: redis-master_pre_notify_stop_0 +@@ -92,7 +93,6 @@ Executing cluster transition: + * Pseudo action: ip-172.17.1.17_stop_0 + * Pseudo action: ip-172.17.4.11_stop_0 + * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 +- * Pseudo action: messaging-1_stop_0 + * Resource action: redis notify on controller-0 + * Resource action: redis notify on controller-2 + * Pseudo action: redis-master_confirmed-post_notify_stopped_0 +diff --git a/cts/scheduler/remote-recover-unknown.dot b/cts/scheduler/remote-recover-unknown.dot +index 5cd760b..29ab59f 100644 +--- a/cts/scheduler/remote-recover-unknown.dot ++++ b/cts/scheduler/remote-recover-unknown.dot +@@ -46,7 +46,6 @@ digraph "g" { + "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] + "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] + "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] +-"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] + "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] + "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] + "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] +diff --git a/cts/scheduler/remote-recover-unknown.exp b/cts/scheduler/remote-recover-unknown.exp +index ac6f004..82cb65f7 100644 +--- a/cts/scheduler/remote-recover-unknown.exp ++++ b/cts/scheduler/remote-recover-unknown.exp +@@ -5,11 +5,7 @@ + + + +- +- +- +- +- ++ + + + +diff --git a/cts/scheduler/remote-recover-unknown.summary b/cts/scheduler/remote-recover-unknown.summary +index 2c60713..4d7a411 100644 +--- a/cts/scheduler/remote-recover-unknown.summary ++++ b/cts/scheduler/remote-recover-unknown.summary +@@ -55,6 +55,7 @@ Transition Summary: + * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) + + Executing cluster transition: ++ * Pseudo action: messaging-1_stop_0 + * Pseudo action: galera-0_stop_0 + * Pseudo action: galera-2_stop_0 + * Pseudo action: redis-master_pre_notify_stop_0 +@@ -94,7 +95,6 @@ Executing cluster transition: + * Pseudo action: ip-172.17.1.17_stop_0 + * Pseudo action: ip-172.17.4.11_stop_0 + * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 +- * Pseudo action: messaging-1_stop_0 + * Resource action: redis notify on controller-0 + * Resource action: redis notify on controller-2 + * Pseudo action: redis-master_confirmed-post_notify_stopped_0 +-- +1.8.3.1 + + +From 9a5f7952c921f7f8eea3c7b0af711df2995a4e60 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 7 Jun 2019 17:11:27 -0500 +Subject: [PATCH 13/13] Low: libpe_status: don't add /var/log mount to bundles + if user did + +--- + lib/pengine/bundle.c | 10 ++++++++-- + 1 file changed, 8 insertions(+), 2 deletions(-) + +diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c +index b223f03..060e73a 100644 +--- a/lib/pengine/bundle.c ++++ b/lib/pengine/bundle.c +@@ -1027,6 +1027,7 @@ pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set) + xmlNode *xml_obj = NULL; + xmlNode *xml_resource = NULL; + pe__bundle_variant_data_t *bundle_data = NULL; ++ bool need_log_mount = TRUE; + + CRM_ASSERT(rsc != NULL); + pe_rsc_trace(rsc, "Processing resource %s...", rsc->id); +@@ -1151,6 +1152,9 @@ pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set) + + if (source && target) { + mount_add(bundle_data, source, target, options, flags); ++ if (strcmp(target, "/var/log") == 0) { ++ need_log_mount = FALSE; ++ } + } else { + pe_err("Invalid mount directive %s", ID(xml_child)); + } +@@ -1253,8 +1257,10 @@ pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set) + mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION, + DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none); + +- mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL, +- pe__bundle_mount_subdir); ++ if (need_log_mount) { ++ mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL, ++ pe__bundle_mount_subdir); ++ } + + port = calloc(1, sizeof(pe__bundle_port_t)); + if(bundle_data->control_port) { +-- +1.8.3.1 + diff --git a/SOURCES/010-fix-history-handing-on-fenced-restart.patch b/SOURCES/010-fix-history-handing-on-fenced-restart.patch new file mode 100644 index 0000000..eeaab70 --- /dev/null +++ b/SOURCES/010-fix-history-handing-on-fenced-restart.patch @@ -0,0 +1,606 @@ +From 14bb468ab404228cae34809420ef0763d3d54482 Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Thu, 13 Jun 2019 15:31:24 +0200 +Subject: [PATCH] Fix: fence-history: fail leftover pending-actions after + fenced-restart + +--- + daemons/fenced/fenced_history.c | 15 +++++++++++++++ + daemons/fenced/fenced_remote.c | 6 +++--- + daemons/fenced/pacemaker-fenced.h | 8 ++++++++ + 3 files changed, 26 insertions(+), 3 deletions(-) + +diff --git a/daemons/fenced/fenced_history.c b/daemons/fenced/fenced_history.c +index 7c129cc..b65b64c 100644 +--- a/daemons/fenced/fenced_history.c ++++ b/daemons/fenced/fenced_history.c +@@ -347,6 +347,21 @@ stonith_merge_in_history_list(GHashTable *history) + + updated = TRUE; + g_hash_table_iter_steal(&iter); ++ ++ if ((op->state != st_failed) && ++ (op->state != st_done) && ++ safe_str_eq(op->originator, stonith_our_uname)) { ++ crm_warn("received pending action we are supposed to be the " ++ "owner but it's not in our records -> fail it"); ++ op->state = st_failed; ++ op->completed = time(NULL); ++ /* use -EHOSTUNREACH to not introduce a new return-code that might ++ trigger unexpected results at other places and to prevent ++ remote_op_done from setting the delegate if not present ++ */ ++ stonith_bcast_result_to_peers(op, -EHOSTUNREACH); ++ } ++ + g_hash_table_insert(stonith_remote_op_list, op->id, op); + /* we could trim the history here but if we bail + * out after trim we might miss more recent entries +diff --git a/daemons/fenced/fenced_remote.c b/daemons/fenced/fenced_remote.c +index 7d61249..5b86f0f 100644 +--- a/daemons/fenced/fenced_remote.c ++++ b/daemons/fenced/fenced_remote.c +@@ -369,8 +369,8 @@ create_op_done_notify(remote_fencing_op_t * op, int rc) + return notify_data; + } + +-static void +-bcast_result_to_peers(remote_fencing_op_t * op, int rc) ++void ++stonith_bcast_result_to_peers(remote_fencing_op_t * op, int rc) + { + static int count = 0; + xmlNode *bcast = create_xml_node(NULL, T_STONITH_REPLY); +@@ -509,7 +509,7 @@ remote_op_done(remote_fencing_op_t * op, xmlNode * data, int rc, int dup) + subt = crm_element_value(data, F_SUBTYPE); + if (dup == FALSE && safe_str_neq(subt, "broadcast")) { + /* Defer notification until the bcast message arrives */ +- bcast_result_to_peers(op, rc); ++ stonith_bcast_result_to_peers(op, rc); + goto remote_op_done_cleanup; + } + +diff --git a/daemons/fenced/pacemaker-fenced.h b/daemons/fenced/pacemaker-fenced.h +index 3a2edbb..a8531a6 100644 +--- a/daemons/fenced/pacemaker-fenced.h ++++ b/daemons/fenced/pacemaker-fenced.h +@@ -149,6 +149,14 @@ typedef struct remote_fencing_op_s { + + } remote_fencing_op_t; + ++/*! ++ * \internal ++ * \brief Broadcast the result of an operation to the peers. ++ * \param op, Operation whose result should be broadcast ++ * \param rc, Result of the operation ++ */ ++void stonith_bcast_result_to_peers(remote_fencing_op_t * op, int rc); ++ + enum st_callback_flags { + st_callback_unknown = 0x0000, + st_callback_notify_fence = 0x0001, +-- +1.8.3.1 + +From a0bc0d3ab5aed64e37b1caae746f5c421696df1b Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Fri, 14 Jun 2019 13:41:43 +0200 +Subject: [PATCH] Fix: controld-fencing: remove-notifications upon + connection-destroy + +--- + daemons/controld/controld_fencing.c | 9 ++++++++- + 1 file changed, 8 insertions(+), 1 deletion(-) + +diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c +index 92336e9..b925bc5 100644 +--- a/daemons/controld/controld_fencing.c ++++ b/daemons/controld/controld_fencing.c +@@ -403,7 +403,14 @@ tengine_stonith_connection_destroy(stonith_t *st, stonith_event_t *e) + } + + if (stonith_api) { +- stonith_api->state = stonith_disconnected; ++ /* the client API won't properly reconnect notifications ++ * if they are still in the table - so remove them ++ */ ++ stonith_api->cmds->remove_notification(st, T_STONITH_NOTIFY_DISCONNECT); ++ stonith_api->cmds->remove_notification(st, T_STONITH_NOTIFY_FENCE); ++ if (stonith_api->state != stonith_disconnected) { ++ stonith_api->cmds->disconnect(st); ++ } + } + + if (AM_I_DC) { +-- +1.8.3.1 + +From 487cdd9e3ec6ab47fde5074acbb2ff564047d59c Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Tue, 18 Jun 2019 14:09:20 +0200 +Subject: [PATCH] Feature: fence-history: add notification upon history-synced + +--- + daemons/fenced/fenced_history.c | 5 +++++ + daemons/fenced/pacemaker-fenced.c | 3 +++ + daemons/fenced/pacemaker-fenced.h | 11 ++++++----- + include/crm/stonith-ng.h | 1 + + 4 files changed, 15 insertions(+), 5 deletions(-) + +diff --git a/daemons/fenced/fenced_history.c b/daemons/fenced/fenced_history.c +index b65b64c..cd08d74 100644 +--- a/daemons/fenced/fenced_history.c ++++ b/daemons/fenced/fenced_history.c +@@ -420,6 +420,11 @@ stonith_fence_history(xmlNode *msg, xmlNode **output, + stonith_fence_history_cleanup(target, + crm_element_value(msg, F_STONITH_CALLID) != NULL); + } else if (options & st_opt_broadcast) { ++ /* there is no clear sign atm for when a history sync ++ is done so send a notification for anything ++ that smells like history-sync ++ */ ++ do_stonith_notify(0, T_STONITH_NOTIFY_HISTORY_SYNCED, 0, NULL); + if (crm_element_value(msg, F_STONITH_CALLID)) { + /* this is coming from the stonith-API + * +diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c +index 7e9bb07..7a87f93 100644 +--- a/daemons/fenced/pacemaker-fenced.c ++++ b/daemons/fenced/pacemaker-fenced.c +@@ -279,6 +279,9 @@ get_stonith_flag(const char *name) + } else if (safe_str_eq(name, T_STONITH_NOTIFY_HISTORY)) { + return st_callback_notify_history; + ++ } else if (safe_str_eq(name, T_STONITH_NOTIFY_HISTORY_SYNCED)) { ++ return st_callback_notify_history_synced; ++ + } + return st_callback_unknown; + } +diff --git a/daemons/fenced/pacemaker-fenced.h b/daemons/fenced/pacemaker-fenced.h +index a8531a6..583cb47 100644 +--- a/daemons/fenced/pacemaker-fenced.h ++++ b/daemons/fenced/pacemaker-fenced.h +@@ -158,11 +158,12 @@ typedef struct remote_fencing_op_s { + void stonith_bcast_result_to_peers(remote_fencing_op_t * op, int rc); + + enum st_callback_flags { +- st_callback_unknown = 0x0000, +- st_callback_notify_fence = 0x0001, +- st_callback_device_add = 0x0004, +- st_callback_device_del = 0x0010, +- st_callback_notify_history = 0x0020 ++ st_callback_unknown = 0x0000, ++ st_callback_notify_fence = 0x0001, ++ st_callback_device_add = 0x0004, ++ st_callback_device_del = 0x0010, ++ st_callback_notify_history = 0x0020, ++ st_callback_notify_history_synced = 0x0040 + }; + + /* +diff --git a/include/crm/stonith-ng.h b/include/crm/stonith-ng.h +index b640732..418a03c 100644 +--- a/include/crm/stonith-ng.h ++++ b/include/crm/stonith-ng.h +@@ -29,6 +29,7 @@ extern "C" { + # define T_STONITH_NOTIFY_DISCONNECT "st_notify_disconnect" + # define T_STONITH_NOTIFY_FENCE "st_notify_fence" + # define T_STONITH_NOTIFY_HISTORY "st_notify_history" ++# define T_STONITH_NOTIFY_HISTORY_SYNCED "st_notify_history_synced" + + /* *INDENT-OFF* */ + enum stonith_state { +-- +1.8.3.1 + +From 03c4455fced74f093deb782198b1ba3076e52015 Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Tue, 18 Jun 2019 14:12:27 +0200 +Subject: [PATCH] Fix: fence-history: resync fence-history after fenced crash + +Setting up a 30s fallback timer to trigger history-sync if the +sync via DC doesn't happen +--- + daemons/controld/controld_callbacks.c | 2 +- + daemons/controld/controld_control.c | 2 + + daemons/controld/controld_fencing.c | 86 ++++++++++++++++++++++++++++++----- + daemons/controld/controld_fencing.h | 3 +- + 4 files changed, 79 insertions(+), 14 deletions(-) + +diff --git a/daemons/controld/controld_callbacks.c b/daemons/controld/controld_callbacks.c +index 3ce7470..48225ac 100644 +--- a/daemons/controld/controld_callbacks.c ++++ b/daemons/controld/controld_callbacks.c +@@ -211,7 +211,7 @@ peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *d + + } else if(AM_I_DC) { + if (appeared) { +- te_trigger_stonith_history_sync(); ++ te_trigger_stonith_history_sync(FALSE); + } else { + erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local); + } +diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c +index e99d605..f3bb20f 100644 +--- a/daemons/controld/controld_control.c ++++ b/daemons/controld/controld_control.c +@@ -259,6 +259,8 @@ crmd_exit(crm_exit_t exit_code) + crm_timer_stop(wait_timer); + crm_timer_stop(recheck_timer); + ++ te_cleanup_stonith_history_sync(NULL, TRUE); ++ + free(transition_timer); transition_timer = NULL; + free(integration_timer); integration_timer = NULL; + free(finalization_timer); finalization_timer = NULL; +diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c +index b925bc5..22fa727 100644 +--- a/daemons/controld/controld_fencing.c ++++ b/daemons/controld/controld_fencing.c +@@ -20,6 +20,9 @@ + # include + #endif + ++static void ++tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event); ++ + /* + * stonith failure counting + * +@@ -394,6 +397,8 @@ fail_incompletable_stonith(crm_graph_t *graph) + static void + tengine_stonith_connection_destroy(stonith_t *st, stonith_event_t *e) + { ++ te_cleanup_stonith_history_sync(st, FALSE); ++ + if (is_set(fsa_input_register, R_ST_REQUIRED)) { + crm_crit("Fencing daemon connection failed"); + mainloop_set_trigger(stonith_reconnect); +@@ -406,11 +411,12 @@ tengine_stonith_connection_destroy(stonith_t *st, stonith_event_t *e) + /* the client API won't properly reconnect notifications + * if they are still in the table - so remove them + */ +- stonith_api->cmds->remove_notification(st, T_STONITH_NOTIFY_DISCONNECT); +- stonith_api->cmds->remove_notification(st, T_STONITH_NOTIFY_FENCE); + if (stonith_api->state != stonith_disconnected) { + stonith_api->cmds->disconnect(st); + } ++ stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT); ++ stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_FENCE); ++ stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_HISTORY_SYNCED); + } + + if (AM_I_DC) { +@@ -622,7 +628,12 @@ te_connect_stonith(gpointer user_data) + stonith_api->cmds->register_notification(stonith_api, + T_STONITH_NOTIFY_FENCE, + tengine_stonith_notify); ++ stonith_api->cmds->register_notification(stonith_api, ++ T_STONITH_NOTIFY_HISTORY_SYNCED, ++ tengine_stonith_history_synced); ++ te_trigger_stonith_history_sync(TRUE); + } ++ + return TRUE; + } + +@@ -649,7 +660,12 @@ controld_disconnect_fencer(bool destroy) + // Prevent fencer connection from coming up again + clear_bit(fsa_input_register, R_ST_REQUIRED); + +- stonith_api->cmds->disconnect(stonith_api); ++ if (stonith_api->state != stonith_disconnected) { ++ stonith_api->cmds->disconnect(stonith_api); ++ } ++ stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT); ++ stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_FENCE); ++ stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_HISTORY_SYNCED); + } + if (destroy) { + if (stonith_api) { +@@ -673,6 +689,7 @@ do_stonith_history_sync(gpointer user_data) + if (stonith_api && (stonith_api->state != stonith_disconnected)) { + stonith_history_t *history = NULL; + ++ te_cleanup_stonith_history_sync(stonith_api, FALSE); + stonith_api->cmds->history(stonith_api, + st_opt_sync_call | st_opt_broadcast, + NULL, &history, 5); +@@ -845,7 +862,33 @@ te_fence_node(crm_graph_t *graph, crm_action_t *action) + */ + + static crm_trigger_t *stonith_history_sync_trigger = NULL; +-static mainloop_timer_t *stonith_history_sync_timer = NULL; ++static mainloop_timer_t *stonith_history_sync_timer_short = NULL; ++static mainloop_timer_t *stonith_history_sync_timer_long = NULL; ++ ++void ++te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers) ++{ ++ if (free_timers) { ++ mainloop_timer_del(stonith_history_sync_timer_short); ++ stonith_history_sync_timer_short = NULL; ++ mainloop_timer_del(stonith_history_sync_timer_long); ++ stonith_history_sync_timer_long = NULL; ++ } else { ++ mainloop_timer_stop(stonith_history_sync_timer_short); ++ mainloop_timer_stop(stonith_history_sync_timer_long); ++ } ++ ++ if (st) { ++ st->cmds->remove_notification(st, T_STONITH_NOTIFY_HISTORY_SYNCED); ++ } ++} ++ ++static void ++tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event) ++{ ++ te_cleanup_stonith_history_sync(st, FALSE); ++ crm_debug("Fence-history synced - cancel all timers"); ++} + + static gboolean + stonith_history_sync_set_trigger(gpointer user_data) +@@ -855,11 +898,18 @@ stonith_history_sync_set_trigger(gpointer user_data) + } + + void +-te_trigger_stonith_history_sync(void) ++te_trigger_stonith_history_sync(bool long_timeout) + { + /* trigger a sync in 5s to give more nodes the + * chance to show up so that we don't create + * unnecessary stonith-history-sync traffic ++ * ++ * the long timeout of 30s is there as a fallback ++ * so that after a successful connection to fenced ++ * we will wait for 30s for the DC to trigger a ++ * history-sync ++ * if this doesn't happen we trigger a sync locally ++ * (e.g. fenced segfaults and is restarted by pacemakerd) + */ + + /* as we are finally checking the stonith-connection +@@ -873,14 +923,26 @@ te_trigger_stonith_history_sync(void) + do_stonith_history_sync, NULL); + } + +- if(stonith_history_sync_timer == NULL) { +- stonith_history_sync_timer = +- mainloop_timer_add("history_sync", 5000, +- FALSE, stonith_history_sync_set_trigger, +- NULL); ++ if (long_timeout) { ++ if(stonith_history_sync_timer_long == NULL) { ++ stonith_history_sync_timer_long = ++ mainloop_timer_add("history_sync_long", 30000, ++ FALSE, stonith_history_sync_set_trigger, ++ NULL); ++ } ++ crm_info("Fence history will be synchronized cluster-wide within 30 seconds"); ++ mainloop_timer_start(stonith_history_sync_timer_long); ++ } else { ++ if(stonith_history_sync_timer_short == NULL) { ++ stonith_history_sync_timer_short = ++ mainloop_timer_add("history_sync_short", 5000, ++ FALSE, stonith_history_sync_set_trigger, ++ NULL); ++ } ++ crm_info("Fence history will be synchronized cluster-wide within 5 seconds"); ++ mainloop_timer_start(stonith_history_sync_timer_short); + } +- crm_info("Fence history will be synchronized cluster-wide within 5 seconds"); +- mainloop_timer_start(stonith_history_sync_timer); ++ + } + + /* end stonith history synchronization functions */ +diff --git a/daemons/controld/controld_fencing.h b/daemons/controld/controld_fencing.h +index 8f7f19b..2fe6d88 100644 +--- a/daemons/controld/controld_fencing.h ++++ b/daemons/controld/controld_fencing.h +@@ -29,6 +29,7 @@ void purge_stonith_cleanup(void); + void execute_stonith_cleanup(void); + + // stonith history synchronization +-void te_trigger_stonith_history_sync(void); ++void te_trigger_stonith_history_sync(bool long_timeout); ++void te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers); + + #endif +-- +1.8.3.1 + +From 2b038831edf6dd345c3f39f0fc27cfbf9503f512 Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Tue, 18 Jun 2019 21:54:49 +0200 +Subject: [PATCH] Fix: st_client: make safe to remove notifications from + notifications + +While cycling over the notification-list just mark for deletion +and delete afterwards. +--- + lib/fencing/st_client.c | 58 +++++++++++++++++++++++++++++++++++++++++++++---- + 1 file changed, 54 insertions(+), 4 deletions(-) + +diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c +index 629887a..ba23ac5 100644 +--- a/lib/fencing/st_client.c ++++ b/lib/fencing/st_client.c +@@ -67,6 +67,8 @@ typedef struct stonith_private_s { + mainloop_io_t *source; + GHashTable *stonith_op_callback_table; + GList *notify_list; ++ int notify_refcnt; ++ bool notify_deletes; + + void (*op_callback) (stonith_t * st, stonith_callback_data_t * data); + +@@ -77,6 +79,7 @@ typedef struct stonith_notify_client_s { + const char *obj_id; /* implement one day */ + const char *obj_type; /* implement one day */ + void (*notify) (stonith_t * st, stonith_event_t * e); ++ bool delete; + + } stonith_notify_client_t; + +@@ -211,6 +214,38 @@ log_action(stonith_action_t *action, pid_t pid) + } + } + ++/* when cycling through the list we don't want to delete items ++ so just mark them and when we know nobody is using the list ++ loop over it to remove the marked items ++ */ ++static void ++foreach_notify_entry (stonith_private_t *private, ++ GFunc func, ++ gpointer user_data) ++{ ++ private->notify_refcnt++; ++ g_list_foreach(private->notify_list, func, user_data); ++ private->notify_refcnt--; ++ if ((private->notify_refcnt == 0) && ++ private->notify_deletes) { ++ GList *list_item = private->notify_list; ++ ++ private->notify_deletes = FALSE; ++ while (list_item != NULL) ++ { ++ stonith_notify_client_t *list_client = list_item->data; ++ GList *next = g_list_next(list_item); ++ ++ if (list_client->delete) { ++ free(list_client); ++ private->notify_list = ++ g_list_delete_link(private->notify_list, list_item); ++ } ++ list_item = next; ++ } ++ } ++} ++ + static void + stonith_connection_destroy(gpointer user_data) + { +@@ -230,7 +265,7 @@ stonith_connection_destroy(gpointer user_data) + crm_xml_add(blob.xml, F_TYPE, T_STONITH_NOTIFY); + crm_xml_add(blob.xml, F_SUBTYPE, T_STONITH_NOTIFY_DISCONNECT); + +- g_list_foreach(native->notify_list, stonith_send_notification, &blob); ++ foreach_notify_entry(native, stonith_send_notification, &blob); + free_xml(blob.xml); + } + +@@ -1140,6 +1175,10 @@ stonithlib_GCompareFunc(gconstpointer a, gconstpointer b) + const stonith_notify_client_t *a_client = a; + const stonith_notify_client_t *b_client = b; + ++ if (a_client->delete || b_client->delete) { ++ /* make entries marked for deletion not findable */ ++ return -1; ++ } + CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0); + rc = strcmp(a_client->event, b_client->event); + if (rc == 0) { +@@ -1394,7 +1433,7 @@ stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata) + stonith_perform_callback(st, blob.xml, 0, 0); + + } else if (safe_str_eq(type, T_STONITH_NOTIFY)) { +- g_list_foreach(private->notify_list, stonith_send_notification, &blob); ++ foreach_notify_entry(private, stonith_send_notification, &blob); + } else if (safe_str_eq(type, T_STONITH_TIMEOUT_VALUE)) { + int call_id = 0; + int timeout = 0; +@@ -1592,8 +1631,13 @@ stonith_api_del_notification(stonith_t * stonith, const char *event) + if (list_item != NULL) { + stonith_notify_client_t *list_client = list_item->data; + +- private->notify_list = g_list_remove(private->notify_list, list_client); +- free(list_client); ++ if (private->notify_refcnt) { ++ list_client->delete = TRUE; ++ private->notify_deletes = TRUE; ++ } else { ++ private->notify_list = g_list_remove(private->notify_list, list_client); ++ free(list_client); ++ } + + crm_trace("Removed callback"); + +@@ -1754,6 +1798,10 @@ stonith_send_notification(gpointer data, gpointer user_data) + crm_warn("Skipping callback - NULL callback client"); + return; + ++ } else if (entry->delete) { ++ crm_trace("Skipping callback - marked for deletion"); ++ return; ++ + } else if (entry->notify == NULL) { + crm_warn("Skipping callback - NULL callback"); + return; +@@ -2037,6 +2085,8 @@ stonith_api_new(void) + private->stonith_op_callback_table = g_hash_table_new_full(g_direct_hash, g_direct_equal, + NULL, stonith_destroy_op_callback); + private->notify_list = NULL; ++ private->notify_refcnt = 0; ++ private->notify_deletes = FALSE; + + new_stonith->call_id = 1; + new_stonith->state = stonith_disconnected; +-- +1.8.3.1 + +From 03765b7803f935f0db149843a0b90aa9c872d922 Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Fri, 21 Jun 2019 14:13:10 +0200 +Subject: [PATCH] Test: CTS: new pattern to identify fenced reconnected + +Now that we are removing notifications upon disconnect a duplicate +notification can't be used as sign for reconnection any more. +--- + cts/patterns.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/cts/patterns.py b/cts/patterns.py +index 1b86ee7..8de67b1 100644 +--- a/cts/patterns.py ++++ b/cts/patterns.py +@@ -303,7 +303,7 @@ class crm_corosync(BasePatterns): + self.components["pacemaker-fenced"] = [ + r"error:.*Connection to (fencer|stonith-ng).* (closed|failed|lost)", + r"Fencing daemon connection failed", +- r"pacemaker-controld.*:\s*warn.*:\s*Callback already present", ++ r"pacemaker-controld.*Fencer successfully connected", + ] + self.components["pacemaker-fenced-ignore"] = [ + r"error:.*Connection to (fencer|stonith-ng).* (closed|failed|lost)", +-- +1.8.3.1 + +From c45c98cd77cb3e0913bcdb18fd6b116c3a25285d Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Fri, 21 Jun 2019 16:40:47 +0200 +Subject: [PATCH] Fix: controld-fencing: add notice-log for successful + fencer-connect + +--- + daemons/controld/controld_fencing.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c +index 22fa727..2428168 100644 +--- a/daemons/controld/controld_fencing.c ++++ b/daemons/controld/controld_fencing.c +@@ -632,6 +632,7 @@ te_connect_stonith(gpointer user_data) + T_STONITH_NOTIFY_HISTORY_SYNCED, + tengine_stonith_history_synced); + te_trigger_stonith_history_sync(TRUE); ++ crm_notice("Fencer successfully connected"); + } + + return TRUE; +-- +1.8.3.1 + diff --git a/SOURCES/011-crm_report.patch b/SOURCES/011-crm_report.patch new file mode 100644 index 0000000..b5b99d8 --- /dev/null +++ b/SOURCES/011-crm_report.patch @@ -0,0 +1,200 @@ +From 13809f57913cc5797d2a9d1ad19eb561a5113845 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 23 Aug 2019 17:28:49 -0500 +Subject: [PATCH 1/5] Fix: tools: correct crm_report argument parsing + +There were a few instances where crm_report's option names passed to getopt, +option names listed in help, and option names checked for did not match. + +Where getopt and checks matched, I went with that, so that anything that +worked before continues to work. +--- + tools/crm_report.in | 10 ++++++---- + 1 file changed, 6 insertions(+), 4 deletions(-) + +diff --git a/tools/crm_report.in b/tools/crm_report.in +index d1bc425..0ccec56 100644 +--- a/tools/crm_report.in ++++ b/tools/crm_report.in +@@ -10,7 +10,7 @@ + + TEMP=`@GETOPT_PATH@ \ + -o hv?xl:f:t:n:T:L:p:c:dSCu:D:MVse: \ +- --long help,cts:,cts-log:,dest:,node:,nodes:,from:,to:,sos-mode,logfile:,as-directory,single-node,cluster:,user:,max-depth:,version,features,rsh: \ ++ --long help,corosync,cts:,cts-log:,dest:,node:,nodes:,from:,to:,sos-mode,logfile:,as-directory,single-node,cluster:,user:,max-depth:,version,features,rsh: \ + -n 'crm_report' -- "$@"` + # The quotes around $TEMP are essential + eval set -- "$TEMP" +@@ -44,6 +44,7 @@ Required option: + + Options: + -V increase verbosity (may be specified multiple times) ++ -h, --help display this message + -v, --version display software version + --features display software features + -t, --to TIME time at which all problems were resolved +@@ -65,9 +66,10 @@ Options: + -C, --corosync force the cluster type to be corosync + -u, --user USER username to use when collecting data from other nodes + (default root) +- -D, --depth search depth to use when attempting to locate files ++ -D, --max-depth search depth to use when attempting to locate files + -e, --rsh command to use to run commands on other nodes + (default ssh -T) ++ -d, --as-directory leave result as a directory tree instead of archiving + --sos-mode use defaults suitable for being called by sosreport tool + (behavior subject to change and not useful to end users) + DEST, --dest DEST custom destination directory or file name +@@ -107,13 +109,13 @@ while true; do + case "$1" in + -x) set -x; shift;; + -V) verbose=`expr $verbose + 1`; shift;; +- -T|--cts-test) tests="$tests $2"; shift; shift;; ++ -T|--cts) tests="$tests $2"; shift; shift;; + --cts-log) ctslog="$2"; shift; shift;; + -f|--from) start_time=`get_time "$2"`; shift; shift;; + -t|--to) end_time=`get_time "$2"`; shift; shift;; + -n|--node|--nodes) nodes="$nodes $2"; shift; shift;; + -S|--single-node) nodes="$host"; shift;; +- -E|-l|--logfile) extra_logs="$extra_logs $2"; shift; shift;; ++ -l|--logfile) extra_logs="$extra_logs $2"; shift; shift;; + -p) sanitize_patterns="$sanitize_patterns $2"; shift; shift;; + -L) log_patterns="$log_patterns `echo $2 | sed 's/ /\\\W/g'`"; shift; shift;; + -d|--as-directory) compress=0; shift;; +-- +1.8.3.1 + + +From 24f0cbb4423a98b41e629c915b79778b39b5ae22 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 23 Aug 2019 17:39:45 -0500 +Subject: [PATCH 3/5] Fix: tools: don't ignore log if unrelated file is too + large + +This fixes a regression in 1.1.12: since cb420a04, findln_by_time() would skip +a log if any file in the current working directory (rather than the log itself) +was larger than 1GB. +--- + tools/report.common.in | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tools/report.common.in b/tools/report.common.in +index 7dd00b3..4fed6bb 100644 +--- a/tools/report.common.in ++++ b/tools/report.common.in +@@ -538,7 +538,7 @@ findln_by_time() { + # Some logs can be massive (over 1,500,000,000 lines have been seen in the wild) + # Even just 'wc -l' on these files can take 10+ minutes + +- local fileSize=`ls -lh | awk '{ print $5 }' | grep -ie G` ++ local fileSize=`ls -lh "$logf" | awk '{ print $5 }' | grep -ie G` + if [ x$fileSize != x ]; then + warning "$logf is ${fileSize} in size and could take many hours to process. Skipping." + return +-- +1.8.3.1 + + +From 885d9acdb8132a437b48d4d9e8121131cbedb3da Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 23 Aug 2019 22:38:51 -0500 +Subject: [PATCH 4/5] Fix: tools: check for tar in crm_report + +crm_report requires tar, so check for its existence up front. +--- + tools/crm_report.in | 4 ++++ + tools/report.collector.in | 2 ++ + tools/report.common.in | 10 ++++++++++ + 3 files changed, 16 insertions(+) + +diff --git a/tools/crm_report.in b/tools/crm_report.in +index 0ccec56..1818879 100644 +--- a/tools/crm_report.in ++++ b/tools/crm_report.in +@@ -419,6 +419,10 @@ getnodes() { + # TODO: Look for something like crm_update_peer + } + ++if [ $compress -eq 1 ]; then ++ require_tar ++fi ++ + if [ "x$tests" != "x" ]; then + do_cts + +diff --git a/tools/report.collector.in b/tools/report.collector.in +index 9419f17..315b785 100644 +--- a/tools/report.collector.in ++++ b/tools/report.collector.in +@@ -747,6 +747,8 @@ collect_logs() { + trap "" 0 + } + ++require_tar ++ + debug "Initializing $REPORT_TARGET subdir" + if [ "$REPORT_MASTER" != "$REPORT_TARGET" ]; then + if [ -e $REPORT_HOME/$REPORT_TARGET ]; then +diff --git a/tools/report.common.in b/tools/report.common.in +index 4fed6bb..73ec0dc 100644 +--- a/tools/report.common.in ++++ b/tools/report.common.in +@@ -114,6 +114,13 @@ fatal() { + exit 1 + } + ++require_tar() { ++ which tar >/dev/null 2>&1 ++ if [ $? -ne 0 ]; then ++ fatal "Required program 'tar' not found, please install and re-run" ++ fi ++} ++ + # check if process of given substring in its name does exist; + # only look for processes originated by user 0 (by UID), "@CRM_DAEMON_USER@" + # or effective user running this script, and/or group 0 (by GID), +@@ -525,6 +532,9 @@ shrink() { + + cd $dir >/dev/null 2>&1 + tar $tar_options $target $base >/dev/null 2>&1 ++ if [ $? -ne 0 ]; then ++ fatal "Could not archive $base, please investigate and collect manually" ++ fi + cd $olddir >/dev/null 2>&1 + + echo $target +-- +1.8.3.1 + + +From 5dcdb1eef727912fe33d7c8d9d2a4076fee7eb70 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 23 Aug 2019 22:15:50 -0500 +Subject: [PATCH 5/5] Build: rpm: add soft dependency on tar and bzip2 + +... which are needed by crm_report. Minimal OS installations are increasingly +popular, and the existence of tar can't be assumed. These are soft dependencies +because they are only needed for crm_report, not cluster functioning, and a +soft dependency allows users to keep a smaller footprint if desired while +providing full functionality to the typical user. +--- + pacemaker.spec.in | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/pacemaker.spec.in b/pacemaker.spec.in +index 0143b63..0f1638b 100644 +--- a/pacemaker.spec.in ++++ b/pacemaker.spec.in +@@ -279,6 +279,9 @@ Group: System Environment/Daemons + Requires: %{name}-libs%{?_isa} = %{version}-%{release} + %if 0%{?fedora} > 22 || 0%{?rhel} > 7 + Recommends: pcmk-cluster-manager = %{version}-%{release} ++# For crm_report ++Recommends: tar ++Recommends: bzip2 + %endif + Requires: perl-TimeDate + Requires: procps-ng +-- +1.8.3.1 + diff --git a/SOURCES/rhbz-url.patch b/SOURCES/rhbz-url.patch deleted file mode 100644 index 9e25e7e..0000000 --- a/SOURCES/rhbz-url.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 9b74fb4d667cf187c1c80aeb39ff3b3c12846421 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 18 Apr 2017 14:17:38 -0500 -Subject: [PATCH] Low: tools: show Red Hat bugzilla URL when using crm_report - ---- - tools/crm_report.in | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/tools/crm_report.in b/tools/crm_report.in -index 26050a7..4715155 100644 ---- a/tools/crm_report.in -+++ b/tools/crm_report.in -@@ -225,7 +225,7 @@ EOF - log "Collected results are available in $fname" - log " " - log "Please create a bug entry at" -- log " http://bugs.clusterlabs.org/enter_bug.cgi?product=Pacemaker" -+ log " https://bugzilla.redhat.com/" - log "Include a description of your problem and attach this tarball" - log " " - log "Thank you for taking time to create this report." --- -1.8.3.1 - diff --git a/SPECS/pacemaker.spec b/SPECS/pacemaker.spec index 4beba17..aec0b85 100644 --- a/SPECS/pacemaker.spec +++ b/SPECS/pacemaker.spec @@ -13,12 +13,12 @@ ## Upstream pacemaker version, and its package version (specversion ## can be incremented to build packages reliably considered "newer" ## than previously built packages with the same pcmkversion) -%global pcmkversion 2.0.1 -%global specversion 4 +%global pcmkversion 2.0.2 +%global specversion 3 ## Upstream commit (or git tag, such as "Pacemaker-" plus the ## {pcmkversion} macro for an official release) to use for this package -%global commit 0eb799156489376e13fb79dca47ea9160e9d4595 +%global commit 744a30d655c9fbd66ad6e103697db0283bb90779 ## Since git v2.11, the extent of abbreviation is autoscaled by default ## (used to be constant of 7), so we need to convey it for non-tags, too. %global commit_abbrev 7 @@ -195,7 +195,7 @@ Name: pacemaker Summary: Scalable High-Availability cluster resource manager Version: %{pcmkversion} -Release: %{pcmk_release}%{?dist}.4 +Release: %{pcmk_release}%{?dist} %if %{defined _unitdir} License: GPLv2+ and LGPLv2+ %else @@ -211,17 +211,17 @@ Source0: https://github.com/%{github_owner}/%{name}/archive/%{commit}/%{na Source1: nagios-agents-metadata-%{nagios_hash}.tar.gz # upstream commits -Patch1: 001-rc4.patch -Patch2: 002-security.patch -Patch3: 003-security-log.patch -Patch4: 004-security-active.patch -Patch5: 005-security-code.patch -Patch6: 006-migration.patch -Patch7: 007-unfencing.patch -Patch8: 008-remote.patch - -# patches that aren't from upstream -Patch100: rhbz-url.patch +Patch1: 001-xmldiffs.patch +Patch2: 002-failed-monitors.patch +Patch3: 003-fencer-logs.patch +Patch4: 004-concurrent-fencing.patch +Patch5: 005-glib-priorities.patch +Patch6: 006-bundle-fixes.patch +Patch7: 007-fork-controld_fencing.patch +Patch8: 008-stonith_admin-header-refactoring.patch +Patch9: 009-improve-pacemaker_remote-handling.patch +Patch10: 010-fix-history-handing-on-fenced-restart.patch +Patch11: 011-crm_report.patch Requires: resource-agents Requires: %{name}-libs%{?_isa} = %{version}-%{release} @@ -277,6 +277,13 @@ BuildRequires: inkscape asciidoc publican %endif Provides: pcmk-cluster-manager = %{version}-%{release} +Provides: pcmk-cluster-manager%{?_isa} = %{version}-%{release} + +# Bundled bits +## Pacemaker uses the crypto/md5-buffer module from gnulib +%if 0%{?fedora} || 0%{?rhel} +Provides: bundled(gnulib) +%endif %description Pacemaker is an advanced, scalable High-Availability cluster resource @@ -297,9 +304,11 @@ License: GPLv2+ and LGPLv2+ Summary: Command line tools for controlling Pacemaker clusters Group: System Environment/Daemons Requires: %{name}-libs%{?_isa} = %{version}-%{release} -#%if 0%{?fedora} > 22 || 0%{?rhel} > 7 +%if 0%{?fedora} > 22 || 0%{?rhel} > 7 #Recommends: pcmk-cluster-manager = %{version}-%{release} -#%endif +Requires: tar +Requires: bzip2 +%endif Requires: perl-TimeDate Requires: procps-ng Requires: psmisc @@ -319,6 +328,8 @@ Summary: Core Pacemaker libraries Group: System Environment/Daemons Requires(pre): shadow-utils Requires: %{name}-schemas = %{version}-%{release} +# sbd 1.4.0+ supports the libpe_status API for pe_working_set_t +Conflicts: sbd < 1.4.0 %description libs Pacemaker is an advanced, scalable High-Availability cluster resource @@ -358,6 +369,7 @@ Requires: procps-ng # -remote can be fully independent of systemd %{?systemd_ordering}%{!?systemd_ordering:%{?systemd_requires}} Provides: pcmk-cluster-manager = %{version}-%{release} +Provides: pcmk-cluster-manager%{?_isa} = %{version}-%{release} %description remote Pacemaker is an advanced, scalable High-Availability cluster resource @@ -466,6 +478,9 @@ export docdir=%{pcmk_docdir} export systemdunitdir=%{?_unitdir}%{!?_unitdir:no} +# RHEL changes pacemaker's concurrent-fencing default to true +export CPPFLAGS="-DDEFAULT_CONCURRENT_FENCING_TRUE" + %if %{with hardening} # prefer distro-provided hardening flags in case they are defined # through _hardening_{c,ld}flags macros, configure script will @@ -490,6 +505,7 @@ export LDFLAGS_HARDENED_LIB="%{?_hardening_ldflags}" %{?gnutls_priorities: --with-gnutls-priorities="%{gnutls_priorities}"} \ --with-initdir=%{_initrddir} \ --localstatedir=%{_var} \ + --with-bug-url=https://bugzilla.redhat.com/ \ --with-nagios \ --with-nagios-metadata-dir=%{_datadir}/pacemaker/nagios/plugins-metadata/ \ --with-nagios-plugin-dir=%{_libdir}/nagios/plugins/ \ @@ -504,7 +520,7 @@ sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool make %{_smp_mflags} V=1 all %check -{ cts/cts-scheduler --run one-or-more-unrunnable-instances \ +{ cts/cts-scheduler --run load-stopped-loop \ && cts/cts-cli \ && touch .CHECKED } 2>&1 | sed 's/[fF]ail/faiil/g' # prevent false positives in rpmlint @@ -715,7 +731,6 @@ exit 0 %{_sbindir}/crm_attribute %{_sbindir}/crm_master -%{_sbindir}/stonith_admin %doc %{_mandir}/man7/pacemaker-controld.* %doc %{_mandir}/man7/pacemaker-schedulerd.* @@ -725,7 +740,6 @@ exit 0 %doc %{_mandir}/man8/crm_attribute.* %doc %{_mandir}/man8/crm_master.* %doc %{_mandir}/man8/pacemakerd.* -%doc %{_mandir}/man8/stonith_admin.* %doc %{_datadir}/pacemaker/alerts @@ -764,6 +778,7 @@ exit 0 %{_sbindir}/crm_mon %{_sbindir}/crm_node %{_sbindir}/crm_resource +%{_sbindir}/crm_rule %{_sbindir}/crm_standby %{_sbindir}/crm_verify %{_sbindir}/crmadmin @@ -772,11 +787,11 @@ exit 0 %{_sbindir}/crm_simulate %{_sbindir}/crm_report %{_sbindir}/crm_ticket -%exclude %{_datadir}/pacemaker/alerts -%exclude %{_datadir}/pacemaker/tests -%{_datadir}/pacemaker -%exclude %{_datadir}/pacemaker/*.rng -%exclude %{_datadir}/pacemaker/*.xsl +%{_sbindir}/stonith_admin +# "dirname" is owned by -schemas, which is a prerequisite +%{_datadir}/pacemaker/report.collector +%{_datadir}/pacemaker/report.common +# XXX "dirname" is not owned by any prerequisite %{_datadir}/snmp/mibs/PCMK-MIB.txt %exclude /usr/lib/ocf/resource.d/pacemaker/controld @@ -797,7 +812,6 @@ exit 0 %exclude %{_mandir}/man8/crm_master.* %exclude %{_mandir}/man8/pacemakerd.* %exclude %{_mandir}/man8/pacemaker-remoted.* -%exclude %{_mandir}/man8/stonith_admin.* %license licenses/GPLv2 %doc COPYING @@ -816,9 +830,8 @@ exit 0 %{_libdir}/libcrmcommon.so.* %{_libdir}/libpe_status.so.* %{_libdir}/libpe_rules.so.* -%{_libdir}/libpengine.so.* +%{_libdir}/libpacemaker.so.* %{_libdir}/libstonithd.so.* -%{_libdir}/libtransitioner.so.* %license licenses/LGPLv2.1 %doc COPYING %doc ChangeLog @@ -878,8 +891,10 @@ exit 0 %files schemas %license licenses/GPLv2 +%dir %{_datadir}/pacemaker %{_datadir}/pacemaker/*.rng %{_datadir}/pacemaker/*.xsl +%{_datadir}/pacemaker/api %files nagios-plugins-metadata %dir %{_datadir}/pacemaker/nagios/plugins-metadata @@ -887,33 +902,44 @@ exit 0 %license %{nagios_name}-%{nagios_hash}/COPYING %changelog -* Mon Aug 5 2019 Ken Gaillot - 2.0.1-4.4 -- Handle losing remote node while it is shutting down -- Resolves: rhbz#1734066 - -* Thu May 9 2019 Klaus Wenninger - 2.0.1-4.3 -- New build with fixed test in gating.yaml -- Resolves: rhbz#1694557 -- Resolves: rhbz#1695247 -- Resolves: rhbz#1697264 -- Resolves: rhbz#1697265 - -* Mon May 6 2019 Ken Gaillot - 2.0.1-4.2 -- New build to apply z-stream tag -- Resolves: rhbz#1694557 -- Resolves: rhbz#1695247 -- Resolves: rhbz#1697264 -- Resolves: rhbz#1697265 - -* Sun May 5 2019 Ken Gaillot - 2.0.1-4.1 +* Mon Aug 26 2019 Ken Gaillot - 2.0.2-3 +- Make pacemaker-cli require tar and bzip2 +- Resolves: rhbz#1741580 + +* Fri Jun 21 2019 Klaus Wenninger - 2.0.2-2 +- Synchronize fence-history on fenced-restart +- Cleanup leftover pending-fence-actions when fenced is restarted +- Improve fencing of remote-nodes +- Resolves: rhbz#1708380 +- Resolves: rhbz#1708378 +- Resolves: rhbz#1721198 +- Resolves: rhbz#1695737 + +* Thu Jun 6 2019 Ken Gaillot - 2.0.2-1 +- Add stonith_admin option to display XML output +- Add new crm_rule tool to check date/time rules +- List any constraints cleared by crm_resource --clear +- crm_resource --validate can now get resource parameters from command line +- Rebase on upstream version 2.0.2 +- Default concurrent-fencing to true +- Resolves: rhbz#1555939 +- Resolves: rhbz#1572116 +- Resolves: rhbz#1631752 +- Resolves: rhbz#1637020 +- Resolves: rhbz#1695737 +- Resolves: rhbz#1715426 + +* Wed May 15 2019 Ken Gaillot - 2.0.1-5 +- Add gating tests for CI +- Restore correct behavior when live migration is interrupted - Improve clients' authentication of IPC servers (CVE-2018-16877) - Fix use-after-free with potential information disclosure (CVE-2019-3885) - Improve pacemakerd authentication of running subdaemons (CVE-2018-16878) -- Restore correct behavior when live migration is interrupted -- Resolves: rhbz#1694557 -- Resolves: rhbz#1695247 -- Resolves: rhbz#1697264 -- Resolves: rhbz#1697265 +- Resolves: rhbz#1682116 +- Resolves: rhbz#1684306 +- Resolves: rhbz#1694558 +- Resolves: rhbz#1694560 +- Resolves: rhbz#1694908 * Tue Jan 29 2019 Ken Gaillot - 2.0.1-4 - Remove duplicate fence history state listing in crm_mon XML output