From 824738210258e8d0486f54726de5fe657b648a96 Mon Sep 17 00:00:00 2001 From: Chris Lumens Date: Fri, 14 Dec 2018 16:58:30 -0500 Subject: [PATCH 01/69] Bug: tests: Add CRM_EX_DIGEST definition to cts-cli.in. This is the value that's returned when unexpected output happens in the CLI regression tests. However since it's not defined in cts-cli, 0 is returned instead. Add a definition for it to the block with all the other error codes. --- cts/cts-cli.in | 1 + 1 file changed, 1 insertion(+) diff --git a/cts/cts-cli.in b/cts/cts-cli.in index 2ce912d..880cebb 100755 --- a/cts/cts-cli.in +++ b/cts/cts-cli.in @@ -55,6 +55,7 @@ CRM_EX_INSUFFICIENT_PRIV=4 CRM_EX_USAGE=64 CRM_EX_CONFIG=78 CRM_EX_OLD=103 +CRM_EX_DIGEST=104 CRM_EX_NOSUCH=105 CRM_EX_UNSAFE=107 CRM_EX_EXISTS=108 -- 1.8.3.1 From f05828dc774059dc39444e186839753a8d5bf8f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= Date: Mon, 17 Dec 2018 18:25:12 +0100 Subject: [PATCH 02/69] Build: spec: Provides: not dissected for extra arch qualification That means that with arch-qualified-only virtual Provides: (introduced as of 73e2c94a3 for which this is a follow-up), the dependants would need to track Requires:/Recommends: etc. in the same qualified form unconditionally (which was not previously true in pacemaker packaging itself as arranged with the said commit, which led to this discovery), since only that would match string-wise. Solution is to declare Provides: for both forms in parallel, providing the depending packages with the benefit of choice (or conversely, to prevent hard to debug install transaction unresolvability issues; Ken did a great work on this front). See also: https://bugzilla.redhat.com/show_bug.cgi?id=1659259 --- pacemaker.spec.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pacemaker.spec.in b/pacemaker.spec.in index bfe68bf..dbc8360 100644 --- a/pacemaker.spec.in +++ b/pacemaker.spec.in @@ -248,6 +248,7 @@ BuildRequires: inkscape asciidoc publican %endif %endif +Provides: pcmk-cluster-manager = %{version}-%{release} Provides: pcmk-cluster-manager%{?_isa} = %{version}-%{release} %description @@ -330,6 +331,7 @@ Requires: procps-ng %endif # -remote can be fully independent of systemd %{?systemd_ordering}%{!?systemd_ordering:%{?systemd_requires}} +Provides: pcmk-cluster-manager = %{version}-%{release} Provides: pcmk-cluster-manager%{?_isa} = %{version}-%{release} %description remote -- 1.8.3.1 From 2a24cd38aea749c94d9c5b2670ed9f52056abeef Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Wed, 12 Dec 2018 16:27:36 -0600 Subject: [PATCH 03/69] Refactor: cts-exec: remove dead code --- cts/cts-exec.in | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/cts/cts-exec.in b/cts/cts-exec.in index c451841..235a966 100644 --- a/cts/cts-exec.in +++ b/cts/cts-exec.in @@ -117,17 +117,6 @@ def output_from_command(command): return pipe_output(test).split("\n") -def write_file(filename, contents, executable=False): - """ Create a file. """ - - print("Installing %s ..." % (filename)) - f = io.open(filename, "w+") - f.write(contents) - f.close() - if executable: - os.chmod(filename, EXECMODE) - - class TestError(Exception): """ Base class for exceptions in this module """ pass -- 1.8.3.1 From 7a749048e5462430c38a0b233b8e0e51f346f98e Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 13 Dec 2018 11:27:57 -0600 Subject: [PATCH 04/69] Fix: tools: stonith_admin -I doesn't require an agent regression since cc4c8411b (2.0.1-rc1) --- tools/stonith_admin.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/stonith_admin.c b/tools/stonith_admin.c index c9ed971..5160505 100644 --- a/tools/stonith_admin.c +++ b/tools/stonith_admin.c @@ -495,14 +495,17 @@ main(int argc, char **argv) case '?': crm_help(flag, CRM_EX_OK); break; - case 'I': + case 'K': - no_connect = 1; required_agent = true; /* fall through */ + case 'I': + no_connect = 1; + /* fall through */ case 'L': action = flag; break; + case 'q': quiet = 1; break; -- 1.8.3.1 From 5f75a663a1b5ea13a4cef114dc13169b01b2b29a Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 18 Dec 2018 12:24:34 -0600 Subject: [PATCH 05/69] Fix: libpe_status: avoid double free of stop_needed list regression in 2.0.1-rc1 introduced by bf69beb3 --- lib/pengine/unpack.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c index ac27121..a445442 100644 --- a/lib/pengine/unpack.c +++ b/lib/pengine/unpack.c @@ -1153,6 +1153,7 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set) } } g_list_free(data_set->stop_needed); + data_set->stop_needed = NULL; } for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { -- 1.8.3.1 From b8c592cb5fe431215b2f7755fe3a9c3a256f8d70 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Mon, 17 Dec 2018 14:16:48 -0600 Subject: [PATCH 06/69] Build: GNUmakefile: update change log substitutions --- GNUmakefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/GNUmakefile b/GNUmakefile index b9f7590..8e5bdd7 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -333,7 +333,15 @@ changes: summary @printf "\n- Features added since $(LAST_RELEASE)\n" @git log --pretty=format:' +%s' --abbrev-commit $(LAST_RELEASE)..HEAD | grep -e Feature: | sed -e 's@Feature:@@' | sort -uf @printf "\n- Changes since $(LAST_RELEASE)\n" - @git log --pretty=format:' +%s' --abbrev-commit $(LAST_RELEASE)..HEAD | grep -e High: -e Fix: -e Bug | sed -e 's@Fix:@@' -e s@High:@@ -e s@Fencing:@fencing:@ -e 's@Bug@ Bug@' -e s@PE:@scheduler:@ -e s@pengine:@scheduler:@ | sort -uf + @git log --pretty=format:' +%s' --no-merges --abbrev-commit $(LAST_RELEASE)..HEAD \ + | grep -e High: -e Fix: -e Bug | sed \ + -e 's@\(Fix\|High\|Bug\):@@' \ + -e 's@\(cib\|pacemaker-based\|based\):@CIB:@' \ + -e 's@\(crmd\|pacemaker-controld\|controld\):@controller:@' \ + -e 's@\(lrmd\|pacemaker-execd\|execd\):@executor:@' \ + -e 's@\(Fencing\|stonithd\|stonith\|pacemaker-fenced\|fenced\):@fencing:@' \ + -e 's@\(PE\|pengine\|pacemaker-schedulerd\|schedulerd\):@scheduler:@' \ + | sort -uf changelog: @make changes > ChangeLog -- 1.8.3.1 From dc51b0e7e3c910c1d6233a259c66f5be3f93eb73 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Mon, 17 Dec 2018 17:29:07 -0600 Subject: [PATCH 07/69] Build: replace: remove uuid_parse.c doesn't appear to have ever been used, likely was carried over from heartbeat --- replace/uuid_parse.c | 515 --------------------------------------------------- 1 file changed, 515 deletions(-) delete mode 100644 replace/uuid_parse.c diff --git a/replace/uuid_parse.c b/replace/uuid_parse.c deleted file mode 100644 index 5da6223..0000000 --- a/replace/uuid_parse.c +++ /dev/null @@ -1,515 +0,0 @@ -/* - * uuid: emulation of e2fsprogs interface if implementation lacking. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - * Original uuid implementation: copyright (C) Theodore Ts'o - * - * This importation into heartbeat: - * Copyright (C) 2004 David Lee - * - */ - -#include -#include -#ifdef HAVE_UNISTD_H -# include -#endif -#ifdef HAVE_STDLIB_H -# include -#endif -#include -#include - -#include - -/* - * Local "replace" implementation of uuid functions. - */ - -#include -#include -#include - -/* UUID Variant definitions */ -#define UUID_VARIANT_NCS 0 -#define UUID_VARIANT_DCE 1 -#define UUID_VARIANT_MICROSOFT 2 -#define UUID_VARIANT_OTHER 3 - -/* UUID Type definitions */ -#define UUID_TYPE_DCE_TIME 1 -#define UUID_TYPE_DCE_RANDOM 4 - -/* For uuid_compare() */ -#define UUCMP(u1,u2) if (u1 != u2) return((u1 < u2) ? -1 : 1); - -/************************************ - * Private types - ************************************/ - -#define longlong long long - -/* - * Offset between 15-Oct-1582 and 1-Jan-70 - */ -#define TIME_OFFSET_HIGH 0x01B21DD2 -#define TIME_OFFSET_LOW 0x13814000 - -#if (SIZEOF_INT == 4) -typedef unsigned int __u32; -#elif (SIZEOF_LONG == 4) -typedef unsigned long __u32; -#endif - -#if (SIZEOF_INT == 2) -typedef int __s16; -typedef unsigned int __u16; -#elif (SIZEOF_SHORT == 2) -typedef short __s16; -typedef unsigned short __u16; -#endif - -typedef unsigned char __u8; - -struct uuid { - __u32 time_low; - __u16 time_mid; - __u16 time_hi_and_version; - __u16 clock_seq; - __u8 node[6]; -}; - -/************************************ - * internal routines - ************************************/ -static void -uuid_pack(const struct uuid *uu, uuid_t ptr) -{ - __u32 tmp; - unsigned char *out = ptr; - - tmp = uu->time_low; - out[3] = (unsigned char)tmp; - tmp >>= 8; - out[2] = (unsigned char)tmp; - tmp >>= 8; - out[1] = (unsigned char)tmp; - tmp >>= 8; - out[0] = (unsigned char)tmp; - - tmp = uu->time_mid; - out[5] = (unsigned char)tmp; - tmp >>= 8; - out[4] = (unsigned char)tmp; - - tmp = uu->time_hi_and_version; - out[7] = (unsigned char)tmp; - tmp >>= 8; - out[6] = (unsigned char)tmp; - - tmp = uu->clock_seq; - out[9] = (unsigned char)tmp; - tmp >>= 8; - out[8] = (unsigned char)tmp; - - memcpy(out + 10, uu->node, 6); -} - -static void -uuid_unpack(const uuid_t in, struct uuid *uu) -{ - const __u8 *ptr = in; - __u32 tmp; - - tmp = *ptr++; - tmp = (tmp << 8) | *ptr++; - tmp = (tmp << 8) | *ptr++; - tmp = (tmp << 8) | *ptr++; - uu->time_low = tmp; - - tmp = *ptr++; - tmp = (tmp << 8) | *ptr++; - uu->time_mid = tmp; - - tmp = *ptr++; - tmp = (tmp << 8) | *ptr++; - uu->time_hi_and_version = tmp; - - tmp = *ptr++; - tmp = (tmp << 8) | *ptr++; - uu->clock_seq = tmp; - - memcpy(uu->node, ptr, 6); -} - -/************************************ - * Main routines, except uuid_generate*() - ************************************/ -void -uuid_clear(uuid_t uu) -{ - memset(uu, 0, 16); -} - -int -uuid_compare(const uuid_t uu1, const uuid_t uu2) -{ - struct uuid uuid1, uuid2; - - uuid_unpack(uu1, &uuid1); - uuid_unpack(uu2, &uuid2); - - UUCMP(uuid1.time_low, uuid2.time_low); - UUCMP(uuid1.time_mid, uuid2.time_mid); - UUCMP(uuid1.time_hi_and_version, uuid2.time_hi_and_version); - UUCMP(uuid1.clock_seq, uuid2.clock_seq); - return memcmp(uuid1.node, uuid2.node, 6); -} - -void -uuid_copy(uuid_t dst, const uuid_t src) -{ - unsigned char *cp1; - const unsigned char *cp2; - int i; - - for (i = 0, cp1 = dst, cp2 = src; i < 16; i++) - *cp1++ = *cp2++; -} - -/* if uu is the null uuid, return 1 else 0 */ -int -uuid_is_null(const uuid_t uu) -{ - const unsigned char *cp; - int i; - - for (i = 0, cp = uu; i < 16; i++) - if (*cp++) - return 0; - return 1; -} - -/* 36byte-string=>uuid */ -int -uuid_parse(const char *in, uuid_t uu) -{ - struct uuid uuid; - int i; - const char *cp; - char buf[3]; - - if (strlen(in) != 36) - return -1; - for (i = 0, cp = in; i <= 36; i++, cp++) { - if ((i == 8) || (i == 13) || (i == 18) || (i == 23)) { - if (*cp == '-') - continue; - else - return -1; - } - if (i == 36) - if (*cp == 0) - continue; - if (!isxdigit((int)*cp)) - return -1; - } - uuid.time_low = strtoul(in, NULL, 16); - uuid.time_mid = strtoul(in + 9, NULL, 16); - uuid.time_hi_and_version = strtoul(in + 14, NULL, 16); - uuid.clock_seq = strtoul(in + 19, NULL, 16); - cp = in + 24; - buf[2] = 0; - for (i = 0; i < 6; i++) { - buf[0] = *cp++; - buf[1] = *cp++; - uuid.node[i] = strtoul(buf, NULL, 16); - } - - uuid_pack(&uuid, uu); - return 0; -} - -/* uuid=>36byte-string-with-null */ -void -uuid_unparse(const uuid_t uu, char *out) -{ - struct uuid uuid; - - uuid_unpack(uu, &uuid); - sprintf(out, - "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x", - uuid.time_low, uuid.time_mid, uuid.time_hi_and_version, - uuid.clock_seq >> 8, uuid.clock_seq & 0xFF, - uuid.node[0], uuid.node[1], uuid.node[2], uuid.node[3], uuid.node[4], uuid.node[5]); -} - -/************************************ - * Main routines: uuid_generate*() - ************************************/ - -#include -#include -#include - -#ifdef HAVE_SYS_IOCTL_H -# include -#endif -#ifdef HAVE_SYS_SOCKET_H -# include -#endif -#ifdef HAVE_SYS_SOCKIO_H -# include -#endif -#ifdef HAVE_NET_IF_H -# include -#endif -#ifdef HAVE_NETINET_IN_H -# include -#endif - -#ifdef HAVE_SRANDOM -# define srand(x) srandom(x) -# define rand() random() -#endif - -static int -get_random_fd(void) -{ - struct timeval tv; - static int fd = -2; - int i; - - if (fd == -2) { - gettimeofday(&tv, 0); - fd = open("/dev/urandom", O_RDONLY); - if (fd == -1) - fd = open("/dev/random", O_RDONLY | O_NONBLOCK); - srand((getpid() << 16) ^ getuid() ^ tv.tv_sec ^ tv.tv_usec); - } - /* Crank the random number generator a few times */ - gettimeofday(&tv, 0); - for (i = (tv.tv_sec ^ tv.tv_usec) & 0x1F; i > 0; i--) - rand(); - return fd; -} - -/* - * Generate a series of random bytes. Use /dev/urandom if possible, - * and if not, use srandom/random. - */ -static void -get_random_bytes(void *buf, int nbytes) -{ - int i, n = nbytes, fd = get_random_fd(); - int lose_counter = 0; - unsigned char *cp = (unsigned char *)buf; - - if (fd >= 0) { - while (n > 0) { - i = read(fd, cp, n); - if (i <= 0) { - if (lose_counter++ > 16) - break; - continue; - } - n -= i; - cp += i; - lose_counter = 0; - } - } - - /* - * We do this all the time, but this is the only source of - * randomness if /dev/random/urandom is out to lunch. - */ - for (cp = buf, i = 0; i < nbytes; i++) - *cp++ ^= (rand() >> 7) & 0xFF; - return; -} - -/* - * Get the ethernet hardware address, if we can find it... - */ -static int -get_node_id(unsigned char *node_id) -{ -#ifdef HAVE_NET_IF_H - int sd; - struct ifreq ifr, *ifrp; - struct ifconf ifc; - char buf[1024]; - int n, i; - unsigned char *a; - -/* - * BSD 4.4 defines the size of an ifreq to be - * max(sizeof(ifreq), sizeof(ifreq.ifr_name)+ifreq.ifr_addr.sa_len - * However, under earlier systems, sa_len isn't present, so the size is - * just sizeof(struct ifreq) - */ -# ifdef HAVE_SA_LEN -# ifndef max -# define max(a,b) ((a) > (b) ? (a) : (b)) -# endif -# define ifreq_size(i) max(sizeof(struct ifreq),\ - sizeof((i).ifr_name)+(i).ifr_addr.sa_len) -# else -# define ifreq_size(i) sizeof(struct ifreq) -# endif /* HAVE_SA_LEN */ - - sd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP); - if (sd < 0) { - return -1; - } - memset(buf, 0, sizeof(buf)); - ifc.ifc_len = sizeof(buf); - ifc.ifc_buf = buf; - if (ioctl(sd, SIOCGIFCONF, (char *)&ifc) < 0) { - close(sd); - return -1; - } - n = ifc.ifc_len; - for (i = 0; i < n; i += ifreq_size(*ifr)) { - ifrp = (struct ifreq *)((char *)ifc.ifc_buf + i); - strncpy(ifr.ifr_name, ifrp->ifr_name, IFNAMSIZ); -# ifdef SIOCGIFHWADDR - if (ioctl(sd, SIOCGIFHWADDR, &ifr) < 0) - continue; - a = (unsigned char *)&ifr.ifr_hwaddr.sa_data; -# else -# ifdef SIOCGENADDR - if (ioctl(sd, SIOCGENADDR, &ifr) < 0) - continue; - a = (unsigned char *)ifr.ifr_enaddr; -# else - /* - * XXX we don't have a way of getting the hardware - * address - */ - close(sd); - return 0; -# endif /* SIOCGENADDR */ -# endif /* SIOCGIFHWADDR */ - if (!a[0] && !a[1] && !a[2] && !a[3] && !a[4] && !a[5]) - continue; - if (node_id) { - memcpy(node_id, a, 6); - close(sd); - return 1; - } - } - close(sd); -#endif - return 0; -} - -/* Assume that the gettimeofday() has microsecond granularity */ -#define MAX_ADJUSTMENT 10 - -static int -get_clock(__u32 * clock_high, __u32 * clock_low, __u16 * ret_clock_seq) -{ - static int adjustment = 0; - static struct timeval last = { 0, 0 }; - static __u16 clock_seq; - struct timeval tv; - unsigned longlong clock_reg; - - try_again: - gettimeofday(&tv, 0); - if ((last.tv_sec == 0) && (last.tv_usec == 0)) { - get_random_bytes(&clock_seq, sizeof(clock_seq)); - clock_seq &= 0x1FFF; - last = tv; - last.tv_sec--; - } - if ((tv.tv_sec < last.tv_sec) || ((tv.tv_sec == last.tv_sec) && (tv.tv_usec < last.tv_usec))) { - clock_seq = (clock_seq + 1) & 0x1FFF; - adjustment = 0; - last = tv; - } else if ((tv.tv_sec == last.tv_sec) && (tv.tv_usec == last.tv_usec)) { - if (adjustment >= MAX_ADJUSTMENT) - goto try_again; - adjustment++; - } else { - adjustment = 0; - last = tv; - } - - clock_reg = tv.tv_usec * 10 + adjustment; - clock_reg += ((unsigned longlong)tv.tv_sec) * 10000000; - clock_reg += (((unsigned longlong)0x01B21DD2) << 32) + 0x13814000; - - *clock_high = clock_reg >> 32; - *clock_low = clock_reg; - *ret_clock_seq = clock_seq; - return 0; -} - -/* create a new uuid, based on randomness */ -void -uuid_generate_random(uuid_t out) -{ - uuid_t buf; - struct uuid uu; - - get_random_bytes(buf, sizeof(buf)); - uuid_unpack(buf, &uu); - - uu.clock_seq = (uu.clock_seq & 0x3FFF) | 0x8000; - uu.time_hi_and_version = (uu.time_hi_and_version & 0x0FFF) | 0x4000; - uuid_pack(&uu, out); -} - -/* create a new uuid, based on time */ -static void -uuid_generate_time(uuid_t out) -{ - static unsigned char node_id[6]; - static int has_init = 0; - struct uuid uu; - __u32 clock_mid; - - if (!has_init) { - if (get_node_id(node_id) <= 0) { - get_random_bytes(node_id, 6); - /* - * Set multicast bit, to prevent conflicts - * with IEEE 802 addresses obtained from - * network cards - */ - node_id[0] |= 0x80; - } - has_init = 1; - } - get_clock(&clock_mid, &uu.time_low, &uu.clock_seq); - uu.clock_seq |= 0x8000; - uu.time_mid = (__u16) clock_mid; - uu.time_hi_and_version = (clock_mid >> 16) | 0x1000; - memcpy(uu.node, node_id, 6); - uuid_pack(&uu, out); -} - -void -uuid_generate(uuid_t out) -{ - if (get_random_fd() >= 0) { - uuid_generate_random(out); - } else { - uuid_generate_time(out); - } -} -- 1.8.3.1 From d57aa84c1ce8b432c2a90615b67564f881d3883f Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Mon, 17 Dec 2018 17:46:06 -0600 Subject: [PATCH 08/69] Build: spec: declare bundled gnulib --- pacemaker.spec.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pacemaker.spec.in b/pacemaker.spec.in index dbc8360..6b2b268 100644 --- a/pacemaker.spec.in +++ b/pacemaker.spec.in @@ -251,6 +251,9 @@ BuildRequires: inkscape asciidoc publican Provides: pcmk-cluster-manager = %{version}-%{release} Provides: pcmk-cluster-manager%{?_isa} = %{version}-%{release} +# Pacemaker uses the crypto/md5 module from gnulib +Provides: bundled(gnulib) + %description Pacemaker is an advanced, scalable High-Availability cluster resource manager. -- 1.8.3.1 From 15814c6c0dc844cac87023ddcebf083a35beef4d Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 13 Dec 2018 10:25:14 -0600 Subject: [PATCH 09/69] Doc: ChangeLog: update for 2.0.1-rc2 release --- ChangeLog | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 875fe5c..23d8307 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,11 @@ +* Wed Dec 19 2018 Ken Gaillot Pacemaker-2.0.1-rc2 +- Changesets: 12 +- Diff: 2 files changed, 6 insertions(+), 2 deletions(-) + +- Changes since Pacemaker-2.0.1-rc1 + + libpe_status: avoid double free of stop_needed list (regression in 2.0.1-rc1) + + tools: stonith_admin -I doesn't require an agent (regression in 2.0.1-rc1) + * Wed Dec 12 2018 Ken Gaillot Pacemaker-2.0.1-rc1 - Changesets: 481 164 files changed, 7717 insertions(+), 4398 deletions(-) @@ -15,6 +23,9 @@ (regression since 1.1.12) + Pacemaker Remote: avoid unnecessary downtime when moving resource to Pacemaker Remote node that fails to come up (regression since 1.1.18) + + scheduler: clone notifications could be scheduled for a stopped + Pacemaker Remote node and block all further cluster actions + (regression since 2.0.0) + tools: crm_resource -C could fail to clean up all failures in one run (regression since 2.0.0) + build: spec file now puts XML schemas in new pacemaker-schemas package @@ -38,7 +49,6 @@ + scheduler: start unique clone instances in numerical order + scheduler: convert unique clones to anonymous clones if not supported by standard + scheduler: associate pending tasks with correct clone instance - + scheduler: don't send clone notifications to a stopped remote node + scheduler: ensure bundle clone notifications are directed to correct host + scheduler: avoid improper monitor rescheduling or fail count clearing for bundles + scheduler: honor asymmetric orderings even when restarting -- 1.8.3.1 From 74262eb5602bbb4d6406fc0a92766d5fdae9b303 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 20 Dec 2018 10:48:10 -0600 Subject: [PATCH 10/69] Refactor: controller: remove dead code --- daemons/controld/controld_fsa.h | 15 --------------- daemons/controld/controld_messages.h | 2 -- 2 files changed, 17 deletions(-) diff --git a/daemons/controld/controld_fsa.h b/daemons/controld/controld_fsa.h index 1da88ff..a1f4dfd 100644 --- a/daemons/controld/controld_fsa.h +++ b/daemons/controld/controld_fsa.h @@ -530,11 +530,6 @@ void do_pe_invoke(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data); -/* A_ERROR */ -void do_error(long long action, enum crmd_fsa_cause cause, - enum crmd_fsa_state cur_state, - enum crmd_fsa_input cur_input, fsa_data_t *msg_data); - /* A_LOG */ void do_log(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, @@ -665,11 +660,6 @@ void do_cl_join_finalize_respond(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_input current_input, fsa_data_t *msg_data); -/* A_UPDATE_NODESTATUS */ -void do_update_node_status(long long action, enum crmd_fsa_cause cause, - enum crmd_fsa_state cur_state, - enum crmd_fsa_input cur_input, fsa_data_t *msg_data); - /* A_LRM_INVOKE */ void do_lrm_invoke(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, @@ -685,11 +675,6 @@ void do_te_invoke(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t *msg_data); -/* A_TE_INVOKE */ -void do_te_copyto(long long action, enum crmd_fsa_cause cause, - enum crmd_fsa_state cur_state, - enum crmd_fsa_input cur_input, fsa_data_t *msg_data); - /* A_SHUTDOWN_REQ */ void do_shutdown_req(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, diff --git a/daemons/controld/controld_messages.h b/daemons/controld/controld_messages.h index 8c80b5c..4c7e777 100644 --- a/daemons/controld/controld_messages.h +++ b/daemons/controld/controld_messages.h @@ -74,8 +74,6 @@ gboolean is_message(void); extern gboolean relay_message(xmlNode * relay_message, gboolean originated_locally); -extern void process_message(xmlNode * msg, gboolean originated_locally, const char *src_node_name); - extern gboolean send_msg_via_ipc(xmlNode * msg, const char *sys); gboolean crmd_is_proxy_session(const char *session); -- 1.8.3.1 From 005d76b7038f94d7ed4067a1e7c936baac2f0aba Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 20 Dec 2018 10:51:06 -0600 Subject: [PATCH 11/69] Refactor: libstonithd: make function declaration C++-compatible just an argument name, so still backward-compatible --- include/crm/stonith-ng.h | 2 +- lib/fencing/st_client.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/crm/stonith-ng.h b/include/crm/stonith-ng.h index aa225e2..62a72d9 100644 --- a/include/crm/stonith-ng.h +++ b/include/crm/stonith-ng.h @@ -83,7 +83,7 @@ enum stonith_namespace { }; enum stonith_namespace stonith_text2namespace(const char *namespace_s); -const char *stonith_namespace2text(enum stonith_namespace namespace); +const char *stonith_namespace2text(enum stonith_namespace st_namespace); enum stonith_namespace stonith_get_namespace(const char *agent, const char *namespace_s); diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c index 456bc80..9376105 100644 --- a/lib/fencing/st_client.c +++ b/lib/fencing/st_client.c @@ -152,9 +152,9 @@ stonith_text2namespace(const char *namespace_s) * \return Namespace name as string */ const char * -stonith_namespace2text(enum stonith_namespace namespace) +stonith_namespace2text(enum stonith_namespace st_namespace) { - switch (namespace) { + switch (st_namespace) { case st_namespace_any: return "any"; case st_namespace_rhcs: return "stonith-ng"; case st_namespace_internal: return "internal"; -- 1.8.3.1 From 1a83b5834b20fea1ff1d509659b13776b75a167f Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Fri, 21 Dec 2018 11:55:07 -0600 Subject: [PATCH 12/69] Low: libpe_status: avoid use-after-free when logging at trace level --- lib/pengine/status.c | 59 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/lib/pengine/status.c b/lib/pengine/status.c index dcbdc16..0684f52 100644 --- a/lib/pengine/status.c +++ b/lib/pengine/status.c @@ -125,6 +125,17 @@ cluster_status(pe_working_set_t * data_set) return TRUE; } +/*! + * \internal + * \brief Free a list of pe_resource_t + * + * \param[in] resources List to free + * + * \note When a working set's resource list is freed, that includes the original + * storage for the uname and id of any Pacemaker Remote nodes in the + * working set's node list, so take care not to use those afterward. + * \todo Refactor pe_node_t to strdup() the node name. + */ static void pe_free_resources(GListPtr resources) { @@ -158,32 +169,36 @@ pe_free_actions(GListPtr actions) static void pe_free_nodes(GListPtr nodes) { - GListPtr iterator = nodes; + for (GList *iterator = nodes; iterator != NULL; iterator = iterator->next) { + pe_node_t *node = (pe_node_t *) iterator->data; - while (iterator != NULL) { - node_t *node = (node_t *) iterator->data; - struct pe_node_shared_s *details = node->details; + // Shouldn't be possible, but to be safe ... + if (node == NULL) { + continue; + } + if (node->details == NULL) { + free(node); + continue; + } - iterator = iterator->next; + /* This is called after pe_free_resources(), which means that we can't + * use node->details->uname for Pacemaker Remote nodes. + */ + crm_trace("Freeing node %s", (is_remote_node(node)? + "(Pacemaker Remote)" : node->details->uname)); - crm_trace("deleting node"); - print_node("delete", node, FALSE); - - if (details != NULL) { - crm_trace("%s is being deleted", details->uname); - if (details->attrs != NULL) { - g_hash_table_destroy(details->attrs); - } - if (details->utilization != NULL) { - g_hash_table_destroy(details->utilization); - } - if (details->digest_cache != NULL) { - g_hash_table_destroy(details->digest_cache); - } - g_list_free(details->running_rsc); - g_list_free(details->allocated_rsc); - free(details); + if (node->details->attrs != NULL) { + g_hash_table_destroy(node->details->attrs); + } + if (node->details->utilization != NULL) { + g_hash_table_destroy(node->details->utilization); + } + if (node->details->digest_cache != NULL) { + g_hash_table_destroy(node->details->digest_cache); } + g_list_free(node->details->running_rsc); + g_list_free(node->details->allocated_rsc); + free(node->details); free(node); } if (nodes != NULL) { -- 1.8.3.1 From 847ff91992ad1e657bfda7b3f4e7446e0e91b398 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Fri, 21 Dec 2018 14:46:59 -0600 Subject: [PATCH 13/69] Log: libpe_status: improve trace messages when finding actions Previously, "Node mismatch" would be logged for matches. --- lib/pengine/utils.c | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c index 924085b..8b44c8b 100644 --- a/lib/pengine/utils.c +++ b/lib/pengine/utils.c @@ -1465,27 +1465,35 @@ find_actions(GListPtr input, const char *key, const node_t *on_node) GListPtr find_actions_exact(GListPtr input, const char *key, node_t * on_node) { - GListPtr gIter = input; - GListPtr result = NULL; + GList *result = NULL; CRM_CHECK(key != NULL, return NULL); - for (; gIter != NULL; gIter = gIter->next) { - action_t *action = (action_t *) gIter->data; + if (on_node == NULL) { + crm_trace("Not searching for action %s because node not specified", + key); + return NULL; + } - crm_trace("Matching %s against %s", key, action->uuid); - if (safe_str_neq(key, action->uuid)) { - crm_trace("Key mismatch: %s vs. %s", key, action->uuid); - continue; + for (GList *gIter = input; gIter != NULL; gIter = gIter->next) { + pe_action_t *action = (pe_action_t *) gIter->data; - } else if (on_node == NULL || action->node == NULL) { - crm_trace("on_node=%p, action->node=%p", on_node, action->node); - continue; + if (action->node == NULL) { + crm_trace("Skipping comparison of %s vs action %s without node", + key, action->uuid); - } else if (safe_str_eq(on_node->details->id, action->node->details->id)) { + } else if (safe_str_neq(key, action->uuid)) { + crm_trace("Desired action %s doesn't match %s", key, action->uuid); + + } else if (safe_str_neq(on_node->details->id, + action->node->details->id)) { + crm_trace("Action %s desired node ID %s doesn't match %s", + key, on_node->details->id, action->node->details->id); + + } else { + crm_trace("Action %s matches", key); result = g_list_prepend(result, action); } - crm_trace("Node mismatch: %s vs. %s", on_node->details->id, action->node->details->id); } return result; -- 1.8.3.1 From 61ecbc3e059e84691dc23908fd9d5663ba828bb7 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Fri, 21 Dec 2018 16:34:44 -0600 Subject: [PATCH 14/69] Log: libpe_status: downgrade remote node fence-before-clear message Previously, check_operation_expiry() would order remote node fencing before clearing of the connection fail count (and log a notice) when the remote node was unclean. However, at the point that is called, the nodes are always unclean (link_rsc2remotenode() sets unclean after unpacking resources, and unpack_node_loop() won't clear it until after unpacking status). Now, skip the unclean check (it is always true, and not completely necessary since the fence op is created as optional), and lower the log message to info. --- lib/pengine/unpack.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c index a445442..77fd79d 100644 --- a/lib/pengine/unpack.c +++ b/lib/pengine/unpack.c @@ -2932,19 +2932,31 @@ static bool check_operation_expiry(resource_t *rsc, node_t *node, int rc, xmlNod } if (clear_reason != NULL) { - node_t *remote_node = pe_find_node(data_set->nodes, rsc->id); + // Schedule clearing of the fail count pe_action_t *clear_op = pe__clear_failcount(rsc, node, clear_reason, data_set); if (is_set(data_set->flags, pe_flag_stonith_enabled) - && rsc->remote_reconnect_ms - && remote_node - && remote_node->details->unclean) { + && rsc->remote_reconnect_ms) { - action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL, data_set); - crm_notice("Waiting for %s to complete before clearing %s failure for remote node %s", fence?fence->uuid:"nil", task, rsc->id); + pe_node_t *remote_node = pe_find_node(data_set->nodes, rsc->id); - order_actions(fence, clear_op, pe_order_implies_then); + if (remote_node) { + /* If we're clearing a remote connection due to a reconnect + * interval, we want to wait until any scheduled fencing + * completes. + * + * We could limit this to remote_node->details->unclean, but at + * this point, that's always true (it won't be reliable until + * after unpack_node_loop() is done). + */ + pe_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL, + data_set); + + crm_info("Clearing %s failure will wait until any scheduled " + "fencing of %s completes", task, rsc->id); + order_actions(fence, clear_op, pe_order_implies_then); + } } } -- 1.8.3.1 From 3ecfacd6d0b2f26769058f627f0c0df28af13a6c Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 3 Jan 2019 10:40:32 -0600 Subject: [PATCH 15/69] Log: scheduler: downgrade clone pre-allocation message When allocating clone instances, we pre-allocate instances to their existing location if possible. If not, we would previously log a notice about "Pre-allocation failed", which might make this seem more significant than it is (it's normal when the instance is moving). So, downgrade it to info, and don't say it "failed". This also includes some trivial refactoring for efficiency and clarity. --- daemons/schedulerd/sched_clone.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/daemons/schedulerd/sched_clone.c b/daemons/schedulerd/sched_clone.c index 9b42a83..8f01f56 100644 --- a/daemons/schedulerd/sched_clone.c +++ b/daemons/schedulerd/sched_clone.c @@ -442,18 +442,19 @@ color_instance(resource_t * rsc, node_t * prefer, gboolean all_coloc, int limit, backup = node_hash_dup(rsc->allowed_nodes); chosen = rsc->cmds->allocate(rsc, prefer, data_set); + if (chosen && prefer && (chosen->details != prefer->details)) { + crm_info("Not pre-allocating %s to %s because %s is better", + rsc->id, prefer->details->uname, chosen->details->uname); + g_hash_table_destroy(rsc->allowed_nodes); + rsc->allowed_nodes = backup; + native_deallocate(rsc); + chosen = NULL; + backup = NULL; + } if (chosen) { - node_t *local_node = parent_node_instance(rsc, chosen); - if (prefer && (chosen->details != prefer->details)) { - crm_notice("Pre-allocation failed: got %s instead of %s", - chosen->details->uname, prefer->details->uname); - g_hash_table_destroy(rsc->allowed_nodes); - rsc->allowed_nodes = backup; - native_deallocate(rsc); - chosen = NULL; - backup = NULL; - - } else if (local_node) { + pe_node_t *local_node = parent_node_instance(rsc, chosen); + + if (local_node) { local_node->count++; } else if (is_set(rsc->flags, pe_rsc_managed)) { -- 1.8.3.1 From 6678aa6cc2c3d515f851002b725e2f96ce8ce634 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 3 Jan 2019 10:24:35 -0600 Subject: [PATCH 16/69] Build: move project maintenance helpers to new subdirectory so their purpose is clear --- GNUmakefile | 1 - bumplibs.sh | 128 ---------------------------------- maint/README | 2 + maint/bumplibs.sh | 128 ++++++++++++++++++++++++++++++++++ maint/travisci_build_coverity_scan.sh | 102 +++++++++++++++++++++++++++ 8 files changed, 234 insertions(+), 233 deletions(-) delete mode 100755 bumplibs.sh create mode 100644 maint/README create mode 100755 maint/bumplibs.sh create mode 100644 maint/travisci_build_coverity_scan.sh diff --git a/GNUmakefile b/GNUmakefile index 8e5bdd7..60de623 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -347,7 +347,6 @@ changelog: @make changes > ChangeLog @printf "\n">> ChangeLog git show $(LAST_RELEASE):ChangeLog >> ChangeLog - @echo -e "\033[1;35m -- Don't forget to run the bumplibs.sh script! --\033[0m" DO_NOT_INDENT = lib/gnu daemons/controld/controld_fsa.h diff --git a/bumplibs.sh b/bumplibs.sh deleted file mode 100755 index 2044efa..0000000 --- a/bumplibs.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/bash - -declare -A headers -headers[crmcommon]="include/crm/common include/crm/crm.h" -headers[crmcluster]="include/crm/cluster.h" -headers[crmservice]="include/crm/services.h" -headers[transitioner]="include/crm/transition.h" -headers[cib]="include/crm/cib.h include/crm/cib/util.h" -headers[pe_rules]="include/crm/pengine/rules.h" -headers[pe_status]="include/crm/pengine/common.h include/crm/pengine/complex.h include/crm/pengine/rules.h include/crm/pengine/status.h" -headers[pengine]="include/crm/pengine/common.h include/crm/pengine/complex.h include/crm/pengine/rules.h include/crm/pengine/status.h" -headers[stonithd]="include/crm/stonith-ng.h" -headers[lrmd]="include/crm/lrmd.h" - -if [ ! -z $1 ]; then - LAST_RELEASE=$1 -else - LAST_RELEASE=`test -e /Volumes || git tag -l | grep Pacemaker | grep -v rc | sort -Vr | head -n 1` -fi -libs=$(find . -name "*.am" -exec grep "lib.*_la_LDFLAGS.*version-info" \{\} \; | sed -e s/_la_LDFLAGS.*// -e s/^lib//) -for lib in $libs; do - if [ -z "${headers[$lib]}" ]; then - echo "Unknown headers for lib$lib" - exit 0 - fi - git diff -w $LAST_RELEASE..HEAD ${headers[$lib]} - echo "" - - am=`find . -name Makefile.am -exec grep -lr "lib${lib}_la.*version-info" \{\} \;` - am_dir=`dirname $am` - - if - grep "lib${lib}_la_SOURCES.*\\\\" $am - then - echo -e "\033[1;35m -- Sources list for lib$lib is probably truncated! --\033[0m" - echo "" - fi - - sources=`grep "lib${lib}_la_SOURCES" $am | sed s/.*=// | sed 's:$(top_builddir)/::' | sed 's:$(top_srcdir)/::' | sed 's:\\\::' | sed 's:$(libpe_rules_la_SOURCES):rules.c\ common.c:'` - - full_sources="" - for f in $sources; do - if - echo $f | grep -q "/" - then - full_sources="$full_sources $f" - else - full_sources="$full_sources $am_dir/$f" - fi - done - - lines=`git diff -w $LAST_RELEASE..HEAD ${headers[$lib]} $full_sources | wc -l` - - if [ $lines -gt 0 ]; then - echo "- Headers: ${headers[$lib]}" - echo "- Sources: $full_sources" - echo "- Changed Sources since $LAST_RELEASE:" - git diff -w $LAST_RELEASE..HEAD --stat $full_sources - echo "" - echo "New arguments to functions or changes to the middle of structs are incompatible additions" - echo "" - echo "Where possible:" - echo "- move new fields to the end of structs" - echo "- use bitfields instead of booleans" - echo "- when adding arguments, create new functions that the old version can call" - echo "" - read -p "Are the changes to lib$lib: [a]dditions, [i]ncompatible additions, [r]emovals or [f]ixes? [None]: " CHANGE - - git show $LAST_RELEASE:$am | grep version-info - VER=`git show $LAST_RELEASE:$am | grep "lib.*${lib}_la.*version-info" | sed s/.*version-info// | awk '{print $1}'` - VER_NOW=`grep "lib.*${lib}_la.*version-info" $am | sed s/.*version-info// | awk '{print $1}'` - VER_1=`echo $VER | awk -F: '{print $1}'` - VER_2=`echo $VER | awk -F: '{print $2}'` - VER_3=`echo $VER | awk -F: '{print $3}'` - VER_1_NOW=`echo $VER_NOW | awk -F: '{print $1}'` - - case $CHANGE in - i|I) - echo "New version with incompatible extensions: x+1:0:0" - VER_1=`expr $VER_1 + 1` - VER_2=0 - VER_3=0 - for h in ${headers[$lib]}; do - sed -i.sed "s/lib${lib}.so.${VER_1_NOW}/lib${lib}.so.${VER_1}/" $h - done - ;; - a|A) - echo "New version with backwards compatible extensions: x+1:0:z+1" - VER_1=`expr $VER_1 + 1` - VER_2=0 - VER_3=`expr $VER_3 + 1` - ;; - R|r) - echo "New backwards incompatible version: x+1:0:0" - VER_1=`expr $VER_1 + 1` - VER_2=0 - VER_3=0 - for h in ${headers[$lib]}; do - sed -i.sed "s/lib${lib}.so.${VER_1_NOW}/lib${lib}.so.${VER_1}/" $h - done - ;; - F|f) - echo "Bugfix: x:y+1:z" - VER_2=`expr $VER_2 + 1` - ;; - esac - VER_NEW=$VER_1:$VER_2:$VER_3 - - if [ ! -z $CHANGE ]; then - if [ $VER_NEW != $VER_NOW ]; then - echo "Updating $lib library version: $VER -> $VER_NEW" - sed -i.sed "s/version-info\ $VER_NOW/version-info\ $VER_NEW/" $am - else - echo "No further version changes needed" - sed -i.sed "s/version-info\ $VER_NOW/version-info\ $VER_NEW/" $am - fi - else - echo "Skipping $lib version" - fi - else - echo "No changes to $lib interface" - fi - - read -p "Continue?" - echo "" -done - -git diff -w diff --git a/maint/README b/maint/README new file mode 100644 index 0000000..738e7db --- /dev/null +++ b/maint/README @@ -0,0 +1,2 @@ +This directory contains helpers for the project maintainers. +Everyone else can safely ignore it. diff --git a/maint/bumplibs.sh b/maint/bumplibs.sh new file mode 100755 index 0000000..2044efa --- /dev/null +++ b/maint/bumplibs.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +declare -A headers +headers[crmcommon]="include/crm/common include/crm/crm.h" +headers[crmcluster]="include/crm/cluster.h" +headers[crmservice]="include/crm/services.h" +headers[transitioner]="include/crm/transition.h" +headers[cib]="include/crm/cib.h include/crm/cib/util.h" +headers[pe_rules]="include/crm/pengine/rules.h" +headers[pe_status]="include/crm/pengine/common.h include/crm/pengine/complex.h include/crm/pengine/rules.h include/crm/pengine/status.h" +headers[pengine]="include/crm/pengine/common.h include/crm/pengine/complex.h include/crm/pengine/rules.h include/crm/pengine/status.h" +headers[stonithd]="include/crm/stonith-ng.h" +headers[lrmd]="include/crm/lrmd.h" + +if [ ! -z $1 ]; then + LAST_RELEASE=$1 +else + LAST_RELEASE=`test -e /Volumes || git tag -l | grep Pacemaker | grep -v rc | sort -Vr | head -n 1` +fi +libs=$(find . -name "*.am" -exec grep "lib.*_la_LDFLAGS.*version-info" \{\} \; | sed -e s/_la_LDFLAGS.*// -e s/^lib//) +for lib in $libs; do + if [ -z "${headers[$lib]}" ]; then + echo "Unknown headers for lib$lib" + exit 0 + fi + git diff -w $LAST_RELEASE..HEAD ${headers[$lib]} + echo "" + + am=`find . -name Makefile.am -exec grep -lr "lib${lib}_la.*version-info" \{\} \;` + am_dir=`dirname $am` + + if + grep "lib${lib}_la_SOURCES.*\\\\" $am + then + echo -e "\033[1;35m -- Sources list for lib$lib is probably truncated! --\033[0m" + echo "" + fi + + sources=`grep "lib${lib}_la_SOURCES" $am | sed s/.*=// | sed 's:$(top_builddir)/::' | sed 's:$(top_srcdir)/::' | sed 's:\\\::' | sed 's:$(libpe_rules_la_SOURCES):rules.c\ common.c:'` + + full_sources="" + for f in $sources; do + if + echo $f | grep -q "/" + then + full_sources="$full_sources $f" + else + full_sources="$full_sources $am_dir/$f" + fi + done + + lines=`git diff -w $LAST_RELEASE..HEAD ${headers[$lib]} $full_sources | wc -l` + + if [ $lines -gt 0 ]; then + echo "- Headers: ${headers[$lib]}" + echo "- Sources: $full_sources" + echo "- Changed Sources since $LAST_RELEASE:" + git diff -w $LAST_RELEASE..HEAD --stat $full_sources + echo "" + echo "New arguments to functions or changes to the middle of structs are incompatible additions" + echo "" + echo "Where possible:" + echo "- move new fields to the end of structs" + echo "- use bitfields instead of booleans" + echo "- when adding arguments, create new functions that the old version can call" + echo "" + read -p "Are the changes to lib$lib: [a]dditions, [i]ncompatible additions, [r]emovals or [f]ixes? [None]: " CHANGE + + git show $LAST_RELEASE:$am | grep version-info + VER=`git show $LAST_RELEASE:$am | grep "lib.*${lib}_la.*version-info" | sed s/.*version-info// | awk '{print $1}'` + VER_NOW=`grep "lib.*${lib}_la.*version-info" $am | sed s/.*version-info// | awk '{print $1}'` + VER_1=`echo $VER | awk -F: '{print $1}'` + VER_2=`echo $VER | awk -F: '{print $2}'` + VER_3=`echo $VER | awk -F: '{print $3}'` + VER_1_NOW=`echo $VER_NOW | awk -F: '{print $1}'` + + case $CHANGE in + i|I) + echo "New version with incompatible extensions: x+1:0:0" + VER_1=`expr $VER_1 + 1` + VER_2=0 + VER_3=0 + for h in ${headers[$lib]}; do + sed -i.sed "s/lib${lib}.so.${VER_1_NOW}/lib${lib}.so.${VER_1}/" $h + done + ;; + a|A) + echo "New version with backwards compatible extensions: x+1:0:z+1" + VER_1=`expr $VER_1 + 1` + VER_2=0 + VER_3=`expr $VER_3 + 1` + ;; + R|r) + echo "New backwards incompatible version: x+1:0:0" + VER_1=`expr $VER_1 + 1` + VER_2=0 + VER_3=0 + for h in ${headers[$lib]}; do + sed -i.sed "s/lib${lib}.so.${VER_1_NOW}/lib${lib}.so.${VER_1}/" $h + done + ;; + F|f) + echo "Bugfix: x:y+1:z" + VER_2=`expr $VER_2 + 1` + ;; + esac + VER_NEW=$VER_1:$VER_2:$VER_3 + + if [ ! -z $CHANGE ]; then + if [ $VER_NEW != $VER_NOW ]; then + echo "Updating $lib library version: $VER -> $VER_NEW" + sed -i.sed "s/version-info\ $VER_NOW/version-info\ $VER_NEW/" $am + else + echo "No further version changes needed" + sed -i.sed "s/version-info\ $VER_NOW/version-info\ $VER_NEW/" $am + fi + else + echo "Skipping $lib version" + fi + else + echo "No changes to $lib interface" + fi + + read -p "Continue?" + echo "" +done + +git diff -w diff --git a/maint/travisci_build_coverity_scan.sh b/maint/travisci_build_coverity_scan.sh new file mode 100644 index 0000000..b8b24d2 --- /dev/null +++ b/maint/travisci_build_coverity_scan.sh @@ -0,0 +1,102 @@ +#!/bin/sh + +set -e + +export RED="\033[33;1m" +export NONE="\033[0m" + +if [ -z "$PROJECT_NAME" ]; then + PROJECT_NAME=${TRAVIS_REPO_SLUG} +fi + +# Environment check +echo -e "${RED}Note: PROJECT_NAME and COVERITY_SCAN_TOKEN are available on Project Settings page on scan.coverity.com${NONE}" +[ -z "$PROJECT_NAME" ] && echo "ERROR: PROJECT_NAME must be set" && exit 1 +[ -z "$OWNER_EMAIL" ] && echo "ERROR: OWNER_EMAIL must be set" && exit 1 +[ -z "$COVERITY_SCAN_BRANCH_PATTERN" ] && echo "ERROR: COVERITY_SCAN_BRANCH_PATTERN must be set" && exit 1 +[ -z "$COVERITY_SCAN_BUILD_COMMAND" ] && echo "ERROR: COVERITY_SCAN_BUILD_COMMAND must be set" && exit 1 + +PLATFORM=`uname` +TOOL_ARCHIVE=/tmp/cov-analysis-${PLATFORM}.tgz +TOOL_URL=https://scan.coverity.com/download/${PLATFORM} +TOOL_BASE=/tmp/coverity-scan-analysis +UPLOAD_URL="http://scan5.coverity.com/cgi-bin/upload.py" +SCAN_URL="https://scan.coverity.com" + +# Do not run on pull requests +if [ "${TRAVIS_PULL_REQUEST}" = "true" ]; then + echo -e "${RED}INFO: Skipping Coverity Analysis: branch is a pull request.${NONE}" + exit 0 +fi + +# Verify this branch should run +IS_COVERITY_SCAN_BRANCH=`ruby -e "puts '${TRAVIS_BRANCH}' =~ /\\A$COVERITY_SCAN_BRANCH_PATTERN\\z/ ? 1 : 0"` +if [ "$IS_COVERITY_SCAN_BRANCH" = "1" ]; then + echo -e "${RED}Coverity Scan configured to run on branch ${TRAVIS_BRANCH}${NONE}" +else + echo -e "${RED}Coverity Scan NOT configured to run on branch ${TRAVIS_BRANCH}${NONE}" + exit 0 # Nothing to do, exit with success otherwise the build will be considered failed +fi + +# If COVERITY_SCAN_TOKEN isn't set, then we're probably running from somewhere +# other than ClusterLabs/pacemaker and coverity shouldn't be running anyway +[ -z "$COVERITY_SCAN_TOKEN" ] && echo "${RED}ERROR: COVERITY_SCAN_TOKEN must be set${NONE}" && exit 0 + +# Verify upload is permitted +AUTH_RES=`curl -s --form project="$PROJECT_NAME" --form token="$COVERITY_SCAN_TOKEN" $SCAN_URL/api/upload_permitted` +if [ "$AUTH_RES" = "Access denied" ]; then + echo -e "${RED}Coverity Scan API access denied. Check PROJECT_NAME and COVERITY_SCAN_TOKEN.${NONE}" + exit 1 +else + AUTH=`echo $AUTH_RES | ruby -e "require 'rubygems'; require 'json'; puts JSON[STDIN.read]['upload_permitted']"` + if [ "$AUTH" = "true" ]; then + echo -e "${RED}Coverity Scan analysis authorized per quota.${NONE}" + else + WHEN=`echo $AUTH_RES | ruby -e "require 'rubygems'; require 'json'; puts JSON[STDIN.read]['next_upload_permitted_at']"` + echo -e "${RED}Coverity Scan analysis NOT authorized until $WHEN.${NONE}" + exit 1 + fi +fi + +if [ ! -d $TOOL_BASE ]; then + # Download Coverity Scan Analysis Tool + if [ ! -e $TOOL_ARCHIVE ]; then + echo -e "${RED}Downloading Coverity Scan Analysis Tool...${NONE}" + wget -nv -O $TOOL_ARCHIVE $TOOL_URL --post-data "project=$PROJECT_NAME&token=$COVERITY_SCAN_TOKEN" + fi + + # Extract Coverity Scan Analysis Tool + echo -e "${RED}Extracting Coverity Scan Analysis Tool...${NONE}" + mkdir -p $TOOL_BASE + pushd $TOOL_BASE + tar xzf $TOOL_ARCHIVE + popd +fi + +TOOL_DIR=`find $TOOL_BASE -type d -name 'cov-analysis*'` +export PATH=$TOOL_DIR/bin:$PATH + +# Build +echo -e "${RED}Running Coverity Scan Analysis Tool...${NONE}" +COV_BUILD_OPTIONS="" +#COV_BUILD_OPTIONS="--return-emit-failures 8 --parse-error-threshold 85" +RESULTS_DIR="cov-int" +eval "${COVERITY_SCAN_BUILD_COMMAND_PREPEND}" +COVERITY_UNSUPPORTED=1 cov-build --dir $RESULTS_DIR $COV_BUILD_OPTIONS $COVERITY_SCAN_BUILD_COMMAND + +# Upload results +echo -e "${RED}Tarring Coverity Scan Analysis results...${NONE}" +RESULTS_ARCHIVE=analysis-results.tgz +tar czf $RESULTS_ARCHIVE $RESULTS_DIR +SHA=`git rev-parse --short HEAD` + +echo -e "${RED}Uploading Coverity Scan Analysis results...${NONE}" +curl \ + --progress-bar \ + --form project=$PROJECT_NAME \ + --form token=$COVERITY_SCAN_TOKEN \ + --form email=$OWNER_EMAIL \ + --form file=@$RESULTS_ARCHIVE \ + --form version=$SHA \ + --form description="Travis CI build" \ + $UPLOAD_URL -- 1.8.3.1 From d63ced718d0d9f28f711a234307ec534cb76d7fa Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 20 Dec 2018 16:26:01 -0600 Subject: [PATCH 17/69] Doc: ChangeLog: podman support was added in 2.0.1-rc1 --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 23d8307..e348333 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,7 @@ 164 files changed, 7717 insertions(+), 4398 deletions(-) - Features added since Pacemaker-2.0.0 + + Pacemaker bundles now support podman container management + fencing: SBD may now be used in a cluster that has guest nodes or bundles + fencing: synchronize fencing history among all nodes + fencing: stonith_admin now has option to clear fence history -- 1.8.3.1 From 8b58ea39391e876647ff76d81ac55d05b7cd8880 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 2 Oct 2018 15:26:23 -0500 Subject: [PATCH 18/69] Low: attrd: connect to the CIB before connecting the cluster This avoids a (highly unlikely) race: If the local node wins a writer election and receives a peer update at start-up, between the cluster connection and the CIB connection, we could delay the write due to "cib not connected". However, we would not re-attempt this write once the connection was established, so the value might never be written out (or delayed an unreasonably long time). Rearranging the connections is a simple solution, and also allows us to assume the CIB is connected everywhere else. --- daemons/attrd/attrd_alerts.c | 23 ++++++++---------- daemons/attrd/attrd_commands.c | 7 ++---- daemons/attrd/attrd_utils.c | 9 +++---- daemons/attrd/pacemaker-attrd.c | 54 ++++++++++++++++++++++++++--------------- 4 files changed, 50 insertions(+), 43 deletions(-) diff --git a/daemons/attrd/attrd_alerts.c b/daemons/attrd/attrd_alerts.c index 377e27a..e63d635 100644 --- a/daemons/attrd/attrd_alerts.c +++ b/daemons/attrd/attrd_alerts.c @@ -111,19 +111,16 @@ attrd_read_options(gpointer user_data) { int call_id; - if (the_cib) { - call_id = the_cib->cmds->query(the_cib, XPATH_ALERTS, NULL, - cib_xpath | cib_scope_local); - - the_cib->cmds->register_callback_full(the_cib, call_id, 120, FALSE, - NULL, - "config_query_callback", - config_query_callback, free); - - crm_trace("Querying the CIB... call %d", call_id); - } else { - crm_err("Could not check for alerts configuration: CIB connection not active"); - } + CRM_CHECK(the_cib != NULL, return TRUE); + + call_id = the_cib->cmds->query(the_cib, XPATH_ALERTS, NULL, + cib_xpath | cib_scope_local); + + the_cib->cmds->register_callback_full(the_cib, call_id, 120, FALSE, NULL, + "config_query_callback", + config_query_callback, free); + + crm_trace("Querying the CIB... call %d", call_id); return TRUE; } diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c index f07d503..1cd7aaf 100644 --- a/daemons/attrd/attrd_commands.c +++ b/daemons/attrd/attrd_commands.c @@ -1163,11 +1163,8 @@ write_attribute(attribute_t *a, bool ignore_delay) if (!a->is_private) { /* Defer the write if now's not a good time */ - if (the_cib == NULL) { - crm_info("Write out of '%s' delayed: cib not connected", a->id); - return; - - } else if (a->update && (a->update < last_cib_op_done)) { + CRM_CHECK(the_cib != NULL, return); + if (a->update && (a->update < last_cib_op_done)) { crm_info("Write out of '%s' continuing: update %d considered lost", a->id, a->update); } else if (a->update) { diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c index 9ce784c..8245af6 100644 --- a/daemons/attrd/attrd_utils.c +++ b/daemons/attrd/attrd_utils.c @@ -175,11 +175,10 @@ attrd_init_ipc(qb_ipcs_service_t **ipcs, qb_ipcs_msg_process_fn dispatch_fn) void attrd_cib_disconnect() { - if (the_cib) { - the_cib->cmds->signoff(the_cib); - cib_delete(the_cib); - the_cib = NULL; - } + CRM_CHECK(the_cib != NULL, return); + the_cib->cmds->signoff(the_cib); + cib_delete(the_cib); + the_cib = NULL; } /* strlen("value") */ diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c index d96e666..69d02c0 100644 --- a/daemons/attrd/pacemaker-attrd.c +++ b/daemons/attrd/pacemaker-attrd.c @@ -198,6 +198,22 @@ attrd_cib_connect(int max_retry) goto cleanup; } + return pcmk_ok; + + cleanup: + the_cib->cmds->signoff(the_cib); + cib_delete(the_cib); + the_cib = NULL; + return -ENOTCONN; +} + +/*! + * \internal + * \brief Prepare the CIB after cluster is connected + */ +static void +attrd_cib_init() +{ // We have no attribute values in memory, wipe the CIB to match attrd_erase_attrs(); @@ -206,21 +222,6 @@ attrd_cib_connect(int max_retry) // Always read the CIB at start-up mainloop_set_trigger(attrd_config_read); - - /* Set a private attribute for ourselves with the protocol version we - * support. This lets all nodes determine the minimum supported version - * across all nodes. It also ensures that the writer learns our node name, - * so it can send our attributes to the CIB. - */ - attrd_broadcast_protocol(); - - return pcmk_ok; - - cleanup: - the_cib->cmds->signoff(the_cib); - cib_delete(the_cib); - the_cib = NULL; - return -ENOTCONN; } static int32_t @@ -361,19 +362,32 @@ main(int argc, char **argv) crm_info("Starting up"); attributes = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_attribute); + /* Connect to the CIB before connecting to the cluster or listening for IPC. + * This allows us to assume the CIB is connected whenever we process a + * cluster or IPC message (which also avoids start-up race conditions). + */ + if (attrd_cib_connect(10) != pcmk_ok) { + attrd_exit_status = CRM_EX_FATAL; + goto done; + } + crm_info("CIB connection active"); + if (attrd_cluster_connect() != pcmk_ok) { attrd_exit_status = CRM_EX_FATAL; goto done; } crm_info("Cluster connection active"); + // Initialization that requires the cluster to be connected attrd_election_init(); + attrd_cib_init(); - if (attrd_cib_connect(10) != pcmk_ok) { - attrd_exit_status = CRM_EX_FATAL; - goto done; - } - crm_info("CIB connection active"); + /* Set a private attribute for ourselves with the protocol version we + * support. This lets all nodes determine the minimum supported version + * across all nodes. It also ensures that the writer learns our node name, + * so it can send our attributes to the CIB. + */ + attrd_broadcast_protocol(); attrd_init_ipc(&ipcs, attrd_ipc_dispatch); crm_info("Accepting attribute updates"); -- 1.8.3.1 From 61ae92650c4c3065e5a7dc92e4d1c806d3f24f9e Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Wed, 3 Oct 2018 16:50:12 -0500 Subject: [PATCH 19/69] Low: attrd: don't delay re-attempted writes unless original failed 73e5b6d3 was too aggressive in delaying re-attempted writes; a write could be re-attempted because the value changed since the original write was submitted. Limit the delay to cases where the original write failed. Also, lower a trivial debug log to trace. --- daemons/attrd/attrd_commands.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c index 1cd7aaf..fa0459a 100644 --- a/daemons/attrd/attrd_commands.c +++ b/daemons/attrd/attrd_commands.c @@ -1018,7 +1018,13 @@ attrd_cib_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *u } if (a->changed && attrd_election_won()) { - /* If we're re-attempting a write because the original failed, delay + if (rc == pcmk_ok) { + /* We deferred a write of a new update because this update was in + * progress. Write out the new value without additional delay. + */ + write_attribute(a, FALSE); + + /* We're re-attempting a write because the original failed; delay * the next attempt so we don't potentially flood the CIB manager * and logs with a zillion attempts per second. * @@ -1027,7 +1033,7 @@ attrd_cib_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *u * if all peers similarly fail to write this attribute (which may * indicate a corrupted attribute entry rather than a CIB issue). */ - if (a->timer) { + } else if (a->timer) { // Attribute has a dampening value, so use that as delay if (!mainloop_timer_running(a->timer)) { crm_trace("Delayed re-attempted write (%dms) for %s", @@ -1067,7 +1073,7 @@ write_attributes(bool all, bool ignore_delay) /* When forced write flag is set, ignore delay. */ write_attribute(a, (a->force_write ? TRUE : ignore_delay)); } else { - crm_debug("Skipping unchanged attribute %s", a->id); + crm_trace("Skipping unchanged attribute %s", a->id); } } } -- 1.8.3.1 From 19b18a549013e7ce081d0874c92fdbb6885e0e2c Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 18 Oct 2018 09:45:03 -0500 Subject: [PATCH 20/69] Log: attrd: clear lost updates If an attribute's CIB update is determined to be lost, set the update ID to 0, to avoid logging the same message repeatedly in the (unlikely) chance that the a new write couldn't be scheduled (e.g. if the peer UUIDs are unknown for the values that need to be written). --- daemons/attrd/attrd_commands.c | 1 + 1 file changed, 1 insertion(+) diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c index fa0459a..8e81a66 100644 --- a/daemons/attrd/attrd_commands.c +++ b/daemons/attrd/attrd_commands.c @@ -1172,6 +1172,7 @@ write_attribute(attribute_t *a, bool ignore_delay) CRM_CHECK(the_cib != NULL, return); if (a->update && (a->update < last_cib_op_done)) { crm_info("Write out of '%s' continuing: update %d considered lost", a->id, a->update); + a->update = 0; // Don't log this message again } else if (a->update) { crm_info("Write out of '%s' delayed: update %d in progress", a->id, a->update); -- 1.8.3.1 From e1ff25d0d637664ad8954121f0b9a530eff1a5f9 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Wed, 3 Oct 2018 16:52:48 -0500 Subject: [PATCH 21/69] Low: attrd: don't start a new election when receiving a client update We already start an election (if needed) when an attribute needs to be written. --- daemons/attrd/attrd_commands.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c index 8e81a66..6daacba 100644 --- a/daemons/attrd/attrd_commands.c +++ b/daemons/attrd/attrd_commands.c @@ -297,8 +297,6 @@ attrd_client_update(xmlNode *xml) } } - attrd_start_election_if_needed(); - crm_debug("Broadcasting %s[%s]=%s%s", attr, host, value, (attrd_election_won()? " (writer)" : "")); -- 1.8.3.1 From 546f9f255a0ee53453927fa32847541b1f36c6f1 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 18 Oct 2018 13:44:02 -0500 Subject: [PATCH 22/69] Refactor: attrd: use defined constant for CIB op timeout for readability and possible future reuse --- daemons/attrd/attrd_commands.c | 3 ++- daemons/attrd/pacemaker-attrd.h | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c index 6daacba..4caf76c 100644 --- a/daemons/attrd/attrd_commands.c +++ b/daemons/attrd/attrd_commands.c @@ -1277,7 +1277,8 @@ write_attribute(attribute_t *a, bool ignore_delay) a->update, cib_updates, s_if_plural(cib_updates), a->id, (a->uuid? a->uuid : "n/a"), (a->set? a->set : "n/a")); - the_cib->cmds->register_callback_full(the_cib, a->update, 120, FALSE, + the_cib->cmds->register_callback_full(the_cib, a->update, + CIB_OP_TIMEOUT_S, FALSE, strdup(a->id), "attrd_cib_callback", attrd_cib_callback, free); diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h index bc4947b..2276c73 100644 --- a/daemons/attrd/pacemaker-attrd.h +++ b/daemons/attrd/pacemaker-attrd.h @@ -111,6 +111,8 @@ GHashTable *attributes; #define attrd_send_ack(client, id, flags) \ crm_ipcs_send_ack((client), (id), (flags), "ack", __FUNCTION__, __LINE__) +#define CIB_OP_TIMEOUT_S 120 + void write_attributes(bool all, bool ignore_delay); void attrd_broadcast_protocol(void); void attrd_peer_message(crm_node_t *client, xmlNode *msg); -- 1.8.3.1 From 677f6680526e2636646127f36a3d049e3e81e64b Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 18 Oct 2018 13:48:12 -0500 Subject: [PATCH 23/69] Refactor: attrd: functionize closing IPC server for readability, isolation, and possible future reuse --- daemons/attrd/pacemaker-attrd.c | 21 ++++++++++++++------- daemons/attrd/pacemaker-attrd.h | 1 + 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c index 69d02c0..4088938 100644 --- a/daemons/attrd/pacemaker-attrd.c +++ b/daemons/attrd/pacemaker-attrd.c @@ -224,6 +224,8 @@ attrd_cib_init() mainloop_set_trigger(attrd_config_read); } +static qb_ipcs_service_t *ipcs = NULL; + static int32_t attrd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size) { @@ -289,6 +291,16 @@ attrd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size) return 0; } +void +attrd_ipc_fini() +{ + if (ipcs != NULL) { + crm_client_disconnect_all(ipcs); + qb_ipcs_destroy(ipcs); + ipcs = NULL; + } +} + static int attrd_cluster_connect() { @@ -323,7 +335,6 @@ main(int argc, char **argv) int flag = 0; int index = 0; int argerr = 0; - qb_ipcs_service_t *ipcs = NULL; attrd_init_mainloop(); crm_log_preinit(NULL, argc, argv); @@ -397,14 +408,10 @@ main(int argc, char **argv) crm_info("Shutting down attribute manager"); attrd_election_fini(); - if (ipcs) { - crm_client_disconnect_all(ipcs); - qb_ipcs_destroy(ipcs); - g_hash_table_destroy(attributes); - } - + attrd_ipc_fini(); attrd_lrmd_disconnect(); attrd_cib_disconnect(); + g_hash_table_destroy(attributes); return crm_exit(attrd_exit_status); } diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h index 2276c73..cc8e29e 100644 --- a/daemons/attrd/pacemaker-attrd.h +++ b/daemons/attrd/pacemaker-attrd.h @@ -22,6 +22,7 @@ gboolean attrd_shutting_down(void); void attrd_shutdown(int nsig); void attrd_init_ipc(qb_ipcs_service_t **ipcs, qb_ipcs_msg_process_fn dispatch_fn); +void attrd_ipc_fini(void); void attrd_cib_disconnect(void); -- 1.8.3.1 From 6ddb87f6a364bd5c3be651d0ede0ec1c8f48666c Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 18 Oct 2018 14:33:24 -0500 Subject: [PATCH 24/69] Low: attrd: handle shutdown more cleanly Once shutdown has started, avoid wasting time on such things as updating the alert configuration, responding to most peer messages, starting a new election, or writing out attributes after a CIB replace. This doesn't really matter much since shutdown is self-contained at the moment. There were plans to change that, but they wound up being unnecessary. These changes still seem worthwhile, though. --- daemons/attrd/attrd_alerts.c | 2 +- daemons/attrd/attrd_commands.c | 8 ++++++++ daemons/attrd/attrd_elections.c | 9 +++++++-- daemons/attrd/attrd_utils.c | 25 +++++++++++++++++++++---- daemons/attrd/pacemaker-attrd.c | 6 +++++- 5 files changed, 42 insertions(+), 8 deletions(-) diff --git a/daemons/attrd/attrd_alerts.c b/daemons/attrd/attrd_alerts.c index e63d635..611f270 100644 --- a/daemons/attrd/attrd_alerts.c +++ b/daemons/attrd/attrd_alerts.c @@ -127,7 +127,7 @@ attrd_read_options(gpointer user_data) void attrd_cib_updated_cb(const char *event, xmlNode * msg) { - if (crm_patchset_contains_alert(msg, FALSE)) { + if (!attrd_shutting_down() && crm_patchset_contains_alert(msg, FALSE)) { mainloop_set_trigger(attrd_config_read); } } diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c index 4caf76c..64ab9f1 100644 --- a/daemons/attrd/attrd_commands.c +++ b/daemons/attrd/attrd_commands.c @@ -569,6 +569,14 @@ attrd_peer_message(crm_node_t *peer, xmlNode *xml) return; } + if (attrd_shutting_down()) { + /* If we're shutting down, we want to continue responding to election + * ops as long as we're a cluster member (because our vote may be + * needed). Ignore all other messages. + */ + return; + } + peer_won = attrd_check_for_new_writer(peer, xml); if (safe_str_eq(op, ATTRD_OP_UPDATE) || safe_str_eq(op, ATTRD_OP_UPDATE_BOTH) || safe_str_eq(op, ATTRD_OP_UPDATE_DELAY)) { diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c index 05e1d84..cfe2d89 100644 --- a/daemons/attrd/attrd_elections.c +++ b/daemons/attrd/attrd_elections.c @@ -32,7 +32,9 @@ void attrd_start_election_if_needed() { if ((peer_writer == NULL) - && (election_state(writer) != election_in_progress)) { + && (election_state(writer) != election_in_progress) + && !attrd_shutting_down()) { + crm_info("Starting an election to determine the writer"); election_vote(writer); } @@ -51,7 +53,10 @@ attrd_handle_election_op(const crm_node_t *peer, xmlNode *xml) enum election_result previous = election_state(writer); crm_xml_add(xml, F_CRM_HOST_FROM, peer->uname); - rc = election_count_vote(writer, xml, TRUE); + + // Don't become writer if we're shutting down + rc = election_count_vote(writer, xml, !attrd_shutting_down()); + switch(rc) { case election_start: free(peer_writer); diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c index 8245af6..6096dcc 100644 --- a/daemons/attrd/attrd_utils.c +++ b/daemons/attrd/attrd_utils.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -21,7 +22,9 @@ cib_t *the_cib = NULL; -static gboolean shutting_down = FALSE; +// volatile because attrd_shutdown() can be called for a signal +static volatile bool shutting_down = FALSE; + static GMainLoop *mloop = NULL; /*! @@ -45,11 +48,25 @@ attrd_shutting_down() void attrd_shutdown(int nsig) { + // Tell various functions not to do anthing shutting_down = TRUE; - if ((mloop != NULL) && g_main_loop_is_running(mloop)) { - g_main_loop_quit(mloop); - } else { + + // Don't respond to signals while shutting down + mainloop_destroy_signal(SIGTERM); + mainloop_destroy_signal(SIGCHLD); + mainloop_destroy_signal(SIGPIPE); + mainloop_destroy_signal(SIGUSR1); + mainloop_destroy_signal(SIGUSR2); + mainloop_destroy_signal(SIGTRAP); + + if ((mloop == NULL) || !g_main_loop_is_running(mloop)) { + /* If there's no main loop active, just exit. This should be possible + * only if we get SIGTERM in brief windows at start-up and shutdown. + */ crm_exit(CRM_EX_OK); + } else { + g_main_loop_quit(mloop); + g_main_loop_unref(mloop); } } diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c index 4088938..5097b53 100644 --- a/daemons/attrd/pacemaker-attrd.c +++ b/daemons/attrd/pacemaker-attrd.c @@ -81,8 +81,12 @@ attrd_cpg_destroy(gpointer unused) static void attrd_cib_replaced_cb(const char *event, xmlNode * msg) { - crm_notice("Updating all attributes after %s event", event); + if (attrd_shutting_down()) { + return; + } + if (attrd_election_won()) { + crm_notice("Updating all attributes after %s event", event); write_attributes(TRUE, FALSE); } } -- 1.8.3.1 From 435b7728688010f057681fdd276109a0c6b06ba0 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 3 Jan 2019 15:13:42 -0600 Subject: [PATCH 25/69] Fix: attrd: start new election if writer is lost If a writer is shutting down when it receives an attribute update, it will not write it to the CIB. Previously, a new election wouldn't be held until another attribute needed to be written, which may not happen in a reasonable time (or at all). Now, we trigger a new election when the writer leaves the cluster. rhbz#1535221 --- daemons/attrd/attrd_elections.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c index cfe2d89..a7816a9 100644 --- a/daemons/attrd/attrd_elections.c +++ b/daemons/attrd/attrd_elections.c @@ -1,5 +1,5 @@ /* - * Copyright 2013-2018 Andrew Beekhof + * Copyright 2013-2019 Andrew Beekhof * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. @@ -131,12 +131,20 @@ attrd_declare_winner() void attrd_remove_voter(const crm_node_t *peer) { + election_remove(writer, peer->uname); if (peer_writer && safe_str_eq(peer->uname, peer_writer)) { free(peer_writer); peer_writer = NULL; crm_notice("Lost attribute writer %s", peer->uname); + + /* If the writer received attribute updates during its shutdown, it will + * not have written them to the CIB. Ensure we get a new writer so they + * are written out. This means that every node that sees the writer + * leave will start a new election, but that's better than losing + * attributes. + */ + attrd_start_election_if_needed(); } - election_remove(writer, peer->uname); } void -- 1.8.3.1 From 4dcea7ea009c36ac7f3ebbb65fc02fb5479c3543 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Fri, 4 Jan 2019 18:25:57 -0600 Subject: [PATCH 26/69] Low: attrd: check for alert changes after CIB is replaced Previously, we checked for alert changes when the CIB was updated, but not when it was replaced entirely. --- daemons/attrd/pacemaker-attrd.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c index 5097b53..358e841 100644 --- a/daemons/attrd/pacemaker-attrd.c +++ b/daemons/attrd/pacemaker-attrd.c @@ -1,5 +1,5 @@ /* - * Copyright 2013-2018 Andrew Beekhof + * Copyright 2013-2019 Andrew Beekhof * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. @@ -89,6 +89,9 @@ attrd_cib_replaced_cb(const char *event, xmlNode * msg) crm_notice("Updating all attributes after %s event", event); write_attributes(TRUE, FALSE); } + + // Check for changes in alerts + mainloop_set_trigger(attrd_config_read); } static void -- 1.8.3.1 From c53b2832ecba09012e3a598d3a36655097352328 Mon Sep 17 00:00:00 2001 From: Chris Lumens Date: Wed, 12 Dec 2018 13:38:39 -0500 Subject: [PATCH 27/69] Bug: tools: Fix moving a resource with a lifetime constraint A node can be specified two different ways - with attribute="#uname" and value= on an expression XML node, or with node= on a rsc_location XML node. Both possibilities need to be considered when attempting to clear an existing constraint. cli_resource_clear was previously only considering the second case. This patch rearranges the code to allow for trying both, and then adds the code to try the first case by constructing a different blob of XML to match. See rhbz#1648620 --- tools/crm_resource_ban.c | 104 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 79 insertions(+), 25 deletions(-) diff --git a/tools/crm_resource_ban.c b/tools/crm_resource_ban.c index ef1413f..b74fc0d 100644 --- a/tools/crm_resource_ban.c +++ b/tools/crm_resource_ban.c @@ -191,37 +191,56 @@ cli_resource_prefer(const char *rsc_id, const char *host, cib_t * cib_conn) return rc; } -int -cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn) +/* Nodes can be specified two different ways in the CIB, so we have two different + * functions to try clearing out any constraints on them: + * + * (1) The node could be given by attribute=/value= in an expression XML node. + * That's what resource_clear_node_in_expr handles. That XML looks like this: + * + * + * + * + * + * + * + * + * (2) The mode could be given by node= in an rsc_location XML node. That's + * what resource_clear_node_in_location handles. That XML looks like this: + * + * + */ +static int +resource_clear_node_in_expr(const char *rsc_id, const char *host, cib_t * cib_conn) { int rc = pcmk_ok; - xmlNode *fragment = NULL; - xmlNode *location = NULL; + char *xpath_string = NULL; - if(cib_conn == NULL) { - return -ENOTCONN; - } + xpath_string = crm_strdup_printf("//rsc_location[@id='cli-prefer-%s'][rule[@id='cli-prefer-rule-%s']/expression[@attribute='#uname' and @value='%s']]", + rsc_id, rsc_id, host); - fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS); + rc = cib_conn->cmds->remove(cib_conn, xpath_string, NULL, cib_xpath | cib_options); + if (rc == -ENXIO) { + rc = pcmk_ok; + } - if(host) { - location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); - crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host); + free(xpath_string); + return rc; +} - } else { - GListPtr n = allnodes; - for(; n; n = n->next) { - node_t *target = n->data; +static int +resource_clear_node_in_location(const char *rsc_id, const char *host, cib_t * cib_conn) +{ + int rc = pcmk_ok; + xmlNode *fragment = NULL; + xmlNode *location = NULL; - location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); - crm_xml_set_id(location, "cli-ban-%s-on-%s", - rsc_id, target->details->uname); - } - } + fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS); + location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); + crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host); location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); crm_xml_set_id(location, "cli-prefer-%s", rsc_id); - if(host && do_force == FALSE) { + if (do_force == FALSE) { crm_xml_add(location, XML_CIB_TAG_NODE, host); } @@ -229,13 +248,48 @@ cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_ rc = cib_conn->cmds->remove(cib_conn, XML_CIB_TAG_CONSTRAINTS, fragment, cib_options); if (rc == -ENXIO) { rc = pcmk_ok; + } + + free(fragment); + return rc; +} - } else if (rc != pcmk_ok) { - goto bail; +int +cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn) +{ + int rc = pcmk_ok; + + if(cib_conn == NULL) { + return -ENOTCONN; + } + + if (host) { + rc = resource_clear_node_in_expr(rsc_id, host, cib_conn); + + /* rc does not tell us whether the previous operation did anything, only + * whether it failed or not. Thus, as long as it did not fail, we need + * to try the second clear method. + */ + if (rc == pcmk_ok) { + rc = resource_clear_node_in_location(rsc_id, host, cib_conn); + } + + } else { + GListPtr n = allnodes; + + /* Iterate over all nodes, attempting to clear the constraint from each. + * On the first error, abort. + */ + for(; n; n = n->next) { + node_t *target = n->data; + + rc = cli_resource_clear(rsc_id, target->details->uname, NULL, cib_conn); + if (rc != pcmk_ok) { + break; + } + } } - bail: - free_xml(fragment); return rc; } -- 1.8.3.1 From af3012388849f4ca5c021494dbf7b1b9260c2155 Mon Sep 17 00:00:00 2001 From: Chris Lumens Date: Mon, 7 Jan 2019 13:21:32 -0500 Subject: [PATCH 28/69] Bug: tools: Clear all prefer constraints when performing a move A move is implemented in terms of perfer constraints. If those constraints contain something like a lifetime expression, and older prefer constraints are not cleared out, the result is a mess. The XML that is attempted to insert into the CIB will contain both the older constraint and then the new lifetime expression as sub-nodes of that constraint. This is invalid, so the CIB will throw it out. The fix is to make sure there are no prefer constraints for any nodes when a move is done. Most ban constraints are left alone, because they may still be valid - you may want to move a resource to one node while preserving the ban on another node. Taking care of this is the bulk of the complexity in this patch. One further note - any ban constraints on the destination still need to be removed. Having both a ban and a prefer constraint on the same node may technically be valid XML, but doesn't make any sense. See rhbz#1648620 --- tools/crm_resource.c | 4 ++-- tools/crm_resource.h | 3 ++- tools/crm_resource_ban.c | 17 +++++++++++------ tools/crm_resource_runtime.c | 11 +++++++---- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/tools/crm_resource.c b/tools/crm_resource.c index 2e98999..294c091 100644 --- a/tools/crm_resource.c +++ b/tools/crm_resource.c @@ -982,10 +982,10 @@ main(int argc, char **argv) rc = -pcmk_err_node_unknown; goto bail; } - rc = cli_resource_clear(rsc_id, dest->details->uname, NULL, cib_conn); + rc = cli_resource_clear(rsc_id, dest->details->uname, NULL, cib_conn, TRUE); } else { - rc = cli_resource_clear(rsc_id, NULL, data_set->nodes, cib_conn); + rc = cli_resource_clear(rsc_id, NULL, data_set->nodes, cib_conn, TRUE); } } else if (rsc_cmd == 'M' && host_uname) { diff --git a/tools/crm_resource.h b/tools/crm_resource.h index dd902be..c371ddc 100644 --- a/tools/crm_resource.h +++ b/tools/crm_resource.h @@ -38,7 +38,8 @@ extern const char *attr_set_type; /* ban */ int cli_resource_prefer(const char *rsc_id, const char *host, cib_t * cib_conn); int cli_resource_ban(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn); -int cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn); +int cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn, + bool clear_ban_constraints); int cli_resource_clear_all_expired(xmlNode *root, cib_t *cib_conn, const char *rsc, const char *node, bool scope_master); /* print */ diff --git a/tools/crm_resource_ban.c b/tools/crm_resource_ban.c index b74fc0d..50843f0 100644 --- a/tools/crm_resource_ban.c +++ b/tools/crm_resource_ban.c @@ -228,15 +228,19 @@ resource_clear_node_in_expr(const char *rsc_id, const char *host, cib_t * cib_co } static int -resource_clear_node_in_location(const char *rsc_id, const char *host, cib_t * cib_conn) +resource_clear_node_in_location(const char *rsc_id, const char *host, cib_t * cib_conn, + bool clear_ban_constraints) { int rc = pcmk_ok; xmlNode *fragment = NULL; xmlNode *location = NULL; fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS); - location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); - crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host); + + if (clear_ban_constraints == TRUE) { + location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); + crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host); + } location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION); crm_xml_set_id(location, "cli-prefer-%s", rsc_id); @@ -255,7 +259,8 @@ resource_clear_node_in_location(const char *rsc_id, const char *host, cib_t * ci } int -cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn) +cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn, + bool clear_ban_constraints) { int rc = pcmk_ok; @@ -271,7 +276,7 @@ cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_ * to try the second clear method. */ if (rc == pcmk_ok) { - rc = resource_clear_node_in_location(rsc_id, host, cib_conn); + rc = resource_clear_node_in_location(rsc_id, host, cib_conn, clear_ban_constraints); } } else { @@ -283,7 +288,7 @@ cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes, cib_ for(; n; n = n->next) { node_t *target = n->data; - rc = cli_resource_clear(rsc_id, target->details->uname, NULL, cib_conn); + rc = cli_resource_clear(rsc_id, target->details->uname, NULL, cib_conn, clear_ban_constraints); if (rc != pcmk_ok) { break; } diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c index 27ca3b1..759ca92 100644 --- a/tools/crm_resource_runtime.c +++ b/tools/crm_resource_runtime.c @@ -1378,7 +1378,7 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, int timeout_ms, } if (stop_via_ban) { - rc = cli_resource_clear(rsc_id, host, NULL, cib); + rc = cli_resource_clear(rsc_id, host, NULL, cib, TRUE); } else if (orig_target_role) { rc = cli_resource_update_attribute(rsc, rsc_id, NULL, NULL, @@ -1460,7 +1460,7 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, int timeout_ms, failure: if (stop_via_ban) { - cli_resource_clear(rsc_id, host, NULL, cib); + cli_resource_clear(rsc_id, host, NULL, cib, TRUE); } else if (orig_target_role) { cli_resource_update_attribute(rsc, rsc_id, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, @@ -1874,8 +1874,11 @@ cli_resource_move(resource_t *rsc, const char *rsc_id, const char *host_name, } } - /* Clear any previous constraints for 'dest' */ - cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib); + /* Clear any previous prefer constraints across all nodes. */ + cli_resource_clear(rsc_id, NULL, data_set->nodes, cib, FALSE); + + /* Clear any previous ban constraints on 'dest'. */ + cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib, TRUE); /* Record an explicit preference for 'dest' */ rc = cli_resource_prefer(rsc_id, dest->details->uname, cib); -- 1.8.3.1 From e73a86abff33d9fb4e866c42ead02be7cce38d35 Mon Sep 17 00:00:00 2001 From: Chris Lumens Date: Tue, 18 Dec 2018 13:10:17 -0500 Subject: [PATCH 29/69] Tests: cts-cli: Add tests for more crm_resource options. --- cts/cli/regression.tools.exp | 200 +++++++++++++++++++++++++++++++++++++++++++ cts/cts-cli.in | 19 ++++ 2 files changed, 219 insertions(+) diff --git a/cts/cli/regression.tools.exp b/cts/cli/regression.tools.exp index 7a9acc4..b121096 100644 --- a/cts/cli/regression.tools.exp +++ b/cts/cli/regression.tools.exp @@ -2968,3 +2968,203 @@ Deleted 'test-primitive' option: id=test-primitive-meta_attributes-is-managed na =#=#=#= End test: Delete resource child meta attribute - OK (0) =#=#=#= * Passed: crm_resource - Delete resource child meta attribute +=#=#=#= Begin test: Specify a lifetime when moving a resource =#=#=#= +Migration will take effect until: +=#=#=#= Current cib after: Specify a lifetime when moving a resource =#=#=#= + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +=#=#=#= End test: Specify a lifetime when moving a resource - OK (0) =#=#=#= +* Passed: crm_resource - Specify a lifetime when moving a resource +=#=#=#= Begin test: Try to move a resource previously moved with a lifetime =#=#=#= +=#=#=#= Current cib after: Try to move a resource previously moved with a lifetime =#=#=#= + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +=#=#=#= End test: Try to move a resource previously moved with a lifetime - OK (0) =#=#=#= +* Passed: crm_resource - Try to move a resource previously moved with a lifetime +=#=#=#= Begin test: Ban dummy from node1 for a short time =#=#=#= +WARNING: Creating rsc_location constraint 'cli-ban-dummy-on-node1' with a score of -INFINITY for resource dummy on node1. + This will prevent dummy from running on node1 until the constraint is removed using the clear option or by editing the CIB with an appropriate tool + This will be the case even if node1 is the last node in the cluster +Migration will take effect until: +=#=#=#= Current cib after: Ban dummy from node1 for a short time =#=#=#= + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +=#=#=#= End test: Ban dummy from node1 for a short time - OK (0) =#=#=#= +* Passed: crm_resource - Ban dummy from node1 for a short time +=#=#=#= Begin test: Remove expired constraints =#=#=#= +=#=#=#= Current cib after: Remove expired constraints =#=#=#= + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +=#=#=#= End test: Remove expired constraints - OK (0) =#=#=#= +* Passed: crm_resource - Remove expired constraints diff --git a/cts/cts-cli.in b/cts/cts-cli.in index 880cebb..34c094e 100755 --- a/cts/cts-cli.in +++ b/cts/cts-cli.in @@ -394,6 +394,23 @@ function test_tools() { cmd="crm_resource -r test-primitive --meta -d is-managed" test_assert $CRM_EX_OK + desc="Specify a lifetime when moving a resource" + cmd="crm_resource -r dummy --move --node node2 --lifetime=PT1H" + test_assert $CRM_EX_OK + + desc="Try to move a resource previously moved with a lifetime" + cmd="crm_resource -r dummy --move --node node1" + test_assert $CRM_EX_OK + + desc="Ban dummy from node1 for a short time" + cmd="crm_resource -r dummy -B -N node1 --lifetime=PT1S" + test_assert $CRM_EX_OK + + desc="Remove expired constraints" + sleep 2 + cmd="crm_resource --clear --expired" + test_assert $CRM_EX_OK + unset CIB_shadow_dir rm -f "$TMPXML" "$TMPORIG" } @@ -920,6 +937,8 @@ for t in $tests; do -e 's|^/tmp/cts-cli\.validity\.bad.xml\.[^:]*:|validity.bad.xml:|'\ -e 's/^Entity: line [0-9][0-9]*: //'\ -e 's/\(validation ([0-9][0-9]* of \)[0-9][0-9]*\().*\)/\1X\2/' \ + -e 's/^Migration will take effect until: .*/Migration will take effect until:/' \ + -e 's/ end=\"[-: 0123456789]*Z\?\"/ end=\"\"/' \ "$TMPFILE" > "${TMPFILE}.$$" mv -- "${TMPFILE}.$$" "$TMPFILE" -- 1.8.3.1 From 48f59e65223e018f4a639c4c9c155cb62e7806a9 Mon Sep 17 00:00:00 2001 From: Chris Lumens Date: Thu, 10 Jan 2019 11:13:08 -0500 Subject: [PATCH 30/69] Tests: cts-cli: Use extended regular expressions. FreeBSD sed requires the use of extended regular expressions to use the + operator in a regex. The -E command line argument turns that on. GNU sed also supports this. Doing so flips the meaning of parens, though. An unescaped paren is now used for grouping, and an escaped paren is for a literal paren. --- cts/cts-cli.in | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cts/cts-cli.in b/cts/cts-cli.in index 34c094e..e48d644 100755 --- a/cts/cts-cli.in +++ b/cts/cts-cli.in @@ -924,21 +924,22 @@ for t in $tests; do eval TMPFILE_$t="$TMPFILE" test_$t > "$TMPFILE" - sed -e 's/cib-last-written.*>/>/'\ + sed -E \ + -e 's/cib-last-written.*>/>/'\ -e 's/ last-run=\"[0-9]*\"//'\ -e 's/crm_feature_set="[^"]*" //'\ -e 's/validate-with="[^"]*" //'\ -e 's/Created new pacemaker-.* configuration/Created new pacemaker configuration/'\ - -e 's/.*\(pcmk__.*\)@.*\.c:[0-9][0-9]*)/\1/g' \ - -e 's/.*\(unpack_.*\)@.*\.c:[0-9][0-9]*)/\1/g' \ - -e 's/.*\(update_validation\)@.*\.c:[0-9][0-9]*)/\1/g' \ - -e 's/.*\(apply_upgrade\)@.*\.c:[0-9][0-9]*)/\1/g' \ + -e 's/.*(pcmk__.*)@.*.c:[0-9][0-9]*\)/\1/g' \ + -e 's/.*(unpack_.*)@.*.c:[0-9][0-9]*\)/\1/g' \ + -e 's/.*(update_validation)@.*\.c:[0-9][0-9]*\)/\1/g' \ + -e 's/.*(apply_upgrade)@.*\.c:[0-9][0-9]*\)/\1/g' \ -e 's/ last-rc-change=\"[0-9]*\"//'\ -e 's|^/tmp/cts-cli\.validity\.bad.xml\.[^:]*:|validity.bad.xml:|'\ -e 's/^Entity: line [0-9][0-9]*: //'\ - -e 's/\(validation ([0-9][0-9]* of \)[0-9][0-9]*\().*\)/\1X\2/' \ + -e 's/(validation \([0-9][0-9]* of )[0-9][0-9]*(\).*)/\1X\2/' \ -e 's/^Migration will take effect until: .*/Migration will take effect until:/' \ - -e 's/ end=\"[-: 0123456789]*Z\?\"/ end=\"\"/' \ + -e 's/ end=\"[-: 0123456789]+Z?\"/ end=\"\"/' \ "$TMPFILE" > "${TMPFILE}.$$" mv -- "${TMPFILE}.$$" "$TMPFILE" -- 1.8.3.1 From bf2463c6a1d95e5a281ff324f0fc6416aa28e7b3 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 8 Jan 2019 16:18:13 -0600 Subject: [PATCH 31/69] Log: libcrmcommon: downgrade empty output logging to trace level nothing is not very interesting, so reduce clutter --- lib/common/logging.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common/logging.c b/lib/common/logging.c index bfd82b7..ff5ae74 100644 --- a/lib/common/logging.c +++ b/lib/common/logging.c @@ -997,7 +997,7 @@ crm_log_output_fn(const char *file, const char *function, int line, int level, c const char *offset = NULL; if (output == NULL) { - level = LOG_DEBUG; + level = LOG_TRACE; output = "-- empty --"; } -- 1.8.3.1 From 490ef9f6ed30427b3617236c514dd076ffd0e88f Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 10 Jan 2019 14:36:26 -0600 Subject: [PATCH 32/69] Low: resources: clean serialized file on SIGTERM in Dummy Otherwise it could give a false probe error at next start, confusing whatever else is being tested with a dummy resource. Unfortunately this doesn't help if an in-flight monitor gets cancelled with a SIGKILL, but there's no obvious solution there. --- extra/resources/Dummy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/extra/resources/Dummy b/extra/resources/Dummy index faa0e08..75e4cf5 100755 --- a/extra/resources/Dummy +++ b/extra/resources/Dummy @@ -114,6 +114,10 @@ END trap sigterm_handler TERM sigterm_handler() { ocf_log info "They use TERM to bring us down. No such luck." + + # Since we're likely going to get KILLed, clean up any monitor + # serialization in progress, so the next probe doesn't return an error. + rm -f "${VERIFY_SERIALIZED_FILE}" return } @@ -171,6 +175,7 @@ dummy_monitor() { # two monitor ops have occurred at the same time. # This verifies a condition in pacemaker-execd regression tests. ocf_log err "$VERIFY_SERIALIZED_FILE exists already" + ocf_exit_reason "alternate universe collision" return $OCF_ERR_GENERIC fi -- 1.8.3.1 From fe7172ff757996300c41d3d382d050a69c944bfc Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 8 Jan 2019 15:31:14 -0600 Subject: [PATCH 33/69] Fix: controller: directly acknowledge unrecordable operation results Regression introduced in 2.0.1-rc1 by 0363985dd Before that commit, if an operation result arrived when there was no resource information available, a warning would be logged and the operation would be directly acknowledged. This could occur, for example, if resource history were cleaned while an operation was pending on that resource. After that commit, in that situation, an assertion and error would be logged, and no acknowledgement would be sent, leading to a transition timeout. Restore the direct ack. Also improve related log messages. --- daemons/controld/controld_execd.c | 80 +++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 25 deletions(-) diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c index f7c5cde..26fcced 100644 --- a/daemons/controld/controld_execd.c +++ b/daemons/controld/controld_execd.c @@ -2483,6 +2483,7 @@ process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, int update_id = 0; gboolean remove = FALSE; gboolean removed = FALSE; + bool need_direct_ack = FALSE; lrmd_rsc_info_t *rsc = NULL; const char *node_name = NULL; @@ -2513,7 +2514,6 @@ process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, op_key, op->rsc_id); } } - CRM_LOG_ASSERT(rsc != NULL); // If it's still NULL, there's a bug somewhere // Get node name if available (from executor state or action XML) if (lrm_state) { @@ -2545,51 +2545,81 @@ process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, } if (op->op_status != PCMK_LRM_OP_CANCELLED) { + /* We might not record the result, so directly acknowledge it to the + * originator instead, so it doesn't time out waiting for the result + * (especially important if part of a transition). + */ + need_direct_ack = TRUE; + if (controld_action_is_recordable(op->op_type)) { if (node_name && rsc) { + // We should record the result, and happily, we can update_id = do_update_resource(node_name, rsc, op); + need_direct_ack = FALSE; + + } else if (op->rsc_deleted) { + /* We shouldn't record the result (likely the resource was + * refreshed, cleaned, or removed while this operation was + * in flight). + */ + crm_notice("Not recording %s result in CIB because " + "resource information was removed since it was initiated", + op_key); } else { - // @TODO Should we direct ack? - crm_err("Unable to record %s result in CIB: %s", - op_key, + /* This shouldn't be possible; the executor didn't consider the + * resource deleted, but we couldn't find resource or node + * information. + */ + crm_err("Unable to record %s result in CIB: %s", op_key, (node_name? "No resource information" : "No node name")); } - } else { - send_direct_ack(NULL, NULL, NULL, op, op->rsc_id); } + } else if (op->interval_ms == 0) { - /* This will occur when "crm resource cleanup" is called while actions are in-flight */ - crm_err("Op %s (call=%d): Cancelled", op_key, op->call_id); - send_direct_ack(NULL, NULL, NULL, op, op->rsc_id); + /* A non-recurring operation was cancelled. Most likely, the + * never-initiated action was removed from the executor's pending + * operations list upon resource removal. + */ + need_direct_ack = TRUE; } else if (pending == NULL) { - /* We don't need to do anything for cancelled ops - * that are not in our pending op list. There are no - * transition actions waiting on these operations. */ + /* This recurring operation was cancelled, but was not pending. No + * transition actions are waiting on it, nothing needs to be done. + */ } else if (op->user_data == NULL) { - /* At this point we have a pending entry, but no transition - * key present in the user_data field. report this */ - crm_err("Op %s (call=%d): No user data", op_key, op->call_id); + /* This recurring operation was cancelled and pending, but we don't + * have a transition key. This should never happen. + */ + crm_err("Recurring operation %s was cancelled without transition information", + op_key); } else if (pending->remove) { - /* The tengine canceled this op, we have been waiting for the cancel to finish. */ + /* This recurring operation was cancelled (by us) and pending, and we + * have been waiting for it to finish. + */ if (lrm_state) { erase_lrm_history_by_op(lrm_state, op); } } else if (op->rsc_deleted) { - /* The tengine initiated this op, but it was cancelled outside of the - * tengine's control during a resource cleanup/re-probe request. The tengine - * must be alerted that this operation completed, otherwise the tengine - * will continue waiting for this update to occur until it is timed out. - * We don't want this update going to the cib though, so use a direct ack. */ - crm_trace("Op %s (call=%d): cancelled due to rsc deletion", op_key, op->call_id); - send_direct_ack(NULL, NULL, NULL, op, op->rsc_id); + /* This recurring operation was cancelled (but not by us, and the + * executor does not have resource information, likely due to resource + * cleanup, refresh, or removal) and pending. + */ + crm_debug("Recurring op %s was cancelled due to resource deletion", + op_key); + need_direct_ack = TRUE; } else { - /* Before a stop is called, no need to direct ack */ - crm_trace("Op %s (call=%d): no delete event required", op_key, op->call_id); + /* This recurring operation was cancelled (but not by us, likely by the + * executor before stopping the resource) and pending. We don't need to + * do anything special. + */ + } + + if (need_direct_ack) { + send_direct_ack(NULL, NULL, NULL, op, op->rsc_id); } if(remove == FALSE) { -- 1.8.3.1 From cf64fdd8c842a365f90a28cdf3f374a4ba1e62c2 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 10 Jan 2019 15:10:50 -0600 Subject: [PATCH 34/69] Doc: ChangeLog: update for 2.0.1-rc3 release --- ChangeLog | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ChangeLog b/ChangeLog index e348333..ea65fbe 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,16 @@ +* Thu Jan 10 2019 Ken Gaillot Pacemaker-2.0.1-rc3 +- Changesets: 27 +- Diff: 20 files changed, 375 insertions(+), 195 deletions(-) + +- Changes since Pacemaker-2.0.1-rc2 + + attrd: start new election immediately if writer is lost + + attrd: detect alert configuration changes when CIB is entirely replaced + + controller: avoid transition timeout if resource cleaned while operation + is in-flight (regression in 2.0.1-rc1) + + libstonithd: restore C++ compatibility (regression in 2.0.1-rc1) + + tools: fix crm_resource --clear when lifetime was used with ban/move + + tools: fix crm_resource --move when lifetime was used with previous move + * Wed Dec 19 2018 Ken Gaillot Pacemaker-2.0.1-rc2 - Changesets: 12 - Diff: 2 files changed, 6 insertions(+), 2 deletions(-) -- 1.8.3.1 From bddfb0d3373951be9a200c1b1968e7d925d7fa53 Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Fri, 11 Jan 2019 11:43:49 +0100 Subject: [PATCH 35/69] Test: cts-exec: still run the tests for the other resource classes even without python systemd bindings --- cts/cts-exec.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cts/cts-exec.in b/cts/cts-exec.in index 235a966..30b87dd 100644 --- a/cts/cts-exec.in +++ b/cts/cts-exec.in @@ -387,9 +387,9 @@ class Tests(object): # all systemd tests to fail. import systemd.daemon except ImportError: - print("Fatal error: python systemd bindings not found. Is package installed?", - file=sys.stderr) - sys.exit(CrmExit.ERROR) + print("Python systemd bindings not found.") + print("The tests for systemd class are not going to be run.") + self.rsc_classes.remove("systemd") print("Testing resource classes", repr(self.rsc_classes)) -- 1.8.3.1 From ca267fc163d9f3fc4373d558cc9ab7c096a67171 Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Fri, 18 Jan 2019 17:02:06 +0100 Subject: [PATCH 36/69] Test: cts: service counts as enabled only if it's explicitly enabled With "systemctl is-enabled", we should check if the service is explicitly "enabled" instead of the return code. For example it returns 0 if the service is "static" or "indirect", but they don't really count as "enabled". This also functionizes the check part for further use. --- cts/environment.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/cts/environment.py b/cts/environment.py index 0ce7513..611d0c3 100644 --- a/cts/environment.py +++ b/cts/environment.py @@ -180,20 +180,25 @@ class Environment(object): # default self["syslogd"] = "rsyslog" + def service_is_enabled(self, node, service): + if self["have_systemd"]: + # Systemd + + # With "systemctl is-enabled", we should check if the service is + # explicitly "enabled" instead of the return code. For example it returns + # 0 if the service is "static" or "indirect", but they don't really count + # as "enabled". + return not self.rsh(node, "systemctl is-enabled %s | grep enabled" % service) + + else: + # SYS-V + return not self.rsh(node, "chkconfig --list | grep -e %s.*on" % service) + def detect_at_boot(self): # Detect if the cluster starts at boot if not "at-boot" in self.data: - atboot = 0 - - if self["have_systemd"]: - # Systemd - atboot = atboot or not self.rsh(self.target, "systemctl is-enabled corosync.service") - atboot = atboot or not self.rsh(self.target, "systemctl is-enabled pacemaker.service") - else: - # SYS-V - atboot = atboot or not self.rsh(self.target, "chkconfig --list | grep -e corosync.*on -e pacemaker.*on") - - self["at-boot"] = atboot + self["at-boot"] = self.service_is_enabled(self.target, "corosync") \ + or self.service_is_enabled(self.target, "pacemaker") def detect_ip_offset(self): # Try to determine an offset for IPaddr resources -- 1.8.3.1 From 286569c1a2359a97ad33f77b3b6b63b04d520b76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= Date: Sat, 19 Jan 2019 22:18:37 +0100 Subject: [PATCH 37/69] Revert "Tests: cts-cli: Use extended regular expressions." This reverts commit 48f59e65223e018f4a639c4c9c155cb62e7806a9, because while it's true that GNU sed supports -E switch _now_, it wasn't the case until about v4.1.5+ (silently, see 3a8e165, and explicitly since v4.3, see 8b65e07), meaning that, for instance, RHEL 7 doesn't officially supports it (as sad as it is): https://bugzilla.redhat.com/show_bug.cgi?id=1564789 Also note that while there was an effort to standardize -E switch in POSIX (http://austingroupbugs.net/view.php?id=528) it may have passed without any affect so far (speaking of "2018 edition"): https://pubs.opengroup.org/onlinepubs/9699919799/utilities/sed.html Definitely, it wasn't in IEEE Std 1003.1-2008 that, moreover, we don't take for granted even 10+ years later (cf. ongoing parallel discussion whether and how to check for %m specifier to scanf(3)), the change at hand is not anything else but invalid. Also, -E implementation may have been faulty until sed v4.5: https://lists.gnu.org/archive/html/info-gnu/2018-04/msg00000.html Note that one can make do without extended regular expressions, and in turn, without '+' (or '\+' that is just a GNU-specific extension into basic regular expressions syntax), since the respective substitute can be used: "a+" ~ "aa*" (can be hefty for long patterns, but that's what we reliably have). --- cts/cts-cli.in | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cts/cts-cli.in b/cts/cts-cli.in index e48d644..34c094e 100755 --- a/cts/cts-cli.in +++ b/cts/cts-cli.in @@ -924,22 +924,21 @@ for t in $tests; do eval TMPFILE_$t="$TMPFILE" test_$t > "$TMPFILE" - sed -E \ - -e 's/cib-last-written.*>/>/'\ + sed -e 's/cib-last-written.*>/>/'\ -e 's/ last-run=\"[0-9]*\"//'\ -e 's/crm_feature_set="[^"]*" //'\ -e 's/validate-with="[^"]*" //'\ -e 's/Created new pacemaker-.* configuration/Created new pacemaker configuration/'\ - -e 's/.*(pcmk__.*)@.*.c:[0-9][0-9]*\)/\1/g' \ - -e 's/.*(unpack_.*)@.*.c:[0-9][0-9]*\)/\1/g' \ - -e 's/.*(update_validation)@.*\.c:[0-9][0-9]*\)/\1/g' \ - -e 's/.*(apply_upgrade)@.*\.c:[0-9][0-9]*\)/\1/g' \ + -e 's/.*\(pcmk__.*\)@.*\.c:[0-9][0-9]*)/\1/g' \ + -e 's/.*\(unpack_.*\)@.*\.c:[0-9][0-9]*)/\1/g' \ + -e 's/.*\(update_validation\)@.*\.c:[0-9][0-9]*)/\1/g' \ + -e 's/.*\(apply_upgrade\)@.*\.c:[0-9][0-9]*)/\1/g' \ -e 's/ last-rc-change=\"[0-9]*\"//'\ -e 's|^/tmp/cts-cli\.validity\.bad.xml\.[^:]*:|validity.bad.xml:|'\ -e 's/^Entity: line [0-9][0-9]*: //'\ - -e 's/(validation \([0-9][0-9]* of )[0-9][0-9]*(\).*)/\1X\2/' \ + -e 's/\(validation ([0-9][0-9]* of \)[0-9][0-9]*\().*\)/\1X\2/' \ -e 's/^Migration will take effect until: .*/Migration will take effect until:/' \ - -e 's/ end=\"[-: 0123456789]+Z?\"/ end=\"\"/' \ + -e 's/ end=\"[-: 0123456789]*Z\?\"/ end=\"\"/' \ "$TMPFILE" > "${TMPFILE}.$$" mv -- "${TMPFILE}.$$" "$TMPFILE" -- 1.8.3.1 From 9394c8a585138be0a83adde5c7adb3f3f2032fd6 Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Fri, 18 Jan 2019 18:14:13 +0100 Subject: [PATCH 38/69] Test: cts: temporarily disable any enabled cluster serivces when running remote tests Cluster nodes are reused as remote nodes in remote tests. If cluster services were enabled at boot, in case the remote node got fenced, the cluster node would join instead of the expected remote one. Meanwhile pacemaker_remote would not be able to start. Depending on the chances, the situations might not be able to be orchestrated gracefully any more. --- cts/CTStests.py | 27 +++++++++++++++++++++++++++ cts/environment.py | 18 ++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/cts/CTStests.py b/cts/CTStests.py index 6a4aa51..3d5d88c 100644 --- a/cts/CTStests.py +++ b/cts/CTStests.py @@ -2674,6 +2674,22 @@ class RemoteDriver(CTSTest): if not self.failed: self.remote_node_added = 1 + def disable_services(self, node): + self.corosync_enabled = self.Env.service_is_enabled(node, "corosync") + if self.corosync_enabled: + self.Env.disable_service(node, "corosync") + + self.pacemaker_enabled = self.Env.service_is_enabled(node, "pacemaker") + if self.pacemaker_enabled: + self.Env.disable_service(node, "pacemaker") + + def restore_services(self, node): + if self.corosync_enabled: + self.Env.enable_service(node, "corosync") + + if self.pacemaker_enabled: + self.Env.enable_service(node, "pacemaker") + def stop_pcmk_remote(self, node): # disable pcmk remote for i in range(10): @@ -2703,6 +2719,15 @@ class RemoteDriver(CTSTest): self.rsh(node, "killall -CONT pacemaker-remoted") def start_metal(self, node): + # Cluster nodes are reused as remote nodes in remote tests. If cluster + # services were enabled at boot, in case the remote node got fenced, the + # cluster node would join instead of the expected remote one. Meanwhile + # pacemaker_remote would not be able to start. Depending on the chances, + # the situations might not be able to be orchestrated gracefully any more. + # + # Temporarily disable any enabled cluster serivces. + self.disable_services(node) + pcmk_started = 0 # make sure the resource doesn't already exist for some reason @@ -2875,6 +2900,8 @@ class RemoteDriver(CTSTest): return def cleanup_metal(self, node): + self.restore_services(node) + if self.pcmk_started == 0: return diff --git a/cts/environment.py b/cts/environment.py index 611d0c3..d152578 100644 --- a/cts/environment.py +++ b/cts/environment.py @@ -180,6 +180,24 @@ class Environment(object): # default self["syslogd"] = "rsyslog" + def disable_service(self, node, service): + if self["have_systemd"]: + # Systemd + return self.rsh(node, "systemctl disable %s" % service) + + else: + # SYS-V + return self.rsh(node, "chkconfig %s off" % service) + + def enable_service(self, node, service): + if self["have_systemd"]: + # Systemd + return self.rsh(node, "systemctl enable %s" % service) + + else: + # SYS-V + return self.rsh(node, "chkconfig %s on" % service) + def service_is_enabled(self, node, service): if self["have_systemd"]: # Systemd -- 1.8.3.1 From abb6feeb39c027aa0c3f973ef93e3524ea866936 Mon Sep 17 00:00:00 2001 From: Klaus Wenninger Date: Mon, 21 Jan 2019 13:40:44 +0100 Subject: [PATCH 39/69] Fix: crm_mon: remove duplicity of fence-action-state in xml-output and unnecessary conditionals making code less readable that were probably introduced rearranging the code. rhbz#1667191 --- tools/crm_mon.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tools/crm_mon.c b/tools/crm_mon.c index aad9b71..bc161aa 100644 --- a/tools/crm_mon.c +++ b/tools/crm_mon.c @@ -3175,7 +3175,7 @@ print_stonith_action(FILE *stream, stonith_history_t *event) fprintf(stream, " completed=\"%s\"", run_at_s?run_at_s:""); break; default: - fprintf(stream, " state=\"pending\""); + break; } fprintf(stream, " />\n"); break; @@ -3189,8 +3189,7 @@ print_stonith_action(FILE *stream, stonith_history_t *event) action_s, event->target, event->delegate ? event->delegate : "", event->client, event->origin, - ((!fence_full_history) && (output_format != mon_output_xml))? - "last-successful":"completed", + fence_full_history?"completed":"last-successful", run_at_s?run_at_s:""); break; case st_failed: @@ -3199,8 +3198,7 @@ print_stonith_action(FILE *stream, stonith_history_t *event) action_s, event->target, event->delegate ? event->delegate : "", event->client, event->origin, - ((!fence_full_history) && (output_format != mon_output_xml))? - "last-failed":"completed", + fence_full_history?"completed":"last-failed", run_at_s?run_at_s:""); break; default: @@ -3219,9 +3217,7 @@ print_stonith_action(FILE *stream, stonith_history_t *event) action_s, event->target, event->delegate ? event->delegate : "", event->client, event->origin, - ((!fence_full_history) && - (output_format != mon_output_xml))? - "last-successful":"completed", + fence_full_history?"completed":"last-successful", run_at_s?run_at_s:""); break; case st_failed: @@ -3230,9 +3226,7 @@ print_stonith_action(FILE *stream, stonith_history_t *event) action_s, event->target, event->delegate ? event->delegate : "", event->client, event->origin, - ((!fence_full_history) && - (output_format != mon_output_xml))? - "last-failed":"completed", + fence_full_history?"completed":"last-failed", run_at_s?run_at_s:""); break; default: -- 1.8.3.1 From a2e873635db3dfbb696527372dfaad9f58621f48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= Date: Mon, 21 Jan 2019 20:46:56 +0100 Subject: [PATCH 40/69] Build: 2 *.c: restore GCC 9 -Werror buildability: avoid NULL to '%s' arg Sadly, pacemaker codebase seems to be possibly heavily spoiled with this undefined behaviour when possibly passing NULL corresponding to '%s' format specifier argument, so for the time being, fix just what new GCC 9 started to spot[*] (due to build-time constant NULLs, which is an immediate proof) since these occurrences boil down to mere thinkos. Related to that, would be wise to start rolling out nonnull annotations to preserve more general sanity in this self explanatory aspect. Looking at libqb code (end destination of "crm_log" processing), there's nothing to implicitly mask NULL with a predestined string explicitly (like glibc make do with "(null)" in majority of the cases), so unless merely a blackbox is used for logging (qb_vsnprintf_serialize seems to deal with such a NULL gracefully), passing NULLs where a character array is expected is rather dangerous without the prior knowledge of particular libc (vsnprintf) implementation. [*] https://gcc.gnu.org/git/?p=gcc.git;a=blobdiff;f=gcc/gimple-ssa-sprintf.c;h=456a7d400115713a6600b1ce7bb303b6c971550e;hb=7b2ced2fa2c0597ba083461864c9026c3c9d3720;hpb=b07bf3b9730bebfc9953ea2eeee86a1274e39022 --- daemons/based/based_callbacks.c | 6 +++--- tools/test.iso8601.c | 11 ++++------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/daemons/based/based_callbacks.c b/daemons/based/based_callbacks.c index 723e74c..daef8c7 100644 --- a/daemons/based/based_callbacks.c +++ b/daemons/based/based_callbacks.c @@ -594,8 +594,8 @@ parse_peer_options_v1(int call_type, xmlNode * request, return TRUE; } - crm_trace("Processing %s request sent by %s", op, originator); op = crm_element_value(request, F_CIB_OPERATION); + crm_trace("Processing %s request sent by %s", op, originator); if (safe_str_eq(op, "cib_shutdown_req")) { /* Always process these */ *local_notify = FALSE; @@ -650,11 +650,11 @@ parse_peer_options_v1(int call_type, xmlNode * request, } else if (safe_str_eq(op, "cib_shutdown_req")) { if (reply_to != NULL) { - crm_debug("Processing %s from %s", op, host); + crm_debug("Processing %s from %s", op, originator); *needs_reply = FALSE; } else { - crm_debug("Processing %s reply from %s", op, host); + crm_debug("Processing %s reply from %s", op, originator); } return TRUE; diff --git a/tools/test.iso8601.c b/tools/test.iso8601.c index 6d70af5..93ca481 100644 --- a/tools/test.iso8601.c +++ b/tools/test.iso8601.c @@ -8,6 +8,7 @@ #include #include #include +#include /* CRM_ASSERT */ #include char command = 0; @@ -46,13 +47,9 @@ static struct crm_option long_options[] = { static void log_time_period(int log_level, crm_time_period_t * dtp, int flags) { - char *end = NULL; - char *start = NULL; - - if(dtp) { - start = crm_time_as_string(dtp->start, flags); - end = crm_time_as_string(dtp->end, flags); - } + char *start = crm_time_as_string(dtp->start, flags); + char *end = crm_time_as_string(dtp->end, flags); + CRM_ASSERT(start != NULL && end != NULL); if (log_level < LOG_CRIT) { printf("Period: %s to %s\n", start, end); -- 1.8.3.1 From cb3312337603e894743789aaeb7bd13bbbae9b56 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Mon, 14 Jan 2019 09:12:37 -0600 Subject: [PATCH 41/69] Low: libstonithd: correct header inclusions stonith-ng.h was unnecessarily including libxml/tree.h, and was missing includes for time.h (for time_t) and stdint.h (for uint32_t). --- include/crm/stonith-ng.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/include/crm/stonith-ng.h b/include/crm/stonith-ng.h index 62a72d9..df99b55 100644 --- a/include/crm/stonith-ng.h +++ b/include/crm/stonith-ng.h @@ -1,5 +1,5 @@ /* - * Copyright 2004-2018 Andrew Beekhof + * Copyright 2004-2019 Andrew Beekhof * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. @@ -20,10 +20,9 @@ extern "C" { # include # include -# include - -/* TO-DO: Work out how to drop this requirement */ -# include +# include // bool +# include // uint32_t +# include // time_t # define T_STONITH_NOTIFY_DISCONNECT "st_notify_disconnect" # define T_STONITH_NOTIFY_FENCE "st_notify_fence" -- 1.8.3.1 From 991f5bffcffb43bbd398f5e20773775ae4052d72 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 15 Jan 2019 14:44:47 -0600 Subject: [PATCH 42/69] Doc: Pacemaker Administration: mention version requirement for remote CIB administration --- doc/Pacemaker_Administration/en-US/Ch-Configuring.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/Pacemaker_Administration/en-US/Ch-Configuring.txt b/doc/Pacemaker_Administration/en-US/Ch-Configuring.txt index 5ca9dfc..286479b 100644 --- a/doc/Pacemaker_Administration/en-US/Ch-Configuring.txt +++ b/doc/Pacemaker_Administration/en-US/Ch-Configuring.txt @@ -434,3 +434,9 @@ to set the +remote-tls-port+ (encrypted) or +remote-clear-port+ |========================================================= +[IMPORTANT] +==== +The Pacemaker version on the administration host must be the same or greater +than the version(s) on the cluster nodes. Otherwise, it may not have the schema +files necessary to validate the CIB. +==== -- 1.8.3.1 From 6457d5bd48e9659b4a6fd42a173ab11ce3bb781b Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Wed, 16 Jan 2019 12:28:13 -0600 Subject: [PATCH 43/69] Build: spec: add hint about sbd compatibility The libpe_status API for the pe_working_set_t data type changed incompatibly in 2.0.1. sbd 1.4.0 supports the new API, so add a Conflicts line for versions older than that. Of course, the library soname version change is the hard barrier, but this provides another hint to someone preparing for an upgrade. --- pacemaker.spec.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pacemaker.spec.in b/pacemaker.spec.in index 6b2b268..d11ba1e 100644 --- a/pacemaker.spec.in +++ b/pacemaker.spec.in @@ -296,6 +296,8 @@ Summary: Core Pacemaker libraries Group: System Environment/Daemons Requires(pre): shadow-utils Requires: %{name}-schemas = %{version}-%{release} +# sbd 1.4.0+ supports the libpe_status API for pe_working_set_t +Conflicts: sbd < 1.4.0 %description libs Pacemaker is an advanced, scalable High-Availability cluster resource -- 1.8.3.1 From 7a5444d0cf312feb7f92bed0134a32cd6777c3d9 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Fri, 18 Jan 2019 15:33:39 -0600 Subject: [PATCH 44/69] Doc: GNUmakefile: remove obsolete references to CMAN in comments --- GNUmakefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 60de623..58ee17e 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -1,5 +1,5 @@ # -# Copyright 2008-2018 Andrew Beekhof +# Copyright 2008-2019 Andrew Beekhof # # This source code is licensed under the GNU General Public License version 2 # or later (GPLv2+) WITHOUT ANY WARRANTY. @@ -74,7 +74,7 @@ RSYNC_OPTS = -rlptvzxS --progress # # --with[out] FOO -> --define "_with[out]_FOO --with[out]-FOO" # -# $(1) ... WITH string (e.g., --with pre_release --without cman) +# $(1) ... WITH string (e.g., --with pre_release --without doc) # $(2) ... options following the initial "rpmbuild" in the command # $(3) ... final arguments determined with $2 (e.g., pacemaker.spec) # @@ -214,7 +214,7 @@ mock-sh-%: mock --root=$* $(MOCK_OPTIONS) --shell echo "Done" -# eg. WITH="--with cman" make rpm +# eg. make WITH="--with pre_release" rpm mock-%: make srpm-$(firstword $(shell echo $(@:mock-%=%) | tr '-' ' ')) -rm -rf $(RPM_ROOT)/mock -- 1.8.3.1 From 42ff7282a7e6ea65c8ea861c1f52c50b8f3959e1 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Wed, 23 Jan 2019 14:53:31 -0600 Subject: [PATCH 45/69] Test: cts-exec-helper: allow multiple -V arguments allows for future use by cts-exec --- daemons/execd/cts-exec-helper.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/daemons/execd/cts-exec-helper.c b/daemons/execd/cts-exec-helper.c index 76e6ca4..90e5622 100644 --- a/daemons/execd/cts-exec-helper.c +++ b/daemons/execd/cts-exec-helper.c @@ -1,5 +1,5 @@ /* - * Copyright 2012-2018 David Vossel + * Copyright 2012-2019 David Vossel * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. @@ -469,7 +469,8 @@ main(int argc, char **argv) crm_help(flag, CRM_EX_OK); break; case 'V': - options.verbose = 1; + ++options.verbose; + crm_bump_log_level(argc, argv); break; case 'Q': options.quiet = 1; -- 1.8.3.1 From 18fb7cea3226e13b21a5b147f9bbd60f764f028e Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 22 Jan 2019 13:15:45 -0600 Subject: [PATCH 46/69] Log: libcrmservice: improve logs when cleaning up an operation Previously, services_action_cleanup() got the sense of dbus_pending_call_get_completed() wrong when logging a "did not complete" message. It didn't really matter because the true case should never happen, but this corrects it anyway. This also slightly refactors to use services_set_op_pending() to clear the pending call, to reduce code duplication, with the only real effect being to log additional trace messages. --- lib/services/services.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/services/services.c b/lib/services/services.c index debe9ec..8b22d4a 100644 --- a/lib/services/services.c +++ b/lib/services/services.c @@ -1,5 +1,5 @@ /* - * Copyright 2010-2018 Andrew Beekhof + * Copyright 2010-2019 Andrew Beekhof * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. @@ -460,7 +460,7 @@ services_set_op_pending(svc_action_t *op, DBusPendingCall *pending) void services_action_cleanup(svc_action_t * op) { - if(op->opaque == NULL) { + if ((op == NULL) || (op->opaque == NULL)) { return; } @@ -472,13 +472,16 @@ services_action_cleanup(svc_action_t * op) } if(op->opaque->pending) { - crm_trace("Cleaning up pending dbus call %p %s for %s", op->opaque->pending, op->action, op->rsc); - if(dbus_pending_call_get_completed(op->opaque->pending)) { - crm_warn("Pending dbus call %s for %s did not complete", op->action, op->rsc); + if (dbus_pending_call_get_completed(op->opaque->pending)) { + // This should never be the case + crm_warn("Result of %s op %s was unhandled", + op->standard, op->id); + } else { + crm_debug("Will ignore any result of canceled %s op %s", + op->standard, op->id); } dbus_pending_call_cancel(op->opaque->pending); - dbus_pending_call_unref(op->opaque->pending); - op->opaque->pending = NULL; + services_set_op_pending(op, NULL); } #endif -- 1.8.3.1 From a70e50988f859b352b72e75a051188e01090cf3b Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Wed, 23 Jan 2019 14:15:39 -0600 Subject: [PATCH 47/69] Log: libcrmservice: null-terminate string *before* printing it --- lib/services/services_linux.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c index bd22777..e2685e9 100644 --- a/lib/services/services_linux.c +++ b/lib/services/services_linux.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2016 Andrew Beekhof + * Copyright 2010-2019 Andrew Beekhof * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. @@ -67,8 +67,8 @@ svc_read_output(int fd, svc_action_t * op, bool is_stderr) do { rc = read(fd, buf, buf_read_len); if (rc > 0) { - crm_trace("Got %d chars: %.80s", rc, buf); buf[rc] = 0; + crm_trace("Got %d chars: %.80s", rc, buf); data = realloc_safe(data, len + rc + 1); len += sprintf(data + len, "%s", buf); -- 1.8.3.1 From ab5e25a2ab840e452d2d4d77694356b594e47247 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 22 Jan 2019 13:16:02 -0600 Subject: [PATCH 48/69] Fix: libcrmservice: cancel DBus call when cancelling systemd/upstart actions deabcc5a was incomplete. When cancelling a recurring systemd or upstart operation, we should cancel any pending DBus call immediately (by telling DBus to ignore any result), rather than wait for it to complete. --- lib/services/services.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/services/services.c b/lib/services/services.c index 8b22d4a..20e824f 100644 --- a/lib/services/services.c +++ b/lib/services/services.c @@ -591,7 +591,7 @@ services_action_cancel(const char *name, const char *action, guint interval_ms) /* Tell operation_finalize() not to reschedule the operation */ op->cancel = TRUE; - /* Stop tracking it as a recurring operation, and stop its timer */ + /* Stop tracking it as a recurring operation, and stop its repeat timer */ cancel_recurring_action(op); /* If the op has a PID, it's an in-flight child process, so kill it. @@ -610,19 +610,22 @@ services_action_cancel(const char *name, const char *action, guint interval_ms) goto done; } - /* In-flight systemd and upstart ops don't have a pid. The relevant handlers - * will call operation_finalize() when the operation completes. - * @TODO: Can we request early termination, maybe using - * dbus_pending_call_cancel()? - */ +#if SUPPORT_DBUS + // In-flight systemd and upstart ops don't have a pid if (inflight_systemd_or_upstart(op)) { - crm_info("Will cancel %s op %s when in-flight instance completes", - op->standard, op->id); - cancelled = FALSE; - goto done; + inflight_ops = g_list_remove(inflight_ops, op); + + /* This will cause any result that comes in later to be discarded, so we + * don't call the callback and free the operation twice. + */ + services_action_cleanup(op); } +#endif + + // The rest of this is essentially equivalent to operation_finalize(), + // except without calling handle_blocked_ops() - /* Otherwise, operation is not in-flight, just report as cancelled */ + // Report operation as cancelled op->status = PCMK_LRM_OP_CANCELLED; if (op->opaque->callback) { op->opaque->callback(op); @@ -631,6 +634,7 @@ services_action_cancel(const char *name, const char *action, guint interval_ms) blocked_ops = g_list_remove(blocked_ops, op); services_action_free(op); cancelled = TRUE; + // @TODO Initiate handle_blocked_ops() asynchronously done: free(id); -- 1.8.3.1 From 8001619faefb1bf0db386953b61b491d39a04548 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 22 Jan 2019 16:22:32 -0600 Subject: [PATCH 49/69] Refactor: scheduler: functionize creating shutdown op for readability. Also tweak log message. --- daemons/schedulerd/sched_allocate.c | 12 ++---------- daemons/schedulerd/sched_utils.c | 29 ++++++++++++++++++++++++++++- daemons/schedulerd/sched_utils.h | 3 ++- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/daemons/schedulerd/sched_allocate.c b/daemons/schedulerd/sched_allocate.c index 7d62730..5b28e30 100644 --- a/daemons/schedulerd/sched_allocate.c +++ b/daemons/schedulerd/sched_allocate.c @@ -1,5 +1,5 @@ /* - * Copyright 2004-2018 Andrew Beekhof + * Copyright 2004-2019 Andrew Beekhof * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. @@ -1593,15 +1593,7 @@ stage6(pe_working_set_t * data_set) * if we can come up with a good use for this in the future, we will. */ is_remote_node(node) == FALSE) { - action_t *down_op = NULL; - - crm_notice("Scheduling Node %s for shutdown", node->details->uname); - - down_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_SHUTDOWN, node->details->uname), - CRM_OP_SHUTDOWN, node, FALSE, TRUE, data_set); - - shutdown_constraints(node, down_op, data_set); - add_hash_param(down_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE); + action_t *down_op = sched_shutdown_op(node, data_set); if (node->details->is_dc) { dc_down = down_op; diff --git a/daemons/schedulerd/sched_utils.c b/daemons/schedulerd/sched_utils.c index 153dc27..613dcf3 100644 --- a/daemons/schedulerd/sched_utils.c +++ b/daemons/schedulerd/sched_utils.c @@ -1,5 +1,5 @@ /* - * Copyright 2004-2018 Andrew Beekhof + * Copyright 2004-2019 Andrew Beekhof * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. @@ -442,3 +442,30 @@ pe_cancel_op(pe_resource_t *rsc, const char *task, guint interval_ms, return cancel_op; } + +/*! + * \internal + * \brief Create a shutdown op for a scheduler transition + * + * \param[in] rsc Resource of action to cancel + * \param[in] task Name of action to cancel + * \param[in] interval_ms Interval of action to cancel + * \param[in] node Node of action to cancel + * \param[in] data_set Working set of cluster + * + * \return Created op + */ +pe_action_t * +sched_shutdown_op(pe_node_t *node, pe_working_set_t *data_set) +{ + char *shutdown_id = crm_strdup_printf("%s-%s", CRM_OP_SHUTDOWN, + node->details->uname); + + pe_action_t *shutdown_op = custom_action(NULL, shutdown_id, CRM_OP_SHUTDOWN, + node, FALSE, TRUE, data_set); + + crm_notice("Scheduling shutdown of node %s", node->details->uname); + shutdown_constraints(node, shutdown_op, data_set); + add_hash_param(shutdown_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE); + return shutdown_op; +} diff --git a/daemons/schedulerd/sched_utils.h b/daemons/schedulerd/sched_utils.h index b049dd7..6c3cb46 100644 --- a/daemons/schedulerd/sched_utils.h +++ b/daemons/schedulerd/sched_utils.h @@ -1,5 +1,5 @@ /* - * Copyright 2004-2018 Andrew Beekhof + * Copyright 2004-2019 Andrew Beekhof * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. @@ -70,6 +70,7 @@ pe_action_t *create_pseudo_resource_op(resource_t * rsc, const char *task, bool pe_action_t *pe_cancel_op(pe_resource_t *rsc, const char *name, guint interval_ms, pe_node_t *node, pe_working_set_t *data_set); +pe_action_t *sched_shutdown_op(pe_node_t *node, pe_working_set_t *data_set); # define LOAD_STOPPED "load_stopped" -- 1.8.3.1 From 78836c36123ce4d02b27a0ae3a9d74c6f5dc85d4 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 22 Jan 2019 16:29:36 -0600 Subject: [PATCH 50/69] Fix: scheduler: don't disable waiting for DC fencing 34339dac intended to disable waiting for shutdowns, but it also applied to fencing. Waiting is already disable for shutdowns when creating the op, so the extra line was redundant anyway. --- daemons/schedulerd/sched_allocate.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/daemons/schedulerd/sched_allocate.c b/daemons/schedulerd/sched_allocate.c index 5b28e30..13cbadc 100644 --- a/daemons/schedulerd/sched_allocate.c +++ b/daemons/schedulerd/sched_allocate.c @@ -1623,8 +1623,6 @@ stage6(pe_working_set_t * data_set) crm_trace("Ordering shutdowns before %s on %s (DC)", dc_down->task, dc_down->node->details->uname); - add_hash_param(dc_down->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE); - for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { action_t *node_stop = (action_t *) gIter->data; -- 1.8.3.1 From 9d2a7f0abef0566ee88c94688a4ae4653d81d823 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 22 Jan 2019 16:35:12 -0600 Subject: [PATCH 51/69] Test: cts-scheduler: update regression tests for DC stonith change --- cts/scheduler/concurrent-fencing.exp | 2 +- cts/scheduler/migrate-fencing.exp | 2 +- cts/scheduler/per-op-failcount.exp | 2 +- cts/scheduler/rec-node-13.exp | 2 +- cts/scheduler/rec-node-14.exp | 2 +- cts/scheduler/rec-node-15.exp | 2 +- cts/scheduler/stonith-0.exp | 2 +- cts/scheduler/suicide-needed-inquorate.exp | 2 +- cts/scheduler/systemhealth1.exp | 2 +- cts/scheduler/systemhealthm1.exp | 2 +- cts/scheduler/systemhealthn1.exp | 2 +- cts/scheduler/systemhealtho1.exp | 2 +- cts/scheduler/systemhealthp1.exp | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cts/scheduler/concurrent-fencing.exp b/cts/scheduler/concurrent-fencing.exp index f479556..23295e3 100644 --- a/cts/scheduler/concurrent-fencing.exp +++ b/cts/scheduler/concurrent-fencing.exp @@ -13,7 +13,7 @@ - + diff --git a/cts/scheduler/migrate-fencing.exp b/cts/scheduler/migrate-fencing.exp index 0a5103a..84596e7 100644 --- a/cts/scheduler/migrate-fencing.exp +++ b/cts/scheduler/migrate-fencing.exp @@ -623,7 +623,7 @@ - + diff --git a/cts/scheduler/per-op-failcount.exp b/cts/scheduler/per-op-failcount.exp index d86e486..2a4eec5 100644 --- a/cts/scheduler/per-op-failcount.exp +++ b/cts/scheduler/per-op-failcount.exp @@ -64,7 +64,7 @@ - + diff --git a/cts/scheduler/rec-node-13.exp b/cts/scheduler/rec-node-13.exp index 8c5593b..782475d 100644 --- a/cts/scheduler/rec-node-13.exp +++ b/cts/scheduler/rec-node-13.exp @@ -44,7 +44,7 @@ - + diff --git a/cts/scheduler/rec-node-14.exp b/cts/scheduler/rec-node-14.exp index e95918f..5520755 100644 --- a/cts/scheduler/rec-node-14.exp +++ b/cts/scheduler/rec-node-14.exp @@ -17,7 +17,7 @@ - + diff --git a/cts/scheduler/rec-node-15.exp b/cts/scheduler/rec-node-15.exp index 1207b64..c3ee5b7 100644 --- a/cts/scheduler/rec-node-15.exp +++ b/cts/scheduler/rec-node-15.exp @@ -440,7 +440,7 @@ - + diff --git a/cts/scheduler/stonith-0.exp b/cts/scheduler/stonith-0.exp index 92853d8..6d286d3 100644 --- a/cts/scheduler/stonith-0.exp +++ b/cts/scheduler/stonith-0.exp @@ -395,7 +395,7 @@ - + diff --git a/cts/scheduler/suicide-needed-inquorate.exp b/cts/scheduler/suicide-needed-inquorate.exp index 58619aa..bd0b69c 100644 --- a/cts/scheduler/suicide-needed-inquorate.exp +++ b/cts/scheduler/suicide-needed-inquorate.exp @@ -17,7 +17,7 @@ - + diff --git a/cts/scheduler/systemhealth1.exp b/cts/scheduler/systemhealth1.exp index fb8764c..ac8e6ad 100644 --- a/cts/scheduler/systemhealth1.exp +++ b/cts/scheduler/systemhealth1.exp @@ -13,7 +13,7 @@ - + diff --git a/cts/scheduler/systemhealthm1.exp b/cts/scheduler/systemhealthm1.exp index fb8764c..ac8e6ad 100644 --- a/cts/scheduler/systemhealthm1.exp +++ b/cts/scheduler/systemhealthm1.exp @@ -13,7 +13,7 @@ - + diff --git a/cts/scheduler/systemhealthn1.exp b/cts/scheduler/systemhealthn1.exp index fb8764c..ac8e6ad 100644 --- a/cts/scheduler/systemhealthn1.exp +++ b/cts/scheduler/systemhealthn1.exp @@ -13,7 +13,7 @@ - + diff --git a/cts/scheduler/systemhealtho1.exp b/cts/scheduler/systemhealtho1.exp index fb8764c..ac8e6ad 100644 --- a/cts/scheduler/systemhealtho1.exp +++ b/cts/scheduler/systemhealtho1.exp @@ -13,7 +13,7 @@ - + diff --git a/cts/scheduler/systemhealthp1.exp b/cts/scheduler/systemhealthp1.exp index fb8764c..ac8e6ad 100644 --- a/cts/scheduler/systemhealthp1.exp +++ b/cts/scheduler/systemhealthp1.exp @@ -13,7 +13,7 @@ - + -- 1.8.3.1 From eb78c0451eda1b25310bc2b4e59480ecf89f63b8 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 22 Jan 2019 17:03:09 -0600 Subject: [PATCH 52/69] Refactor: scheduler: improve fence action ordering Remove redundant code (DC fence actions are not added to the remembered lists, so there is no need to check them against the DC later), add more comments, and use g_list_prepend() instead of g_list_append() for efficiency. --- daemons/schedulerd/sched_allocate.c | 57 ++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/daemons/schedulerd/sched_allocate.c b/daemons/schedulerd/sched_allocate.c index 13cbadc..2e13f8a 100644 --- a/daemons/schedulerd/sched_allocate.c +++ b/daemons/schedulerd/sched_allocate.c @@ -1527,7 +1527,6 @@ stage6(pe_working_set_t * data_set) { action_t *dc_down = NULL; action_t *stonith_op = NULL; - action_t *last_stonith = NULL; gboolean integrity_lost = FALSE; gboolean need_stonith = TRUE; GListPtr gIter; @@ -1575,16 +1574,24 @@ stage6(pe_working_set_t * data_set) stonith_constraints(node, stonith_op, data_set); if (node->details->is_dc) { + // Remember if the DC is being fenced dc_down = stonith_op; - } else if (is_set(data_set->flags, pe_flag_concurrent_fencing) == FALSE) { - if (last_stonith) { - order_actions(last_stonith, stonith_op, pe_order_optional); + } else { + + if (is_not_set(data_set->flags, pe_flag_concurrent_fencing) + && (stonith_ops != NULL)) { + /* Concurrent fencing is disabled, so order each non-DC + * fencing in a chain. If there is any DC fencing or + * shutdown, it will be ordered after the last action in the + * chain later. + */ + order_actions((pe_action_t *) stonith_ops->data, + stonith_op, pe_order_optional); } - last_stonith = stonith_op; - } else { - stonith_ops = g_list_append(stonith_ops, stonith_op); + // Remember all non-DC fencing actions in a separate list + stonith_ops = g_list_prepend(stonith_ops, stonith_op); } } else if (node->details->online && node->details->shutdown && @@ -1618,11 +1625,6 @@ stage6(pe_working_set_t * data_set) } if (dc_down != NULL) { - GListPtr gIter = NULL; - - crm_trace("Ordering shutdowns before %s on %s (DC)", - dc_down->task, dc_down->node->details->uname); - for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { action_t *node_stop = (action_t *) gIter->data; @@ -1632,28 +1634,31 @@ stage6(pe_working_set_t * data_set) continue; } - crm_debug("Ordering shutdown on %s before %s on %s", + crm_debug("Ordering shutdown on %s before %s on DC %s", node_stop->node->details->uname, dc_down->task, dc_down->node->details->uname); order_actions(node_stop, dc_down, pe_order_optional); } - if (last_stonith) { - if (dc_down != last_stonith) { - order_actions(last_stonith, dc_down, pe_order_optional); - } + // Order any non-DC fencing before any DC fencing or shutdown - } else { - GListPtr gIter2 = NULL; - - for (gIter2 = stonith_ops; gIter2 != NULL; gIter2 = gIter2->next) { - stonith_op = (action_t *) gIter2->data; - - if (dc_down != stonith_op) { - order_actions(stonith_op, dc_down, pe_order_optional); - } + if (is_set(data_set->flags, pe_flag_concurrent_fencing)) { + /* With concurrent fencing, order each non-DC fencing action + * separately before any DC fencing or shutdown. + */ + for (gIter = stonith_ops; gIter != NULL; gIter = gIter->next) { + order_actions((pe_action_t *) gIter->data, dc_down, + pe_order_optional); } + } else if (stonith_ops) { + /* Without concurrent fencing, the non-DC fencing actions are + * already ordered relative to each other, so we just need to order + * the DC fencing after the last action in the chain (which is the + * first item in the list). + */ + order_actions((pe_action_t *) stonith_ops->data, dc_down, + pe_order_optional); } } g_list_free(stonith_ops); -- 1.8.3.1 From 6b163036a275597bc738cb158075bf51b90ec440 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 22 Jan 2019 19:06:39 -0600 Subject: [PATCH 53/69] Fix: scheduler: don't order non-DC shutdowns before DC fencing because it can cause graph loops --- daemons/schedulerd/sched_allocate.c | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/daemons/schedulerd/sched_allocate.c b/daemons/schedulerd/sched_allocate.c index 2e13f8a..912a38a 100644 --- a/daemons/schedulerd/sched_allocate.c +++ b/daemons/schedulerd/sched_allocate.c @@ -1531,6 +1531,7 @@ stage6(pe_working_set_t * data_set) gboolean need_stonith = TRUE; GListPtr gIter; GListPtr stonith_ops = NULL; + GList *shutdown_ops = NULL; /* Remote ordering constraints need to happen prior to calculate * fencing because it is one more place we will mark the node as @@ -1603,7 +1604,11 @@ stage6(pe_working_set_t * data_set) action_t *down_op = sched_shutdown_op(node, data_set); if (node->details->is_dc) { + // Remember if the DC is being shut down dc_down = down_op; + } else { + // Remember non-DC shutdowns for later ordering + shutdown_ops = g_list_prepend(shutdown_ops, down_op); } } @@ -1625,20 +1630,23 @@ stage6(pe_working_set_t * data_set) } if (dc_down != NULL) { - for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { - action_t *node_stop = (action_t *) gIter->data; - - if (safe_str_neq(CRM_OP_SHUTDOWN, node_stop->task)) { - continue; - } else if (node_stop->node->details->is_dc) { - continue; - } + /* Order any non-DC shutdowns before any DC shutdown, to avoid repeated + * DC elections. However, we don't want to order non-DC shutdowns before + * a DC *fencing*, because even though we don't want a node that's + * shutting down to become DC, the DC fencing could be ordered before a + * clone stop that's also ordered before the shutdowns, thus leading to + * a graph loop. + */ + if (safe_str_eq(dc_down->task, CRM_OP_SHUTDOWN)) { + for (gIter = shutdown_ops; gIter != NULL; gIter = gIter->next) { + action_t *node_stop = (action_t *) gIter->data; - crm_debug("Ordering shutdown on %s before %s on DC %s", - node_stop->node->details->uname, - dc_down->task, dc_down->node->details->uname); + crm_debug("Ordering shutdown on %s before %s on DC %s", + node_stop->node->details->uname, + dc_down->task, dc_down->node->details->uname); - order_actions(node_stop, dc_down, pe_order_optional); + order_actions(node_stop, dc_down, pe_order_optional); + } } // Order any non-DC fencing before any DC fencing or shutdown @@ -1662,6 +1670,7 @@ stage6(pe_working_set_t * data_set) } } g_list_free(stonith_ops); + g_list_free(shutdown_ops); return TRUE; } -- 1.8.3.1 From 493f87ce9d9b122e0fb2c0f83dae2a9b32e648ed Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 22 Jan 2019 19:18:53 -0600 Subject: [PATCH 54/69] Test: scheduler: add regression test for DC fencing ordering --- cts/cts-scheduler.in | 1 + cts/scheduler/dc-fence-ordering.dot | 49 +++ cts/scheduler/dc-fence-ordering.exp | 259 +++++++++++++++ cts/scheduler/dc-fence-ordering.scores | 202 ++++++++++++ cts/scheduler/dc-fence-ordering.summary | 83 +++++ cts/scheduler/dc-fence-ordering.xml | 551 ++++++++++++++++++++++++++++++++ 6 files changed, 1145 insertions(+) create mode 100644 cts/scheduler/dc-fence-ordering.dot create mode 100644 cts/scheduler/dc-fence-ordering.exp create mode 100644 cts/scheduler/dc-fence-ordering.scores create mode 100644 cts/scheduler/dc-fence-ordering.summary create mode 100644 cts/scheduler/dc-fence-ordering.xml diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in index 8b6b2ac..6bf16da 100644 --- a/cts/cts-scheduler.in +++ b/cts/cts-scheduler.in @@ -884,6 +884,7 @@ do_test stonith-1 "Stonith loop - 2" do_test stonith-2 "Stonith loop - 3" do_test stonith-3 "Stonith startup" do_test stonith-4 "Stonith node state" +do_test dc-fence-ordering "DC needs fencing while other nodes are shutting down" do_test bug-1572-1 "Recovery of groups depending on master/slave" do_test bug-1572-2 "Recovery of groups depending on master/slave when the master is never re-promoted" do_test bug-1685 "Depends-on-master ordering" diff --git a/cts/scheduler/dc-fence-ordering.dot b/cts/scheduler/dc-fence-ordering.dot new file mode 100644 index 0000000..b700755 --- /dev/null +++ b/cts/scheduler/dc-fence-ordering.dot @@ -0,0 +1,49 @@ +digraph "g" { +"do_shutdown rhel7-2" [ style=bold color="green" fontcolor="black"] +"do_shutdown rhel7-4" [ style=bold color="green" fontcolor="black"] +"do_shutdown rhel7-5" [ style=bold color="green" fontcolor="black"] +"group-1_stop_0" -> "group-1_stopped_0" [ style = bold] +"group-1_stop_0" -> "petulant_stop_0 rhel7-1" [ style = bold] +"group-1_stop_0" -> "r192.168.122.207_stop_0 rhel7-1" [ style = bold] +"group-1_stop_0" [ style=bold color="green" fontcolor="orange"] +"group-1_stopped_0" -> "promotable-1_demote_0" [ style = bold] +"group-1_stopped_0" [ style=bold color="green" fontcolor="orange"] +"petulant_stop_0 rhel7-1" -> "group-1_stopped_0" [ style = bold] +"petulant_stop_0 rhel7-1" -> "r192.168.122.207_stop_0 rhel7-1" [ style = bold] +"petulant_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] +"promotable-1_demote_0" -> "promotable-1_demoted_0" [ style = bold] +"promotable-1_demote_0" -> "stateful-1_demote_0 rhel7-1" [ style = bold] +"promotable-1_demote_0" [ style=bold color="green" fontcolor="orange"] +"promotable-1_demoted_0" -> "promotable-1_stop_0" [ style = bold] +"promotable-1_demoted_0" [ style=bold color="green" fontcolor="orange"] +"promotable-1_stop_0" -> "promotable-1_stopped_0" [ style = bold] +"promotable-1_stop_0" -> "stateful-1_stop_0 rhel7-1" [ style = bold] +"promotable-1_stop_0" -> "stateful-1_stop_0 rhel7-2" [ style = bold] +"promotable-1_stop_0" -> "stateful-1_stop_0 rhel7-4" [ style = bold] +"promotable-1_stop_0" -> "stateful-1_stop_0 rhel7-5" [ style = bold] +"promotable-1_stop_0" [ style=bold color="green" fontcolor="orange"] +"promotable-1_stopped_0" [ style=bold color="green" fontcolor="orange"] +"r192.168.122.207_stop_0 rhel7-1" -> "group-1_stopped_0" [ style = bold] +"r192.168.122.207_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] +"stateful-1_demote_0 rhel7-1" -> "promotable-1_demoted_0" [ style = bold] +"stateful-1_demote_0 rhel7-1" -> "stateful-1_stop_0 rhel7-1" [ style = bold] +"stateful-1_demote_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] +"stateful-1_stop_0 rhel7-1" -> "promotable-1_stopped_0" [ style = bold] +"stateful-1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] +"stateful-1_stop_0 rhel7-2" -> "do_shutdown rhel7-2" [ style = bold] +"stateful-1_stop_0 rhel7-2" -> "promotable-1_stopped_0" [ style = bold] +"stateful-1_stop_0 rhel7-2" [ style=bold color="green" fontcolor="black"] +"stateful-1_stop_0 rhel7-4" -> "do_shutdown rhel7-4" [ style = bold] +"stateful-1_stop_0 rhel7-4" -> "promotable-1_stopped_0" [ style = bold] +"stateful-1_stop_0 rhel7-4" [ style=bold color="green" fontcolor="black"] +"stateful-1_stop_0 rhel7-5" -> "do_shutdown rhel7-5" [ style = bold] +"stateful-1_stop_0 rhel7-5" -> "promotable-1_stopped_0" [ style = bold] +"stateful-1_stop_0 rhel7-5" [ style=bold color="green" fontcolor="black"] +"stonith 'reboot' rhel7-1" -> "group-1_stop_0" [ style = bold] +"stonith 'reboot' rhel7-1" -> "petulant_stop_0 rhel7-1" [ style = bold] +"stonith 'reboot' rhel7-1" -> "promotable-1_stop_0" [ style = bold] +"stonith 'reboot' rhel7-1" -> "r192.168.122.207_stop_0 rhel7-1" [ style = bold] +"stonith 'reboot' rhel7-1" -> "stateful-1_demote_0 rhel7-1" [ style = bold] +"stonith 'reboot' rhel7-1" -> "stateful-1_stop_0 rhel7-1" [ style = bold] +"stonith 'reboot' rhel7-1" [ style=bold color="green" fontcolor="black"] +} diff --git a/cts/scheduler/dc-fence-ordering.exp b/cts/scheduler/dc-fence-ordering.exp new file mode 100644 index 0000000..429db5a --- /dev/null +++ b/cts/scheduler/dc-fence-ordering.exp @@ -0,0 +1,259 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cts/scheduler/dc-fence-ordering.scores b/cts/scheduler/dc-fence-ordering.scores new file mode 100644 index 0000000..92cdb26 --- /dev/null +++ b/cts/scheduler/dc-fence-ordering.scores @@ -0,0 +1,202 @@ +Allocation scores: +Using the original execution date of: 2018-11-28 18:37:16Z +clone_color: Connectivity allocation score on rhel7-1: 0 +clone_color: Connectivity allocation score on rhel7-2: 0 +clone_color: Connectivity allocation score on rhel7-3: 0 +clone_color: Connectivity allocation score on rhel7-4: 0 +clone_color: Connectivity allocation score on rhel7-5: 0 +clone_color: ping-1:0 allocation score on rhel7-1: 0 +clone_color: ping-1:0 allocation score on rhel7-2: 0 +clone_color: ping-1:0 allocation score on rhel7-3: 0 +clone_color: ping-1:0 allocation score on rhel7-4: 0 +clone_color: ping-1:0 allocation score on rhel7-5: 0 +clone_color: ping-1:1 allocation score on rhel7-1: 0 +clone_color: ping-1:1 allocation score on rhel7-2: 0 +clone_color: ping-1:1 allocation score on rhel7-3: 0 +clone_color: ping-1:1 allocation score on rhel7-4: 0 +clone_color: ping-1:1 allocation score on rhel7-5: 0 +clone_color: ping-1:2 allocation score on rhel7-1: 0 +clone_color: ping-1:2 allocation score on rhel7-2: 0 +clone_color: ping-1:2 allocation score on rhel7-3: 0 +clone_color: ping-1:2 allocation score on rhel7-4: 0 +clone_color: ping-1:2 allocation score on rhel7-5: 0 +clone_color: ping-1:3 allocation score on rhel7-1: 0 +clone_color: ping-1:3 allocation score on rhel7-2: 0 +clone_color: ping-1:3 allocation score on rhel7-3: 0 +clone_color: ping-1:3 allocation score on rhel7-4: 0 +clone_color: ping-1:3 allocation score on rhel7-5: 0 +clone_color: ping-1:4 allocation score on rhel7-1: 0 +clone_color: ping-1:4 allocation score on rhel7-2: 0 +clone_color: ping-1:4 allocation score on rhel7-3: 0 +clone_color: ping-1:4 allocation score on rhel7-4: 0 +clone_color: ping-1:4 allocation score on rhel7-5: 0 +clone_color: promotable-1 allocation score on rhel7-1: -INFINITY +clone_color: promotable-1 allocation score on rhel7-2: -INFINITY +clone_color: promotable-1 allocation score on rhel7-3: -INFINITY +clone_color: promotable-1 allocation score on rhel7-4: -INFINITY +clone_color: promotable-1 allocation score on rhel7-5: -INFINITY +clone_color: stateful-1:0 allocation score on rhel7-1: -INFINITY +clone_color: stateful-1:0 allocation score on rhel7-2: -INFINITY +clone_color: stateful-1:0 allocation score on rhel7-3: -INFINITY +clone_color: stateful-1:0 allocation score on rhel7-4: -INFINITY +clone_color: stateful-1:0 allocation score on rhel7-5: -INFINITY +clone_color: stateful-1:1 allocation score on rhel7-1: -INFINITY +clone_color: stateful-1:1 allocation score on rhel7-2: -INFINITY +clone_color: stateful-1:1 allocation score on rhel7-3: -INFINITY +clone_color: stateful-1:1 allocation score on rhel7-4: -INFINITY +clone_color: stateful-1:1 allocation score on rhel7-5: -INFINITY +clone_color: stateful-1:2 allocation score on rhel7-1: -INFINITY +clone_color: stateful-1:2 allocation score on rhel7-2: -INFINITY +clone_color: stateful-1:2 allocation score on rhel7-3: -INFINITY +clone_color: stateful-1:2 allocation score on rhel7-4: -INFINITY +clone_color: stateful-1:2 allocation score on rhel7-5: -INFINITY +clone_color: stateful-1:3 allocation score on rhel7-1: -INFINITY +clone_color: stateful-1:3 allocation score on rhel7-2: -INFINITY +clone_color: stateful-1:3 allocation score on rhel7-3: -INFINITY +clone_color: stateful-1:3 allocation score on rhel7-4: -INFINITY +clone_color: stateful-1:3 allocation score on rhel7-5: -INFINITY +clone_color: stateful-1:4 allocation score on rhel7-1: -INFINITY +clone_color: stateful-1:4 allocation score on rhel7-2: -INFINITY +clone_color: stateful-1:4 allocation score on rhel7-3: -INFINITY +clone_color: stateful-1:4 allocation score on rhel7-4: -INFINITY +clone_color: stateful-1:4 allocation score on rhel7-5: -INFINITY +group_color: group-1 allocation score on rhel7-1: 0 +group_color: group-1 allocation score on rhel7-2: 0 +group_color: group-1 allocation score on rhel7-3: 0 +group_color: group-1 allocation score on rhel7-4: 0 +group_color: group-1 allocation score on rhel7-5: 0 +group_color: petulant allocation score on rhel7-1: -INFINITY +group_color: petulant allocation score on rhel7-2: 0 +group_color: petulant allocation score on rhel7-3: 0 +group_color: petulant allocation score on rhel7-4: 0 +group_color: petulant allocation score on rhel7-5: 0 +group_color: r192.168.122.207 allocation score on rhel7-1: 0 +group_color: r192.168.122.207 allocation score on rhel7-2: 0 +group_color: r192.168.122.207 allocation score on rhel7-3: 0 +group_color: r192.168.122.207 allocation score on rhel7-4: 0 +group_color: r192.168.122.207 allocation score on rhel7-5: 0 +group_color: r192.168.122.208 allocation score on rhel7-1: 0 +group_color: r192.168.122.208 allocation score on rhel7-2: 0 +group_color: r192.168.122.208 allocation score on rhel7-3: 0 +group_color: r192.168.122.208 allocation score on rhel7-4: 0 +group_color: r192.168.122.208 allocation score on rhel7-5: 0 +native_color: Fencing allocation score on rhel7-1: 0 +native_color: Fencing allocation score on rhel7-2: 0 +native_color: Fencing allocation score on rhel7-3: 0 +native_color: Fencing allocation score on rhel7-4: 0 +native_color: Fencing allocation score on rhel7-5: 0 +native_color: FencingFail allocation score on rhel7-1: 0 +native_color: FencingFail allocation score on rhel7-2: 0 +native_color: FencingFail allocation score on rhel7-3: 0 +native_color: FencingFail allocation score on rhel7-4: 0 +native_color: FencingFail allocation score on rhel7-5: 0 +native_color: FencingPass allocation score on rhel7-1: 0 +native_color: FencingPass allocation score on rhel7-2: 0 +native_color: FencingPass allocation score on rhel7-3: 0 +native_color: FencingPass allocation score on rhel7-4: 0 +native_color: FencingPass allocation score on rhel7-5: 0 +native_color: lsb-dummy allocation score on rhel7-1: -INFINITY +native_color: lsb-dummy allocation score on rhel7-2: -INFINITY +native_color: lsb-dummy allocation score on rhel7-3: -INFINITY +native_color: lsb-dummy allocation score on rhel7-4: -INFINITY +native_color: lsb-dummy allocation score on rhel7-5: -INFINITY +native_color: migrator allocation score on rhel7-1: 0 +native_color: migrator allocation score on rhel7-2: 0 +native_color: migrator allocation score on rhel7-3: 0 +native_color: migrator allocation score on rhel7-4: 0 +native_color: migrator allocation score on rhel7-5: 0 +native_color: petulant allocation score on rhel7-1: -INFINITY +native_color: petulant allocation score on rhel7-2: -INFINITY +native_color: petulant allocation score on rhel7-3: -INFINITY +native_color: petulant allocation score on rhel7-4: -INFINITY +native_color: petulant allocation score on rhel7-5: -INFINITY +native_color: ping-1:0 allocation score on rhel7-1: -INFINITY +native_color: ping-1:0 allocation score on rhel7-2: -INFINITY +native_color: ping-1:0 allocation score on rhel7-3: -INFINITY +native_color: ping-1:0 allocation score on rhel7-4: -INFINITY +native_color: ping-1:0 allocation score on rhel7-5: -INFINITY +native_color: ping-1:1 allocation score on rhel7-1: -INFINITY +native_color: ping-1:1 allocation score on rhel7-2: -INFINITY +native_color: ping-1:1 allocation score on rhel7-3: -INFINITY +native_color: ping-1:1 allocation score on rhel7-4: -INFINITY +native_color: ping-1:1 allocation score on rhel7-5: -INFINITY +native_color: ping-1:2 allocation score on rhel7-1: -INFINITY +native_color: ping-1:2 allocation score on rhel7-2: -INFINITY +native_color: ping-1:2 allocation score on rhel7-3: -INFINITY +native_color: ping-1:2 allocation score on rhel7-4: -INFINITY +native_color: ping-1:2 allocation score on rhel7-5: -INFINITY +native_color: ping-1:3 allocation score on rhel7-1: -INFINITY +native_color: ping-1:3 allocation score on rhel7-2: -INFINITY +native_color: ping-1:3 allocation score on rhel7-3: -INFINITY +native_color: ping-1:3 allocation score on rhel7-4: -INFINITY +native_color: ping-1:3 allocation score on rhel7-5: -INFINITY +native_color: ping-1:4 allocation score on rhel7-1: -INFINITY +native_color: ping-1:4 allocation score on rhel7-2: -INFINITY +native_color: ping-1:4 allocation score on rhel7-3: -INFINITY +native_color: ping-1:4 allocation score on rhel7-4: -INFINITY +native_color: ping-1:4 allocation score on rhel7-5: -INFINITY +native_color: r192.168.122.207 allocation score on rhel7-1: -INFINITY +native_color: r192.168.122.207 allocation score on rhel7-2: -INFINITY +native_color: r192.168.122.207 allocation score on rhel7-3: -INFINITY +native_color: r192.168.122.207 allocation score on rhel7-4: -INFINITY +native_color: r192.168.122.207 allocation score on rhel7-5: -INFINITY +native_color: r192.168.122.208 allocation score on rhel7-1: -INFINITY +native_color: r192.168.122.208 allocation score on rhel7-2: -INFINITY +native_color: r192.168.122.208 allocation score on rhel7-3: -INFINITY +native_color: r192.168.122.208 allocation score on rhel7-4: -INFINITY +native_color: r192.168.122.208 allocation score on rhel7-5: -INFINITY +native_color: rsc_rhel7-1 allocation score on rhel7-1: 100 +native_color: rsc_rhel7-1 allocation score on rhel7-2: 0 +native_color: rsc_rhel7-1 allocation score on rhel7-3: 0 +native_color: rsc_rhel7-1 allocation score on rhel7-4: 0 +native_color: rsc_rhel7-1 allocation score on rhel7-5: 0 +native_color: rsc_rhel7-2 allocation score on rhel7-1: 0 +native_color: rsc_rhel7-2 allocation score on rhel7-2: 100 +native_color: rsc_rhel7-2 allocation score on rhel7-3: 0 +native_color: rsc_rhel7-2 allocation score on rhel7-4: 0 +native_color: rsc_rhel7-2 allocation score on rhel7-5: 0 +native_color: rsc_rhel7-3 allocation score on rhel7-1: 0 +native_color: rsc_rhel7-3 allocation score on rhel7-2: 0 +native_color: rsc_rhel7-3 allocation score on rhel7-3: 100 +native_color: rsc_rhel7-3 allocation score on rhel7-4: 0 +native_color: rsc_rhel7-3 allocation score on rhel7-5: 0 +native_color: rsc_rhel7-4 allocation score on rhel7-1: 0 +native_color: rsc_rhel7-4 allocation score on rhel7-2: 0 +native_color: rsc_rhel7-4 allocation score on rhel7-3: 0 +native_color: rsc_rhel7-4 allocation score on rhel7-4: 100 +native_color: rsc_rhel7-4 allocation score on rhel7-5: 0 +native_color: rsc_rhel7-5 allocation score on rhel7-1: 0 +native_color: rsc_rhel7-5 allocation score on rhel7-2: 0 +native_color: rsc_rhel7-5 allocation score on rhel7-3: 0 +native_color: rsc_rhel7-5 allocation score on rhel7-4: 0 +native_color: rsc_rhel7-5 allocation score on rhel7-5: 100 +native_color: stateful-1:0 allocation score on rhel7-1: -INFINITY +native_color: stateful-1:0 allocation score on rhel7-2: -INFINITY +native_color: stateful-1:0 allocation score on rhel7-3: -INFINITY +native_color: stateful-1:0 allocation score on rhel7-4: -INFINITY +native_color: stateful-1:0 allocation score on rhel7-5: -INFINITY +native_color: stateful-1:1 allocation score on rhel7-1: -INFINITY +native_color: stateful-1:1 allocation score on rhel7-2: -INFINITY +native_color: stateful-1:1 allocation score on rhel7-3: -INFINITY +native_color: stateful-1:1 allocation score on rhel7-4: -INFINITY +native_color: stateful-1:1 allocation score on rhel7-5: -INFINITY +native_color: stateful-1:2 allocation score on rhel7-1: -INFINITY +native_color: stateful-1:2 allocation score on rhel7-2: -INFINITY +native_color: stateful-1:2 allocation score on rhel7-3: -INFINITY +native_color: stateful-1:2 allocation score on rhel7-4: -INFINITY +native_color: stateful-1:2 allocation score on rhel7-5: -INFINITY +native_color: stateful-1:3 allocation score on rhel7-1: -INFINITY +native_color: stateful-1:3 allocation score on rhel7-2: -INFINITY +native_color: stateful-1:3 allocation score on rhel7-3: -INFINITY +native_color: stateful-1:3 allocation score on rhel7-4: -INFINITY +native_color: stateful-1:3 allocation score on rhel7-5: -INFINITY +native_color: stateful-1:4 allocation score on rhel7-1: -INFINITY +native_color: stateful-1:4 allocation score on rhel7-2: -INFINITY +native_color: stateful-1:4 allocation score on rhel7-3: -INFINITY +native_color: stateful-1:4 allocation score on rhel7-4: -INFINITY +native_color: stateful-1:4 allocation score on rhel7-5: -INFINITY +stateful-1:0 promotion score on none: 0 +stateful-1:1 promotion score on none: 0 +stateful-1:2 promotion score on none: 0 +stateful-1:3 promotion score on none: 0 +stateful-1:4 promotion score on none: 0 diff --git a/cts/scheduler/dc-fence-ordering.summary b/cts/scheduler/dc-fence-ordering.summary new file mode 100644 index 0000000..82e5096 --- /dev/null +++ b/cts/scheduler/dc-fence-ordering.summary @@ -0,0 +1,83 @@ +Using the original execution date of: 2018-11-28 18:37:16Z + +Current cluster status: +Node rhel7-1 (1): UNCLEAN (online) +Online: [ rhel7-2 rhel7-4 rhel7-5 ] +OFFLINE: [ rhel7-3 ] + + Fencing (stonith:fence_xvm): Stopped + FencingPass (stonith:fence_dummy): Stopped + FencingFail (stonith:fence_dummy): Stopped + rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Stopped + rsc_rhel7-2 (ocf::heartbeat:IPaddr2): Stopped + rsc_rhel7-3 (ocf::heartbeat:IPaddr2): Stopped + rsc_rhel7-4 (ocf::heartbeat:IPaddr2): Stopped + rsc_rhel7-5 (ocf::heartbeat:IPaddr2): Stopped + migrator (ocf::pacemaker:Dummy): Stopped + Clone Set: Connectivity [ping-1] + Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] + Clone Set: promotable-1 [stateful-1] (promotable) + Masters: [ rhel7-1 ] + Slaves: [ rhel7-2 rhel7-4 rhel7-5 ] + Stopped: [ rhel7-3 ] + Resource Group: group-1 + r192.168.122.207 (ocf::heartbeat:IPaddr2): Started rhel7-1 + petulant (service:pacemaker-cts-dummyd@10): FAILED rhel7-1 + r192.168.122.208 (ocf::heartbeat:IPaddr2): Stopped + lsb-dummy (lsb:LSBDummy): Stopped + +Transition Summary: + * Shutdown rhel7-5 + * Shutdown rhel7-4 + * Shutdown rhel7-2 + * Fence (reboot) rhel7-1 'petulant failed there' + * Stop stateful-1:0 ( Slave rhel7-5 ) due to node availability + * Stop stateful-1:1 ( Master rhel7-1 ) due to node availability + * Stop stateful-1:2 ( Slave rhel7-2 ) due to node availability + * Stop stateful-1:3 ( Slave rhel7-4 ) due to node availability + * Stop r192.168.122.207 ( rhel7-1 ) due to node availability + * Stop petulant ( rhel7-1 ) due to node availability + +Executing cluster transition: + * Fencing rhel7-1 (reboot) + * Pseudo action: group-1_stop_0 + * Pseudo action: petulant_stop_0 + * Pseudo action: r192.168.122.207_stop_0 + * Pseudo action: group-1_stopped_0 + * Pseudo action: promotable-1_demote_0 + * Pseudo action: stateful-1_demote_0 + * Pseudo action: promotable-1_demoted_0 + * Pseudo action: promotable-1_stop_0 + * Resource action: stateful-1 stop on rhel7-5 + * Pseudo action: stateful-1_stop_0 + * Resource action: stateful-1 stop on rhel7-2 + * Resource action: stateful-1 stop on rhel7-4 + * Pseudo action: promotable-1_stopped_0 + * Cluster action: do_shutdown on rhel7-5 + * Cluster action: do_shutdown on rhel7-4 + * Cluster action: do_shutdown on rhel7-2 +Using the original execution date of: 2018-11-28 18:37:16Z + +Revised cluster status: +Online: [ rhel7-2 rhel7-4 rhel7-5 ] +OFFLINE: [ rhel7-1 rhel7-3 ] + + Fencing (stonith:fence_xvm): Stopped + FencingPass (stonith:fence_dummy): Stopped + FencingFail (stonith:fence_dummy): Stopped + rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Stopped + rsc_rhel7-2 (ocf::heartbeat:IPaddr2): Stopped + rsc_rhel7-3 (ocf::heartbeat:IPaddr2): Stopped + rsc_rhel7-4 (ocf::heartbeat:IPaddr2): Stopped + rsc_rhel7-5 (ocf::heartbeat:IPaddr2): Stopped + migrator (ocf::pacemaker:Dummy): Stopped + Clone Set: Connectivity [ping-1] + Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] + Clone Set: promotable-1 [stateful-1] (promotable) + Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] + Resource Group: group-1 + r192.168.122.207 (ocf::heartbeat:IPaddr2): Stopped + petulant (service:pacemaker-cts-dummyd@10): Stopped + r192.168.122.208 (ocf::heartbeat:IPaddr2): Stopped + lsb-dummy (lsb:LSBDummy): Stopped + diff --git a/cts/scheduler/dc-fence-ordering.xml b/cts/scheduler/dc-fence-ordering.xml new file mode 100644 index 0000000..3ce3204 --- /dev/null +++ b/cts/scheduler/dc-fence-ordering.xml @@ -0,0 +1,551 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -- 1.8.3.1 From 220006281e5e7a0d886d71255dd4511dc0d163bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= Date: Sat, 19 Jan 2019 23:17:11 +0100 Subject: [PATCH 55/69] Tests: cts-cli: simplify+fix regexp to catch crm_time_as_string's output Previously, the cts-cli would only work with a zero-offset time zone (suggesting that it was only run in the CI that may be expected to have it like that). Also drop the atypical enumeration of all digits vs. range capture. --- cts/cts-cli.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cts/cts-cli.in b/cts/cts-cli.in index 34c094e..68e1459 100755 --- a/cts/cts-cli.in +++ b/cts/cts-cli.in @@ -938,7 +938,7 @@ for t in $tests; do -e 's/^Entity: line [0-9][0-9]*: //'\ -e 's/\(validation ([0-9][0-9]* of \)[0-9][0-9]*\().*\)/\1X\2/' \ -e 's/^Migration will take effect until: .*/Migration will take effect until:/' \ - -e 's/ end=\"[-: 0123456789]*Z\?\"/ end=\"\"/' \ + -e 's/ end=\"[0-9][-+: 0-9]*Z*\"/ end=\"\"/' \ "$TMPFILE" > "${TMPFILE}.$$" mv -- "${TMPFILE}.$$" "$TMPFILE" -- 1.8.3.1 From d6f54953ebf9539146378699f96ddd1f1e330311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= Date: Sun, 20 Jan 2019 11:21:05 +0100 Subject: [PATCH 57/69] Build: spec: swap a scheduler test to avoid glib 2.59.0+ incompatibility Related to the previous "Build: configure+cts_scheduler: allow skipping hash-table-broken tests" commit. --- pacemaker.spec.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pacemaker.spec.in b/pacemaker.spec.in index 6b2b268..1394e9b 100644 --- a/pacemaker.spec.in +++ b/pacemaker.spec.in @@ -458,7 +458,7 @@ sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool make %{_smp_mflags} V=1 all %check -{ cts/cts-scheduler --run one-or-more-unrunnable-instances \ +{ cts/cts-scheduler --run load-stopped-loop \ && cts/cts-cli \ && touch .CHECKED } 2>&1 | sed 's/[fF]ail/faiil/g' # prevent false positives in rpmlint -- 1.8.3.1 From 25e14406c763eb52c80487c9ce884cbeca86bbb0 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 24 Jan 2019 11:48:05 -0600 Subject: [PATCH 58/69] Log: attrd: log previous writer *before* clearing it --- daemons/attrd/attrd_elections.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c index a7816a9..025dae5 100644 --- a/daemons/attrd/attrd_elections.c +++ b/daemons/attrd/attrd_elections.c @@ -59,10 +59,10 @@ attrd_handle_election_op(const crm_node_t *peer, xmlNode *xml) switch(rc) { case election_start: - free(peer_writer); - peer_writer = NULL; crm_debug("Unsetting writer (was %s) and starting new election", peer_writer? peer_writer : "unset"); + free(peer_writer); + peer_writer = NULL; election_vote(writer); break; -- 1.8.3.1 From 5957a2bf9680320e81333b7e58caee93e054b1db Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 24 Jan 2019 17:48:10 -0600 Subject: [PATCH 59/69] Low: libcrmcluster: *really* write only one election storm black box 464d481 didn't do what it thought it did --- lib/cluster/election.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cluster/election.c b/lib/cluster/election.c index d904075..13c8377 100644 --- a/lib/cluster/election.c +++ b/lib/cluster/election.c @@ -1,5 +1,5 @@ /* - * Copyright 2004-2018 Andrew Beekhof + * Copyright 2004-2019 Andrew Beekhof * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. @@ -644,6 +644,7 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) * write a blackbox on every Nth occurrence. */ crm_write_blackbox(0, NULL); + wrote_blackbox = TRUE; } } } -- 1.8.3.1 From 818ff56236ea481ee57b1e6da8067524ae943ebb Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 24 Jan 2019 17:49:55 -0600 Subject: [PATCH 60/69] Refactor: libcrmcluster: move static variables into election structure Makes more sense, and allows for future methods to poke at them This would behave differently from the previous code if a second election object is created. However, that has never been a targeted use case, and the new behavior would make more sense anyway. --- lib/cluster/election.c | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/lib/cluster/election.c b/lib/cluster/election.c index 13c8377..c0ffb47 100644 --- a/lib/cluster/election.c +++ b/lib/cluster/election.c @@ -28,6 +28,10 @@ struct election_s { GSourceFunc cb; // Function to call if election is won GHashTable *voted; // Key = node name, value = how node voted mainloop_timer_t *timeout; // When to abort if all votes not received + int election_wins; // Track wins, for storm detection + bool wrote_blackbox; // Write a storm blackbox at most once + time_t expires; // When storm detection period ends + time_t last_election_loss; // When dampening period ends }; static void election_complete(election_t *e) @@ -531,11 +535,6 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) time_t tm_now = time(NULL); struct vote vote; - // @TODO these should be in election_t - static int election_wins = 0; - static time_t expires = 0; - static time_t last_election_loss = 0; - CRM_CHECK(message != NULL, return election_error); if (parse_election_message(e, message, &vote) == FALSE) { return election_error; @@ -615,24 +614,23 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) } } - if (expires < tm_now) { - election_wins = 0; - expires = tm_now + STORM_INTERVAL; + if (e->expires < tm_now) { + e->election_wins = 0; + e->expires = tm_now + STORM_INTERVAL; } else if (done == FALSE && we_lose == FALSE) { int peers = 1 + g_hash_table_size(crm_peer_cache); - static bool wrote_blackbox = FALSE; // @TODO move to election_t /* If every node has to vote down every other node, thats N*(N-1) total elections * Allow some leeway before _really_ complaining */ - election_wins++; - if (election_wins > (peers * peers)) { + e->election_wins++; + if (e->election_wins > (peers * peers)) { crm_warn("%s election storm detected: %d wins in %d seconds", - e->name, election_wins, STORM_INTERVAL); - election_wins = 0; - expires = tm_now + STORM_INTERVAL; - if (wrote_blackbox == FALSE) { + e->name, e->election_wins, STORM_INTERVAL); + e->election_wins = 0; + e->expires = tm_now + STORM_INTERVAL; + if (e->wrote_blackbox == FALSE) { /* It's questionable whether a black box (from every node in the * cluster) would be truly helpful in diagnosing an election * storm. It's also highly doubtful a production environment @@ -644,7 +642,7 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) * write a blackbox on every Nth occurrence. */ crm_write_blackbox(0, NULL); - wrote_blackbox = TRUE; + e->wrote_blackbox = TRUE; } } } @@ -657,21 +655,21 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) return e->state; } else if (we_lose == FALSE) { - if (last_election_loss == 0 - || tm_now - last_election_loss > (time_t) LOSS_DAMPEN) { + if ((e->last_election_loss == 0) + || ((tm_now - e->last_election_loss) > (time_t) LOSS_DAMPEN)) { do_crm_log(log_level, "%s round %d (owner node ID %s) pass: %s from %s (%s)", e->name, vote.election_id, vote.election_owner, vote.op, vote.from, reason); - last_election_loss = 0; + e->last_election_loss = 0; election_timeout_stop(e); /* Start a new election by voting down this, and other, peers */ e->state = election_start; return e->state; } else { - char *loss_time = ctime(&last_election_loss); + char *loss_time = ctime(&e->last_election_loss); if (loss_time) { // Show only HH:MM:SS @@ -684,7 +682,7 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) } } - last_election_loss = tm_now; + e->last_election_loss = tm_now; do_crm_log(log_level, "%s round %d (owner node ID %s) lost: %s from %s (%s)", e->name, vote.election_id, vote.election_owner, vote.op, -- 1.8.3.1 From 294ec2091501ce023528219ba61f174a80e5867b Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 24 Jan 2019 17:57:16 -0600 Subject: [PATCH 61/69] Doc: libcrmcluster: comment on election dampening failure case --- lib/cluster/election.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/cluster/election.c b/lib/cluster/election.c index c0ffb47..c7f46ba 100644 --- a/lib/cluster/election.c +++ b/lib/cluster/election.c @@ -655,6 +655,19 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) return e->state; } else if (we_lose == FALSE) { + /* We track the time of the last election loss to implement an election + * dampening period, reducing the likelihood of an election storm. If + * this node has lost within the dampening period, don't start a new + * election, even if we win against a peer's vote -- the peer we lost to + * should win again. + * + * @TODO This has a problem case: if an election winner immediately + * leaves the cluster, and a new election is immediately called, all + * nodes could lose, with no new winner elected. The ideal solution + * would be to tie the election structure with the peer caches, which + * would allow us to clear the dampening when the previous winner + * leaves (and would allow other improvements as well). + */ if ((e->last_election_loss == 0) || ((tm_now - e->last_election_loss) > (time_t) LOSS_DAMPEN)) { -- 1.8.3.1 From 59ab73a6138c72b8e2fe5613dc6fca3fced51a17 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 24 Jan 2019 18:01:00 -0600 Subject: [PATCH 62/69] Refactor: libcrmcommon: add method to clear election dampening --- include/crm/cluster/election.h | 3 ++- lib/cluster/election.c | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/include/crm/cluster/election.h b/include/crm/cluster/election.h index ab82ae6..0facc85 100644 --- a/include/crm/cluster/election.h +++ b/include/crm/cluster/election.h @@ -1,5 +1,5 @@ /* - * Copyright 2009-2018 Andrew Beekhof + * Copyright 2009-2019 Andrew Beekhof * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. @@ -76,6 +76,7 @@ bool election_check(election_t *e); void election_remove(election_t *e, const char *uname); enum election_result election_state(election_t *e); enum election_result election_count_vote(election_t *e, xmlNode *vote, bool can_win); +void election_clear_dampening(election_t *e); #ifdef __cplusplus } diff --git a/lib/cluster/election.c b/lib/cluster/election.c index c7f46ba..5df5c9c 100644 --- a/lib/cluster/election.c +++ b/lib/cluster/election.c @@ -706,3 +706,14 @@ election_count_vote(election_t *e, xmlNode *message, bool can_win) e->state = election_lost; return e->state; } + +/*! + * \brief Reset any election dampening currently in effect + * + * \param[in] e Election object to clear + */ +void +election_clear_dampening(election_t *e) +{ + e->last_election_loss = 0; +} -- 1.8.3.1 From 5c80a96423e246b04cd7a05253d7be7e032d8527 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Thu, 24 Jan 2019 18:08:07 -0600 Subject: [PATCH 63/69] Fix: attrd: clear election dampening when the writer leaves Otherwise we can easily get the failure case described in 294ec209, due to the recent changes in 435b7728. --- daemons/attrd/attrd_elections.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c index 025dae5..18a1c6f 100644 --- a/daemons/attrd/attrd_elections.c +++ b/daemons/attrd/attrd_elections.c @@ -137,6 +137,11 @@ attrd_remove_voter(const crm_node_t *peer) peer_writer = NULL; crm_notice("Lost attribute writer %s", peer->uname); + /* Clear any election dampening in effect. Otherwise, if the lost writer + * had just won, the election could fizzle out with no new writer. + */ + election_clear_dampening(writer); + /* If the writer received attribute updates during its shutdown, it will * not have written them to the CIB. Ensure we get a new writer so they * are written out. This means that every node that sees the writer -- 1.8.3.1 From 40055e8e4afb436762ed8ebb383677e525788fbb Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Fri, 25 Jan 2019 11:17:22 -0600 Subject: [PATCH 64/69] Doc: Pacemaker Explained: clarify section on colocated resource sets Most significantly, reorganize the examples such that resource A is always placed first, to make them easier to compare. Also, change the example PNG graphic to be generated from (new) SVG source. --- doc/Makefile.am | 20 +- doc/Pacemaker_Explained/en-US/Ch-Constraints.txt | 89 +++-- doc/shared/en-US/images/pcmk-colocated-sets.svg | 436 +++++++++++++++++++++++ doc/shared/en-US/images/three-sets-complex.png | Bin 82045 -> 0 bytes 4 files changed, 490 insertions(+), 55 deletions(-) create mode 100644 doc/shared/en-US/images/pcmk-colocated-sets.svg delete mode 100644 doc/shared/en-US/images/three-sets-complex.png diff --git a/doc/Makefile.am b/doc/Makefile.am index 1c4a1d3..aa9ff63 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -1,21 +1,8 @@ # -# doc: Pacemaker code +# Copyright 2004-2019 Andrew Beekhof # -# Copyright (C) 2008 Andrew Beekhof -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# This source code is licensed under the GNU General Public License version 2 +# or later (GPLv2+) WITHOUT ANY WARRANTY. # include $(top_srcdir)/Makefile.common @@ -73,7 +60,6 @@ PNGS_ORIGINAL = Pacemaker_Remote/en-US/images/pcmk-ha-cluster-stack.png \ shared/en-US/images/Partitioning.png \ shared/en-US/images/Welcome.png \ shared/en-US/images/resource-set.png \ - shared/en-US/images/three-sets-complex.png \ shared/en-US/images/three-sets.png \ shared/en-US/images/two-sets.png PNGS_GENERATED = $(SVGS:%.svg=%-small.png) \ diff --git a/doc/Pacemaker_Explained/en-US/Ch-Constraints.txt b/doc/Pacemaker_Explained/en-US/Ch-Constraints.txt index 3f76d8a..c28c8e9 100644 --- a/doc/Pacemaker_Explained/en-US/Ch-Constraints.txt +++ b/doc/Pacemaker_Explained/en-US/Ch-Constraints.txt @@ -753,15 +753,15 @@ to be an unordered set using "AND" logic by default, and adding Another common situation is for an administrator to create a set of colocated resources. -One way to do this would be to define a resource group (see +The simplest way to do this is to define a resource group (see <>), but that cannot always accurately express the desired -state. +relationships. For example, maybe the resources do not need to be ordered. Another way would be to define each relationship as an individual constraint, -but that causes a constraint explosion as the number of resources and -combinations grow. An example of this approach: +but that causes a difficult-to-follow constraint explosion as the number of +resources and combinations grow. -.Chain of colocated resources +.Colocation chain as individual constraints, where A is placed first, then B, then C, then D ====== [source,XML] ------- @@ -773,12 +773,9 @@ combinations grow. An example of this approach: ------- ====== -To make things easier, resource sets (see <>) can be used -within colocation constraints. As with the chained version, a -resource that can't be active prevents any resource that must be -colocated with it from being active. For example, if +B+ is not -able to run, then both +C+ and by inference +D+ must also remain -stopped. Here is an example +resource_set+: +To express complicated relationships with a simplified syntax +footnote:[which is not the same as saying easy to follow], +<> can be used within colocation constraints. .Equivalent colocation chain expressed using +resource_set+ ====== @@ -797,6 +794,19 @@ stopped. Here is an example +resource_set+: ------- ====== +[NOTE] +==== +Within a +resource_set+, the resources are listed in the order they are +_placed_, which is the reverse of the order in which they are _colocated_. +In the above example, resource +A+ is placed before resource +B+, which is +the same as saying resource +B+ is colocated with resource +A+. +==== + +As with individual constraints, a resource that can't be active prevents any +resource that must be colocated with it from being active. In both of the two +previous examples, if +B+ is unable to run, then both +C+ and by inference +D+ +must remain stopped. + [IMPORTANT] ========= If you use a higher-level tool, pay attention to how it exposes this @@ -804,38 +814,47 @@ functionality. Depending on the tool, creating a set +A B+ may be equivalent to +A with B+, or +B with A+. ========= -This notation can also be used to tell the cluster that sets of resources must -be colocated relative to each other, where the individual members of each set -may or may not depend on each other being active (controlled by the -+sequential+ property). +Resource sets can also be used to tell the cluster that entire _sets_ of +resources must be colocated relative to each other, while the individual +members within any one set may or may not be colocated relative to each other +(determined by the set's +sequential+ property). -In this example, +A+, +B+, and +C+ will each be colocated with +D+. -+D+ must be active, but any of +A+, +B+, or +C+ may be inactive without -affecting any other resources. +In the following example, resources +B+, +C+, and +D+ will each be colocated +with +A+ (which will be placed first). +A+ must be able to run in order for any +of the resources to run, but any of +B+, +C+, or +D+ may be stopped without +affecting any of the others. -.Using colocated sets to specify a common peer +.Using colocated sets to specify a shared dependency ====== [source,XML] ------- - - + - - + + + ------- ====== +[NOTE] +==== +Pay close attention to the order in which resources and sets are listed. +While the members of any one sequential set are placed first to last (i.e., the +colocation dependency is last with first), multiple sets are placed last to +first (i.e. the colocation dependency is first with last). +==== + [IMPORTANT] ==== -A colocated set with +sequential=false+ makes sense only if there is another -set in the constraint. Otherwise, the constraint has no effect. +A colocated set with +sequential="false"+ makes sense only if there is +another set in the constraint. Otherwise, the constraint has no effect. ==== There is no inherent limit to the number and size of the sets used. @@ -848,15 +867,15 @@ before it must also be active. If desired, you can restrict the dependency to instances of promotable clone resources that are in a specific role, using the set's +role+ property. -.Colocation chain in which the members of the middle set have no interdependencies, and the last listed set (which the cluster places first) is restricted to instances in master status. +.Colocation in which the members of the middle set have no interdependencies, and the last set listed applies only to instances in the master role ====== [source,XML] ------- - - + + @@ -864,24 +883,18 @@ resources that are in a specific role, using the set's +role+ property. - - + + ------- ====== -.Visual representation the above example (resources to the left are placed first) -image::images/three-sets-complex.png["Colocation chain",width="16cm",height="9cm",align="center"] +.Visual representation of the above example (resources are placed from left to right) +image::images/pcmk-colocated-sets.png["Colocation chain",width="960px",height="431px",align="center"] [NOTE] ==== -Pay close attention to the order in which resources and sets are listed. -While the colocation dependency for members of any one set is last-to-first, -the colocation dependency for multiple sets is first-to-last. In the above -example, +B+ is colocated with +A+, but +colocated-set-1+ is -colocated with +colocated-set-2+. - Unlike ordered sets, colocated sets do not use the +require-all+ option. ==== diff --git a/doc/shared/en-US/images/pcmk-colocated-sets.svg b/doc/shared/en-US/images/pcmk-colocated-sets.svg new file mode 100644 index 0000000..9e53fc4 --- /dev/null +++ b/doc/shared/en-US/images/pcmk-colocated-sets.svg @@ -0,0 +1,436 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + A + A + + + + B + B + + + + C + C + + + + D + D + + + + E + E + + + + F + F + + + + G + G + + + + + + + + + + + diff --git a/doc/shared/en-US/images/three-sets-complex.png b/doc/shared/en-US/images/three-sets-complex.png deleted file mode 100644 index 4318f6c62f0e72d155a84f9b847705ee50d5dc78..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 82045 zcmZ_02{@GD*FXNu7>s>S5@S%dEK$fdgG4DyDiv8OTiK%Q#*i&#i>%p_Eo9BU3<_By zBKs1ujh!)!8UN`U`u^Vc{a@F(T=P5^&wZc!oX`23&wcLag`SQk1MLY~000;+YhAn! z01yfQ0DXs2k*~lU9rMY*XdJcfI0FD3+n*m0knr|60Pq2qFRI>f17U03BkYFnjc+eH zd+;tRF|BO9c{mS;pVfv=U8fOX&X7Jk@SaxdmAa^gsE#q$Tg1t1+MOVX6gW02I`nOX znP?`hp)fzU5cI2Bvo#pR>fynvwossbx%r_YyDA(BS{|+2rH1Kr+dPu>K!^ zNYp~Mp^0v{`h@=1KLxSOsgiHlAft$K>Gv$BZyO>H%zUhI{kN!WVfzjTB>u2*~s||K&*$xfwhJut{o!BWd>^yi@p$)haFn!jWGVC~ux^a8U^)V!1Pe-;< zA;96rt#Y@3mxrUpo*^5?C8qq*!T<6VXPFbk7*cxg$`yY{>&(nBEDFvUfC?i+o#GIZ zC5CS2llG40GptQs$EBIv9*w`NVb%^&puxJ|me`a_>||bL>F-R+>S~|=fu!9Sh3wzW zKL$qps;-QelZ0*kwwQh5^lua=K}%eP&UrVZj;4+FQ(*07sP@pC>$0oJMNL1r{fS-{ zIw{5eV?Gn9o9KtYeo)*NP$qV0Ct5TGs#d%@vf($@?hbU)O#aY;AEXL_XlE*u=vLVm zL^fw#(!WE~5KQUZ7MP^iH$NlgLK-gwxDK%|o4YD3Y_l63( zn8D6>T!d0ft?XbQzhgfR!E$_(NEwYr{dE!xg zj1~|DR}J2vpO#%vuT+`{GQamV%k(gEeO1{aq|Lf7-b_ARoy+J(cY6bPw>GG-tjtnA zn;A_Q@*5Odc{Px=61tn{!%3psLY4Gg&0j-9#RNTtDa`PE%#A1!oca?z=(L94!ySh` zUD@@Y#t{gau~Jvwdu1!~MV7N!9f3hX>KF7rIiFJf$ehZLrbT}U^>}y`&s5rcqr)$y ziJ!1}KH)pIe%8Zo=hw3gC7dwK@dN#Pii? zm|b1=FUz^@U)$Hug&7xw8$aFn&NqW}^Z%THG96|zt@*0!{!7m9k>e2dM$?$LSKpYk zM;~KozuH+K17H${WDBAbulO3nBg81?!Wy>(4i`d73S2X#=}Tg6RNTji>Zu+@h#k15 z@3-uTd@2Opt()kuFS1wg@nfJG&<>Ecwpa+Clg=95)oz3QV4e}wtbmu6*K|Nd{e8lr z=Df1Q%rtn4G?>i@{mv!dr<^Bz8$~%;qI-8lI)dI|Ho^!WFRomuyuX5|W^IS20>`Y!hVx|~dPu7ke<3-}b zo>HIpUrlhD?3DY_5+YDOM;Y;rCi|ca%|I3&K$;xkWv_I&4xlU*0cOrNH=EV`2ubA! zWPtUlZF~Ab2jloc>fAb!R62sNh^=HU*Ng|0)U@Qf6(dH~=+aKn#1n6PJp2?zM87~v z3;D~apGb{)NrW{02316}Ps!dHe*b6TAvX%s=#fr_kk1GxZW$qMiBso1hE=vQN@K>* zgLrpLz)3L-T5vQQ-=Tj!hocEjHhZeza7O7P*P`IZ7>Qx`TDPett^-(x!s24qHgsjn zW1KF@i(iES3g-=Z9WuYq11Ro#fmRrN#;(g|+WRH6q>!(C8WLf57YDjdin$e{N z!o?S+c`2Zhl&*&0iI)^o!9jEmduw+ZQ}Eg^T`U!rPDO}8CNiY~+4giE3l57$@xzhv z7rKMzmyM@466?q-YL$KRV0hh;fJ1JfJ2Jb=3W5YhbV1P46Efv@arN~IcHxE{Cps@_ z?TpX?4u6biD|x6(}2HIZ@|CgrtXr5k&@V$}&uj^9YKMjN_>fT!7G>tDUQ2wx! zxF1M|H)fnLg{}@lBJl8L0i}~}V!T(-Kh^~oG{H5}LFvrHge8Rin(X(zWs}bogiTs{x;_9>1ixb(C7Qe%weXUk7WdD(bT~ z9LZmN=ddMMIJcKwZ&WLmPDUA#Wpc5fZHb6Ac-rnwFhw3CX zv`4u(f9C*ctyrA*b_bT*^p8b@?A@YF%2~{yq~M2n)dKXSU$R{lO5!8Xt|leTIMD2D z(l`=$>0zseRV)t&4tmVBq!zZ9to9-GYLdl*5iTYC}IA+7vsm zVImwuaGqYH4#Qr)FY`q>|1jkP(A8Tf)N00W!AGXhV4w|-u)0#l0%=-$AKixZjC%?~G|T3Prk`ya%>f@D6qM4P#GB z>vZ6lYm{m1$SNgRTa+5Wewc_-eaM7yv+Ih;q3Os3WB#TFU~D0j|oz~fyA-yg2F5tt}J176)+z8oB2 zv(}komHg@%Xu#?y^; zUHL_j@}@UChPI>PvtJwVjnP_0ySA%KBQePM2#s%&UE;=zR>^~vFycd|vAfjFxR{rH zfV$OG*w*8}&{TiAU7qQ4UlKCT?7y>vEI97arF=$qOw5uY^(u4|K9X+NtvuoQT?bUh`KOdr7l+hjFof`_45}aq~*xnD8s0Yix}**{tR{-ARv}@HmBB zD;w|CqlqxS4A*<8QzY!nIh)0bUb5O}Q~w#{uLium+dz-SL@4E0)5MJZGz;dnT9#$1 zJR)Zw$WD;W8fkjKI?9lMn(+7yb-gdD4IFA%D~AN{r}i~h*&4+lhQd^Piy-((b^gG$ z)rWlR)rn(zAa;HGFIRoFv`?vCIX)R_yFD9jK{3`~ZL`Kfxrw0JiW8hnq}-(k6;d>P zAP?oMa$MP3>rwsqKkGsd7A>$C<smuxhX#4!CX&0w9MM(2^o*cd3yzN~iO~hMhmU8h)#5 zwW6M$eUA~)KBQKu6lB_&l01Gu#z zYTc4J!#Go$!Js;6eWz28oieVc8km;Yw;x75{{-3c-BJhTyRSHNsu@EfwZA?zb0=|8 z*O^!(akwugGvVpdO^?@T{ z{kb{e^Le^vy|Sl?Jiz^Fit>M1%Y*Fad9(18!})5U{Z3Qf6a_n3VA8+bBybk!vbTc^ z#;+sW=*a}QajZo4&Z0*B@l3R zK%US4Iu{be;gG+d@>6rELAqDaSJuQ!y*rfR)?LPlkIc(a%wf8e*KY@{GDiJ|I%XnS zyEo#9Q7LW!TtwGQG3|aq0CoE@+Y2&$DLi}u$P;Y49Dg6^8~TSqu)x0@RIA7U3IRlEj|VMGLA)Lsk4J)a?k3UV{f7Ykd4s zOY4otMTx9q^ZnF)sdniQ7x)BH<3U{X7Y?DA6GXegeon;gIv2XUXCLP=8uN4S zG4N4PYJVlcFTy>2L~WJ#omYq(?c^)<#t1YqOB*$=AsO-#Z`F#JPGoFObd~&Q8x_l$ zR>;z|uK(c9irKB4h|5r7Q(yTit=+0S`e*f8&)cTVO$eOEIfByB&EUYIjX0*m#sB}e=XCV zK^gLj4RMybvO$TW_qb2n%Z`}R6m$M{eAfO4b6hR>~n2v zDfU|}XQ;v-*7NhA*3+#Ki<<1CwF}`@>4wSXY)htw<%WS(dP4H4SF0u~luXMXZ!C2w zuGf?j3_1jnE?h@khOB9#sr(Ki!S*%af3?m#kU;~$c8&2eh`dRpI&IGWOAPXuc8y0= z+sWy@NWg@2{L4Bx(F6G!-5%HT9gbMgY0etp+!8=E84QPno{yi7?vMqSwVZ7Gi1wlB!(>9C zPF^Jn=MR1Q|1prS)hT4dYnPAo6}8>PiF9A=IKYpJ26>mV7F;UNk#S7dx!kke@!`Vh z`rqzOuBQ+Snw~ZTxSDS{kqNbK4?mBgEE~gg-C)L*EQyr-rjh$O(%p#ga7qT{1)qML zYa7f?HMYC1u+v_=AKM>=I}`drn$H1or}Y5(eV<}63S)C;! z8tBx62+H+W^}^KZ_K|^h1-=f|TvR(T_Cs*Drd}&j#Ot;5XnyY-32zAEoI+TlBv0`Q zsvWL)R=N1&b~te(xM0+%INZl#47J4MFgRsVP!?FV!=={rC9hPsNaONePA z4Oi**OOaxh#)5Xhai1O4(QiCDDIU{1ImvS0;%@&mcRzre34k?1dICs(`s~hsaKUU5 zBK+`o33!6E5!7_qB(!|SEp1s%G@6RkrSWqUJp6p3nH;l0IXDrZ(e5)`p-n_3CrB?g zB=Y0Rp8mrRh=ui6t%zH4gq6D4JmQ4hNsW*mr91WmO<&F{o|k#lJN1mP>um63FW)t( zaDOUF4x8fryG;C}D$PVttoc}f4BfQ#Sx@0M;v|wWf2;LtxdatzJ%chwWW$HZ`WArTdQ?oJiwr23%1US0* zky}9hM~AL{RCEBVwuJls%iZhyuSFZTSW%3D_s$lc@SdvlWuP!B0fPfj2czdBr|?wY z!@Y(?KA<7R@AS9|;;#Lm9hpkyUowINlb?iR{e<_JuU#1AO;!t(=#^&hH)_^6yKHV; zzoTz7{X`W%Y?Xzb-QT5DR61+ho_me(b=;$m)HnMvF<*RWXq~O3&%$T8&N4JcOqO@N z9O1JRLfoz2@grJ|Z@Ul_VQxX*?HK|%CQ>suc78Y?X+gic7(oVOqb^WrhOL6FP zYo9DqI^!m6%M9^6t*V-DmGH!~$X=fF!GpzE+6kpT<0tSV<)!;{s^WvhfWs1mthfaD z3`kYNmMeu&iUF(3Pn1?Ol_0kQIb|Jct_H31znTw`E_elMA-Q55nJTdkRBzSckTTs>9Xf7($-m!-w4 zR4xkfiTYV7wq?w{F-(VHxaUz5YC3kmgM=4z?)L|qtwoA~YSu`1J`b%1GlX9oT>Wg$ z9j|o1N&d8B#{e~Aub8-;Vti>4>rm&}zaX{|Sw2Gx*~p+Iivr$(v=Amv+eMe{6jNob ztsPFD8pRzfLuvdhz5Tz;a1q%CghCF79t8wk5d~Y1-_7m-udd&JUi~Zd0_B5&nFhHT zXlpd}s~|A2{GroLl>K$#(u?$0KNx?j4}5=7MWkL)3Ve;-C#4U&>@?TDhu(I3nBVs$ zPQ?_KNZ;V|aMzzfm!~9h{((KrHL>L^0fAV%8~w!Ese`nh7#1jHhCXG=QxwAJ z=~?^xis`N$Vz`5JbK|>dGl3QK>*$a2eKV51^%}f;-wP$V1al%?&qZ1@4v%`07(O;WxIakYKxha}?0lYCVH1;{X z>*VcEh}t?c)9MQj4)V$mVE8h*ffj_%7Y-WI@v{oci#W}wYTr*gW6jsfe0qs3CLj~t z4y5n`ZF$f*jRjJmLL0=KrH?oi82;=L@)QZ{9nh^Ih*KJ?{qB5S{dDyLr<#brn|}e@ z@5z@?=v?r|SAy0QfwnO3I-kUP`D4i&P~~;adnP{i1A$C6L`TR}F37nYcH8;O?Dqn3 z&D*1~lxCNac3wjZ0&Egygh2G#2k(2A`<*fxBE)D=(>^-}xdqB)7UNbJZeH4e-yvNR z$T5XFDLvt{N)mG4%XybHhdObH-B+(DTP9X~nPSe$nB`B?GCq9!T4>qvNR(9l!|zp9 z2n*H!AsUz%nrXG)3wPmdYEoOQBH3!E10!fD0g}PAg%`9KdP#xlefQmFPJSoEPWtg1 zbAdbIcvj!}qRp87Dbt!60#W>KdwARzDFyQ;J#d9us){t1C)GvIpAM7Zm+rK?VjbM^ zA@%MKvWB*u2=Uu66R|q}32&S?TaPrz@We$rd{o6|FR2p;+lr>HT3Yh=`O-zn;NBx? zbVz1XuaQZBEZt$LpDG5Tjj3b>UQC+W2&Qp@mm=D~z(=alZ8^JX&{g^O*WP^?c62+@ zTEr4q!JpSn{~q#cyCQi<`5yac>Ct3m*^q82Lwn^{%HX_7HLo*FjuvQ12FxpT-15r& z;xmV{8=ewBsGpCSyPyk1R&{B0;z9{ol%wN~vD99&AeG^uOs2};53*-I3J!WgH#Fh3 z!}Tvn)K={AA>j+xiI=(1E+tDh$=&usD77wO&MGRj`mggsYewJNj??=Jaq5JZMTK%9t#w~F$BAKApJcCu26`^ zJ$yI1dB|iyEVnhuHhm^P%>t@>G{bw?3-Fg@jY061T(SY3Gxse4CpF;!`Jtz2+0{qUJ3H;(C=Y-Qz_T2BCSqw`f`jj*7&wVU>!UfRCkBZAJ4oR z1xbPIzEk-DBW_guNLK6Kjt90_40$V9`DD*LJ>}>5?qu0Bu_dcS^X)cWOaI`^CMjKE z%MloD0RQXRGpq|oTS)(J0wo(!0|`xPU3o1P4Lr^O^U-u&^D*LiCz{zkWvZ2bn>FII zVO~~{*sqb$A5yEDX}rufa;+&&@AtmTcF9eevY_vn*i}r_QPB*xfB0fVysJ#J8tm{B zcAj5&t7SNNSM#^CgeY{cng{X*Qi!(?9C#* z?!Ft`o+gNNVl|!aeZFVOP&*|b*t&2`qmwy zNWXj>jCct}jW5*AxP(iK;YcyEztue_AQS%u?g_mxzKuw4PJ&<&EL zzh`TkhN3>zsN?1bm_;G6BWiXfi`W6nr+%47LahsUe`A{cSG1ZvAQv6B9aT^P@hWc6A(6=V)Ka)gy|rF~g3fgK?BJw7Fl;pD zsOw8AWy{b^F6~BAn|Cm24(fcabpMHc?>&d8dbq#oU|Jl1mdSc|ox)T$qKDr>ao=Sd z_N=Ju2`_DoJg1VRwR2yn)W58YTURJL>P#7cfS~c`rfh6bZrY$ejb)b*HDH@v55P=6 zfnU_LHp;=sEonH$qtf?oNb&7=YUY*|$Bae$OmXY-wJvVYfboK6c;jSOxdq8sM5^}_ zw_oq_ZJ7LGyk`@wu=%s6Z9VcV!+9l*ocqGhr+5PyNPS)clf#&1mE0qr`knRy+EiT> zX5#JDht=?zS>cTCf`c+K7@(O*7CTzX?&nB}8;wb@I2WIX^pDqu)kH!XjkXFWoAqg1 z>`sB2;$qe8ribsN$~Y_@Y&3S&%7*{K28|Oe6Hju#=%Ow&b_-I&+-CxXKTS)q3;s-M z!o@x@87uG5(b$A}FGY%hkL6d^nB~8reEjjuE($V`DkZ`j5xjFpfPTym9 z@DZsgJ+zelom2Adb35HfSk5R)CiRR9@rLz(m5JndfU-R*%I1nU4_MeAb<#w^cEoz8 zOwkrPrg5TXRL6X&)P1Dn_jVEp8;kI0n%RlrN#ZSmd30%CHZwK)6a)JxzRI4XycUn_ zfMA*fQPWE;!+Y97mzKV;O*LR;Oe-HerVGz2avV>1x$%*Hbq4zQ@%PvpVcV5vXXi+> zHQkqXizvV&@|;pia+BY~56y`&fH;mvABFGSbAa;dfRdC902a>54?>5yMN?0nD9(X> zO|h-m!1tK?%Sa3+cERjkR2kvkaBJhfOOC9mb)c|>G4m$Wtiz5KmC<7fOq0FiAX4y! zOA)HH{-!V$X_)ElONeevKY5r5CCxoG`QlY-7%M7#xR`uT9=cdE*)?~`ikP(SoC&sz zkXYP2)?A4(WG!~~58N2h5ZXLi?ucmoufz1h-wpFMZhe)sb8k-R!cAzS#AepLUAnY- zJ=%-OYStR!UsD#^DF@>?^jPfG`7R}kPwSd89xrAhE}wr~q346cmlCJyw?T;~p1{X{ z?#g*}BG|zr2tG?n|Mf(#0L~e@*`tGN}%z%YPPL?!is{u&#brAhz0IN?8;#M z-jA75L%@>4Z9p%7xpBL&dZ%D?*|&6jv6TQjSr`p^)*@-kNm6f+VxI-i3jPwCeK}jy zdiqdyXFT`Ei#O*DXa9P!tTvy|)Ya_{d_+>*>L5r$D@adaI(JK5-i|IWTgZR5L#LgZ z`EK5@B;}Lf@}QV&annz5XyO$|a<{dQg`e#kG8DQ3xMFVxWM5@u>Z ziMbsd_^9a?jJJHZHM{=Bw%uO3@>uJ;o3^_3OAccERof}VD;DV~w^=97dyKXSod|RW zrKQXU+LvBAk`Gg_@eu%BMZH6fo7p#u#!;3W1<1)# zTSBeZOyp>i!_d@BW@mCjpOD5qvxCa8cm>du+<5%kL^ zBWVmFR|PeYg910^+g{9F`~b6?FDFJGLzsKaYTCNHQ98&kBp$b%^``}~e@fP7{|Gy% zm*l(2`Axn%%QT7eKeGD384&c%j_TybYr1znZk>oxL+)Zr3b4`*(APku4(GyipdHWb zsw^GA5t-2sR1Q74+AD^5bmcr}J?VGNi%#M~6THUMv*NxFVOGTwv3t* zVUiZYZ>7WR`^ytiWyN~0FShYNi6uR_1zhc5ye&uI@AOBjQH3yxg4Q?Kx&HP2=#d4U z)Qj+-1K@fp1T+-CT{a;%3PI7tOg3hxUPQDSs3ISNe#CceR#T}>?ZgmNU7~+7j2)Jy zVGP@el)f4RYjke^xmCV!jhic}?7XS-em$|?^phzmAMuG(S9dVOxdl62r0!zv(fD!X zPH91>a#5};XZ~agovm#Dug9PP+roKMK4juzdX((15OidbO>SR{ahKGuw49aiq#s4% zpX#mlYjp<@fCA`-FGZj>^_UyDgm0+id^WvSj*_x;+YwX#pNbE=gfn-eRjo<+gDBD+ z#qqDC-(IXb@FH1acBL$)d{&Io@D@iRZJ-_v;HUA$LdP6T|7_n?eYUe`4CA7K^Q7ydkMZ* zh;J@=oEANKVe-N|9`UOT0S1RbeV)b^N}fxlHiCo39FlNKdAqDSSm5pa_*@bk6& ztMRyGSU@#XL9r1u|LVT(S9K4ac{pC^dxHL?_MLkXt|>otPpL!ro@;mY?Loszd8scP zzk-gln2>?oeO3x{*wyrA!#vqn!#LT}A1?tS+BSZy1(0X3XI2H*MPTResZZ?mo_?;+ zgc=c?p}4`F9e!F=^QVDZ%97-U08Gg~JzU<;dmxy**a2#RxqU$lFjpF&{kH*aN4Ece zB@|^EWQBiP#Mkm}J;;Ro`I1MoOy`#tD4^Zq;}w$m<_^EEc?LIJ?hcopxagjo84;na zdSVT}6Uc1(eo&3%CdQN|-_|4<4YuhALl$?lb#qK}joK;qn&%S4D0-X#)P2WRN|NAa zwKO?tr>31Z6g0b}3dQLKU(C^{oRhR-=WCEDLv)2(FidKS)i7mWh?e3-k@T9roz$V_ zdP|+mnJhGveDl!y7gPTT|AGG{{0C^$Cc&eR(SoiT*5fdgme4$jGjiT+by_QaPw4#Q zArA|fssL!Y`D>xZ2Vm8ZYkr=QmQdX34Nc;;6x_<`E|?6o#_s#E4p!?OdeFFW;f&4q zGE25Lx;c=sKKhaRoNES{sKt>K>`wJvNAO}d-!PMXT6eHht(OD!)gMz&3e77n@)<%> zSCn)`-k=Wlk&qB4vWch9O&==aypVpHEapo@vUHCtH!&^__kQ*x=GLRS$Mppdw$Ft! z&)s|aCBd81Tpbz&M)#Z4 zc{l1}R?8wzeAZA+O9%Xv^{&1=WT;R*GZRhUFvQTv-_Mh#q-D->7r(K0H371hKf?qX zE_r8CdcJ=u9pOk|^}UOw>9+S_T~+pm?zRT0*qRdbQBRc@eWWxyzts&|56Pt%mBqi9 zyq&440esU$h55ZXl!lRzeim|fD-YwLYM(0ir^aAc$e};Z%PV`*XNG}+J0~a`o4DFw zPxphlqKRGY>D9rY!bQ!iRb=ms6z$=Ypux_tKId2NnF!9=Ol(d0XGxzSJH#SW21MSyGP}NwX*PATU9$gH47StYr#ctK@(n6J-dX?C2rK%i z!=5qfBqTw{z58*6yoG3do!yCJKQADEzsDiIYw?xS@z@z% zJ1x7xM}$bGnP?eiz1kqk$F4>D6AHiHnktY!U$Sdh9A2F zvJ|)I*m`(@n+-Zy$}BaR3D8;qVPC#zcrK>V3$67flqn9?cDt`9e@rTmD!G}A;iXdph<=e30H*YwqJ-0uOR=alfd+~Vo zBD#VGLA|}bxG@0+LgTX%YC1hCWC-rsidv110gmp1f~cPs(`3R-Gb30J!p9z&9Jh6v zQk6Dq;^dCflF2uwfXfTiRd0jQ+@yx)bwEm<-1m9U!5;tm)!D>^gpdrbI4&tUIjg9s zDA}`TS>}OTS3r7)JH}+PX>IxcwjW@O=EMv?4&)ZpJ!0Zjf~K*xQ&!vn+R!%RGot`P_ty6#o zFm{nl&>%OUJEfD&3NAH8fkbi1gnJ78&{L|c-S^%ysmcvP+cj=|&XDqHUn!wKKF+wk zsc}M-CL&M<+&xzt2V2TCCzuOm030If4Xjrijv{LxpDFnqFhL$MSB9et7B3y6?LgPZ7lKXHrzVf(B5b&{V&6 zt$mRl6!!x3+l<&qC+Q!`aMEFJ!Q<6?h)Tnu9xOzjNB z;0y~o;)(@kE#92q>u@_v{^~4~`Oo&OLS8?w;6~oyT997(@PGl}u*Y@GVgIF6rcDb+ zo1bzFjMz%hMcATlMPBi(U#4$(T>`EUr08g1pVWh5!sEFIZs`60E{d%d5CVAhGd9j* zv9=vhv|ih_CHVjzwFRTD_}a%V=X>stDl+E1n5}Yy*4PaB+&#@aIp*hP6c$s#iave4 z0`W!FRwo4btxM~7XpVGfh?tjYaUmit*PRRJ-{QyY!OHRq5xw&i}1WL)@TbKghe(mytWy5a?2Bm3#?!2YE0u1wq!sW0Ffs&rSB-b7in*AOmRkS zUT{E+lb&03asa`W0|Dul+=01|HU zToO32yh>j&%zL)jmnRx9yZo_|AINj;MxJ$^Ze?ldk@hk-d-m*&{$VRH67rVpS|_Lu zT)mJ6%q+TgSrr-6^D;;xXR%Q}5e#_HQ~Sx;jl zqnFP8n*}nhM+8HVOpmMu556K($OXWdZ9Zw#PI^S@(a5~j;Ym8BFpa$rM8B)P@P z5vf~K^*6T{UH~)m7m>VdY(yX)(D<8^Y^SaxGde|lCZtg^yDyg?) z(bd;e9n)XEXVy~lf_{Tbt7Eo(*il92fMvI}l_wd)-0bGT&YCB!PmRQ{UM(LEKK>9@ zru1oa@s*2EqM9-x1q-K$f~{qMICX!hM|8abO}~)+O?`LlE1xp%De~R`mL*Z;Ukz9R zHaXn$M?z#zqm$-mXJ^&3{T*rGC6CmCaY))6X5?gG=;jTmStSrA_x9spc&+DmOD_c_ z9FvyjU0LQGz8}?J$KM5wQ$C6lG!uHl?T7MLOAYE~GAJ^T=rX)$a3NxyLBi*b?jwyy zLjJDXn)M?bAwXfnb>G5CT<3ms(A%$Sg>WHZ3JfE?bLRA`M-$7IdW_L>QUjZ74bQ1Z zUyB+aKprFgpOUE~=a%N*Y+7~&7|*Ns+_rz%Vzt=Z++1}`BQ4tR5wtcyIyFjdA|Oft zH|1k77^Yi|T%1>vl3=G`PEp4PEHz5B`SYCS+*Jl_)SD&QfyfW+yDUtjNLGCBYd`kD z;yIuGy)jHSwZ$dHTFk^mnynmx#arE{dv?yXGpviAl1^&=Jv3FCFD~NwPmSN%<%tcP zyoD(g9iiNE*soDS|2<*|`6SmwISf5?np11e|FvqM$u}dy!ArcD@JJpnh8&=K!nM$$?M6QW!7pkYHey zUy>*72AS$Tzl2>aM|_Fvg3giW!itt1DvE571_`^Q>Qy|Qzb?!p+m`Nj2wLHgo?hg4 z8K2`?x*UX}tcAm4d*ps8$ktC2PjQ#@7OMrkWo62JH|-1tU*6>pYUnGzWR7EW*W}aq zPGh0>@5fD%Wt5bDbhvB6l)sLJ?mboi*u0Ft92W#IrpebwO`T3*rPbMQ7y49o>uWv^7aVNSMPsOL(xf(#&P}(jI$Ze7J1yv$hq- zcc}N~D9C53V0(dMSAh50DSCN>(*qI2PY!p>%1w@kT;s}tpcyCursJ`pR{s4fdVfMz`s8*f&VrcH^B7Q zSt1nfr=bTI@|a@ahug_GY2RnAo<7#X%Q(+U?MHuiz|l}8Zjjf7iEl}YYqB0LE@iWU zzQiA&hacMq>8+IC>!3$-%TTP}0lyVl6)K4L-h3$_TOapGjjMx>yYc)sCx+W|r1gn& zIJd_WBdc2Wv3RCQsJ4xMUG+%U-Xbrn-p3Y=5B#u^jxOU72#bZT zayQf6LWUk*2C-sn$;Tbg4MLEtEbqj9(DAYR&fNCU-%8|skp$B{X1K&wfHwjAoW*KW z89&_irYMXB9&Q)bdKV3vJ>k|B1u;$8@wmFyPZ<(hVcG!S^sFxMr5Dr_1S~%p*pu8& zR}u!|VD|4rH-p7S;2oYLtE&;-iy0iAT|dv;KH~QljR*k-j9CG0_iRY;dbd~U*^(w1 zbjqIB)W&le85s)Un;Hjs5wk3$H?nM?PCT80pv98JGNKK0kVlgrf^Jvl{*}3Egh8WAC2$C1SYB64Zi~< zMFU#88UP2)O-g{MzY6$B(LqX7^+;H3UTAnYQ{hvNL)u#%g(NJmENui)sc>Mw^?$f| z7|7Qke75zw3%PJKOvP(}>__F|=hto}4m)n7ssU}F(ic}iL(nD^IA!N03t`G|u`HyP zlDHhY*E;pFk;yd;3RPr1mByRXW~5?I+?fK1UtM-8cvjpC>G{;IUz~#$Y64e162nCw z$zFHZy^hYT-3ZMsk_0#aX8Z*Nni=v+ zSZi;kUI%DFqLgM`l6ix(l7y?iV$Z5c7)qzEaBlZAT+j?OfjMS+S6XHF=KGoF$aqNP zW`&Fj(``KYB%*=*IS`c2prY%f&CsyKIV_Ll$H9(=2_)Ug6m5{xDe|GA%aM9#WLCkh zS7`K}Qgy52y_VX{kq8r^%gSgHxa`7cgljAx?#yYc#!THwuV@q zLRZYr#;j`%gJSo!oA85PdGdK6fZ=bOmM_J%3HJidtcb8EK#us-^*Zfzh_B8vA)F|_ zy#%jzDz`UW{;@=26Q+0<_;6~O$pXN!3dcg-0RL63*q+949}#9h#SwLv9|8L>TTZyF zU2HN@8g$y%s~k&dMELKGTtfu-46)5r_F z-Rn+MFOu~tKc58l5K07l%gq44xHJ>pKgjNe7LH7V6)1N;Tx+OH4BdS7z~l$W)!uNF zrBWMZKUQP2y*&DIhOwliL{>qeO8ai66uRbDC1DyAUB91tt4pwW|x5= z$nse2<8u2^T60DR_ci66T5I9f-b=8#Z%ZSiBG-7STc*bc}k~_H<7{r zm>{b)CsJIL$O2i?1!v2GXR1qw$tsK754+0FO8M7R7>(QUv1vhL85qU7GA^H#oSyy`eHhZfJhBS)7eI^i%qZo5;hOeBkO4DDCV3XRP? zhUz~FABu>c(PYbdZhp==*jga64A7Y}&J_q5Y2UmFEsVCGe3f;N120EwwTJrTFkWZp zx;*mYno!Y?hqku1^)5Gwe_zuuFiWnr2WEwCkN$bN#h>nRuwqi z3dZ%yAu;ckxGI;WgwC$6;nvn*Rk)OrQjK9kfy)7CBQKk6qqK8u-yLdrh3CP}&dzC2 zbfL>+Bc%wqEKt8v8XPC`WO<}gPs02`L#|=L36;rz(+T}Eouyag4fX%@33&fu`@BkS z%!y;FEa>tIX?`gv_|si+96kEA&66kJ?MEuk2H(FKc;E|IVRDt@!d@==_Z1xjG+?|F zpaO?JFRx%@V^h@6Uyw&aQlXgBetPue#cnN(MIY|*V^5LmVQfZ8M%mxFkQEwJMofbJ zaj)}c^^CjsB>Pt(=eO5G!vArNLO2M!9@aThn8U5I z9Vk~es`9ej-24h3IFZj0J8)LfX*$@PUX3f@?2*Z;lgK8kCer74ADXPfeZ$i0LaIqY z?Bhy8TlPhMDFAHs?QPN5ub11cvglK9a~wkJ@Fg<7*s16G{VfVzLNK+S+gK8jaFz{7 ztRf$|Q`9ioiU2IrCUQB_)6<*1#iR_7hoY8?*dJpr#_L!AO?tchtJ7cEn(}}+vf#1M z4)WGFc(=rdxg%1zA$iwU^PcEm`ye67Z^$F=x}vx9_o>O;5fDphSkJ_u?QSri8 ze$wE02|&#(0ZS;xvH6PIQ3px&t3Q*)`e$GGJ6R%2>rYEvUEN`F?MP_PzQJGE2`O|1 zEORg&ot<2k1&4A6#{SRSV(b+K9{uf{!307Ij}eAGJr0Zq;hp}Z#;qANLyky;4lCv- zrIZmRm1!kL1$?whiGNlp@mt`~-(zOooFFXO9rKOT@8!HPt{|I0_p4ZtQOR?LSd9lkEsFQMB9=X5`gO&I7 z^~J$cCFzxd*wRG*nn&T!#<{>T%D--jy#yxXN)Z*eT~mT>US|x8@90&aCJ!*W%eLv6h`~3diYu7ujGmrB;kE<+|Y{h~~ z#%DXtO1mb8bo&-TS|dR?p6zV_7&C;^$4bVpzoq`Oisy|o9Mb*Ty%`>E3F5>BE-js@ zggQ{UYL5B>MOID%oIaJm!!v6Nyr{uh1CE@9e;45-fZuc+!}ouE*Zy}C5AlDMHw9h! zgJtm+-4OSj8my9OLmSnWaBi&NKDkA^k%`HSqnTd7zo$K+#R=8PaQ7?28=RYc#ia*c6DKD(K|nH|qO>0dZ58%qt0$SiynM-gCwk^TP*LqoR#!WiZSv1>@x|Ac zFcRwR1svvrI9wlOu_f;M)MCyX(l1=@Wa;cI*d!97OZ{Isr$y(6xuq!T=f7}cXcOs* zd}+31}Z}mSYXyM%!76=Q4xg#U2K>y#LZ~7N}K2&V}>ws{Bp@kaAUi>t< z*hgdM9!eHdN%j!B7BMD)$WWO7@q4hEC&c%8|9|>X2yc7^A@6f-`hR?d($aE!q^QBY zz>c-|=N>J}GFzM6WR|jhM@L=)kYzFE&i~G%hVI}B$1Og32I>EvHsgL-T^*Bte!_Fy zQR(F#F7OhSu@d*bErYc5^k@jA4l@#;baxv_|04;hI5n_L&0>=Lr?ZLbHZBFmbA7^v zv8=CZVs+CSXgD{17%vO~9jrw4_xBql{%^jV@aFBb=oNGRi*fhg>1t*bk(G5Q(#>9Q zqSX*4xcJ=C)%9#UH8h<77mhZG;e_np{mtIY%kShAi~TE_QV&AX@g_?xAh;kNuT2pj zch-ohq$NC>w6wG`&-DRSUDx^gyBY7h)XvAwUBp0<$ZUjaKUU7 zYYsja0UIf+oL3jl<PuHgSpkGREcv`Ubn z!0CROD_JHkUdW+e^7vX^T^%@U=>2bX!*cyb2M+rT@@{-0tNo1!Od)Bpgy4a7v~i|8 zM?4f~p6{hlD$%+fZ;tCxc(ERZq!;G^`tazcs7q?htJ^sY8$)8XZ zJo-tx9ZY~}!#f#mX}WjDD{~q=IR5nCtKGS2)%T%$efIC+mGShU-$+|nbZ}rrU%L2x z+_esogf1`R$|FjmArGZ$T^Dz`Q@{Kt232y!nVh4wJfnY~g-=`Oj#P8{5Fwp8#fwt~ zCK{u=o}8pH5C~$CQZD*Rid_-x$9eu+=Re+>1Gn=U$Zw5*tNnbWOWHgO_J8}=I-(gV zUFB{D;iS~@E*vehDVg{t85kI(3`1W2cS`xj2~tF4{=4fi-W*{PimV?RfI6bG!Or14 zb?Ze+0eGt->;EkJUEEHc!oME;yYnMeyhXM3e{5rcFv*dsc9aUPF5X9#_VsIpTAr0* zi76S}m#X96eST^g#VL>9*Qh7|N(8YI9?PvKdabzdibK%}0pZ|K`tkS)kqX|0v*lv! z@bHT_=H@kylO-LC>Hl+Owl}_r*+k=Cs*fSY(92$YjGA0ivJmcuMPvw${S{6YL#}YX z7!EpITe@xdpX`C;!;LriEAcb&=A>nnL}f7MrlF$fQ7+eX%EX* z+mc@Vjkordetv@-QJ3?rr7j~MbW>ujkiXSEoHvtR_0q#Q&Xe3jV~~oKpVQQ%-P(z;!;&` z!(mqoA3ZL-mvZ6hYaTa8lBlG(<{@qW>T15t!vorBC@FBdzT1#FzqC-@6LR^-X~h<> z)1Rf1l~Ww+!8FkuE0Z3knlo#(ML!@i!ojog}~b7fh#=2W8bju#L;dwqguBVgdfIJ8fmi!Bw&5BVLYxK?9vXs zI0RGdsYO{A^N+fa~BAJw-#NrI>T!j4T`We$g>}`Y|FN@WDn=w0bd2l{v@p*t@9|yPF z4BibOa%wH^irY8K6CB0fg{&Uc;i}Ea?b5=;U{~5M1lOB^a#G;@)s)r^iC_ehzxIMT z19aX1sAty2jn^~Lhi;}4zL(t5mglR1_AE=1xf@7d!6Z0 z(hb5hloWTTUdAb4P`e1*bKKFSS#@EJ!!qvxJyuN(edf#KUSTJ|+I0MR9acFb1>>}n z&N3I=#oEe3L-rq_|GaVrbcy}S$k^xOaJg6SAZ~iHa6iPI2$!aN#tiRD@f^Ap7C=); zd|hOGEPdtbW4i&FXL>@tz4IU=^V+%lWySy{hcdEC|G0mEM|KUconub9w}hiq%1+U1h-!>Vx)Qle!MR-l(5f&sz$ z55*$K)wr0Elv7&60DZ0n?M8MB5&_x41*r+jZEDwZQi)$CNoKkBlk zx(9*kqT(JC9wc@qre(KW{@MRH>>_c`{Vi(HIcufi*L$<=AZ%o|1&v=q$UN%%)>Div(l9h&eQ4q|5>EOI@QQ6WDlC#~&7 zP7L(M@e!WOf*SmW)>Mgnc1t>eMlRFG9b9mjX)UbdYri1n^woI1pk$=ej~A7z6Y+x5 zcQVUOdfj3suiP#+>_-P;&P0c+My)C3rH3oHtx?Lnj963z6n?f8R^~#j@I9ScwyM^r zILQ%CY7bVtvTQQ@Ww49o42kV7V`pYb#(B*!V9}LnB4(*Q;J)*MFzzDPLZKWR_?NKd ztJ^+7IEJeR>X9{tFF`YWgu4v_bymeb3skUdDBi%%YHzBul~pUQ@ai;imq3P5?EX+J zUMUHTQW|nu|3tb}!QW+1&0zV>!+6Nkq*nK58P+1TlI#~njlX;FvKls{!hk?4uS0^7 z-X_>kmEl_CPOl6*Wa@Xp*JauW?=$P@i_u%rE;HX5iLq_8LM!X-T@Hp+4K}v8VV!_( z$Y^4<9-bcUQ2!dCu1j9rWh?Jy-)64G$st#ZXxI!$*@_AA&gJx8YRo7GthoNNdY2Xd zGFGpw1&f#FFVOl{|1y($jdf@IrHCRLrOSsdEGpW; z)%RCD2hDd9XB7Mh8cuTFnfo$zI%>Ea){pk$Ug*-`sPdx(YhC26a!ty7BAswi-s2any*pHN#t@teymCOBHN~< z7cnY#s7d3Up-LJ%SUr0BWl54?+Xja{K)utV``i8j-E^z3Z1e;M@`iU-aGjQ7A-wTY zGmk0{i4NSOW@M!#6-SpQF>`04FvItM{)L++l^U%646KR`s)bB)Cw;MqUK-p>^*Y2( z^-d7VY;>;>9TX@a;@B!ji*(nAJviJ58mLyKO4S=qT1w1)kLH3}NCu?5fgF+FKUmY& zbXs#Es6GX9ofM9+x_!ybE_cfkp?@mvoTX&0#fLe3`{ZI+V^mR5+9|tU`faIj%%3Q` zN|W$AoF-eqAH&!TY~E`iI3fh`Q1$lwwoA{8Pgu@-pfC6FW*EnkM%ol4k_t!rbtJdv z5r#VoMS<_P?FHE*zR@ow@RKbSzT`9{)$4l8^#04VF#&7z_hgrud)cg~Th;OM0THie zT^_paEhyl^#6ki&4aJ_C=rX#EaWp7(8s?rl$3`rw*i zzR`vr7?bk(U48q#eqVuc=O~*Lk=1D(V6D{?JIkeBNNb*H6RGI9YbIKIw)llD9}-wciQh>wQDNhn;s3Cbu;DdN*Fhi;wN4%;lp_ zn~OGWmaypE4KN?+6A~Ix_USn4Wgx+Qfxm!X5$Oc%fNcaku5=1FU0)t8Uj33w&w#>> zOxe3osj5Ja2u%k=Qpd;7D`g`upaIaAw6cRM`3#%~YZ3jg5_q2?^l4kR%QTyq0 z(ecG(LQ(RNe$7#A0`cH&(9dr~A!XTGFKIY|qlIYtMJi3~GsBfgmf~V>>u3( zSf1^#Nsn;-44Asm5uT=QD0>^(sqX*y)_0nvj$1oc##OXD;xYs>gCels`)5Z^DVLo^ z`?7WWwUe4{)fp_1{=U8`9B9|;IAOjwduSgV#P;M0ja-r>YXCiS7Yc9jiG2|RyTO)i zFO_ITZIZ57CvIykR3gi|gse zI4;v5(N@)4)RJlV@f^**)?Go!o{tJf$J+Q;f5Up(JHuW`bUa0KxVSp>-X*kxuZAw` z9Qc>NYY9IfRo_~U>qFN)YIv@zm(SmR`&juQFq)KWepK!8lpb-;vl}i`d&>FXcYWKD zLIP6(12s>$LYN5vWgkOaEgyM(o4fpEt5)FFP4OY5vj`#@JE%-iKxj;i4Z^JA1%BC{ zj#LU#c;bNG!Z-cgZO38!h9BRWI-9hkFB^yM;+p&#lcw(VHYV{mQ?>R#9o*Tm^BF*o zdtRJCoMI8{*O^qzH4;HjIBidz(^!3ilrZb6;hOk46h_9zWHv2xNXZM{U`5(&;*V%Z z71v_>K+gxI#RbNejATjS%?J^;R$LPue>6mA#VVae8XkLqpcEZM65Q=6LdP9&y<3hZ z^lA^6F2)Ef>s}V9%8NtUJ#K2@uZUZ0K~;Iba{Ksc$QVQWq1Z z&~AM$Z(^D(#>r*4$6*K;laVjn4B}VXxAPhnI_ti6kcKsDs79s%xb8H8QMrvOrNOV0w@FM!L z+&H8^^cvHe)_B>Hygu41^+FeP2)3CwSKC!fKN?j_>zg1|8pbth;417@1qB6u-|&N7 zM0ECZpW3?8EiE!SvHQE|PVy<#?p;nz_zCPn+~Gk@9xEwH_*a=qieoLEVg1Msn~UmZ zrW%W8fK9LCb9}sIAhxxTg_?yqZKeNXQ1C`eJIO(-DoZ<)OIi|v&%tl%SumDQ`mR|@ zY@2cLt7yzCtFiiRxsbsHmP++r?SR9h&x4$wmJTARMGaWUjThA6be=1aCbOE2{_69XrMr5PvLWoG=n%px$8|yc-b`=3~qRZ zwaVyzeE+|_0GDqomA4u1SNf72d9sao!n0<5-+y!j$f3-RGMgN1l#1v+i42goJf-At z!uTKu5)Ce3sk}Q7bU}#G?Uq$MY&hO2{Igj1DE3yU+fu&@7ued8NA{i-JCm#6ZOYqk zY3QTA!~^M{$j*_*rk(6@9^?s?N+h!mRG10629I$#yhQ)hRofn9r<#?g5}Y;?^#0tY z_@z}F=p3;wH7CYGH-sVLjvb6u?^$@>DhjP-vKsVQv1x@`1h~z2D&q59WA>5muSR

fg*)2e>A*TF0Re%JxYh=xzI!b(_+am)sdDS#5U_x2huMy5 z(9Vy}ZXgo@wB#x2sHeQGC<+!RlK1QSr38EHdtZejR}*F$vk<-55_;>TpE(7A@%13vg++s@D85ZaFj z{SJn1DbAr=i5XLh|Dm?Uk-N_93Ki%Q-ghSL#rjT~FB*|v(ZAHg*iwdH_pe!>3$7vh z$yv)1IL|w#2^xM3L*6sole~x-jPdJErC?=u*Un$>r_S%b@)BpQ(Q+lYm%VjnN#Ou4 zv8ylykC*rPtS?i{7RT-@x0in?z&DbPsr`tyl$5+%N%$62#7~sN6w)e2Ai%-&arsQmcz!PIySh!jowvu5_dhObJ#ES% z<5Xqr=YCpA@DjRVwW}!^7bKr`_X_i6Za({cz_=^1&1jxp18Rm?hAH8Af+w$+PvR;w znLg3pvZ}vwGojK02y-$?X+=JuMc|lp*}5PF_zQXV!BgO=sI_{f2;%eWWZQnlRTlCl zE05_oNcCNUVZp($jQg%J^G^B6psyjFcCGnxSOp{GA!@%OCkV?7rlRUpq6=m?jKVcSCmVrM}E8s_ZFGwW!IL2@c|BN7Apqd;9i& z%5BTc5hYQXPmNFg23`owR~?E>@CR%W`LXqsDxdv(@Wu0u8L&<);)8KeyY-M~3ikyL zcJ(4~Z|~x85#e59Y+9LFH2~lhkZ>gpJDu~A%%AXb?=_8*%CD@9uql#`s+#8WBQn?d zkxF!{bw47Kv7cfdpzPoW>%S?6fsVkh2=T>=>WTQkUF`QpZ`4agO-y(>i_cTZ>3=q+c_7{j-m# zM@M&)c&N9e&9c@x;UFqY`{K~5?PDStAqPkB2VLyx zw|!X9AAp5LJ+^vQ@0gp18d+BoQLE`Ule``*SHo6OS7$HnGWRJ2mjwCFNj98wnjhC{ z%1;T2t)Gn$%Ir;+&0`)$89tmN=a$GK`@C_`^r!=LUkqar_aa3LctY6X*CECQm^7dq zu_D8lx;xiZ))zZsfRr~gEVeQTuBJGfC30gwcsi(R+0Gw7JHQ)T^kkQ>6nLxs)1+rW zfQmu}KS#8iBjCgOE)!&{WD~ek=wIyKX76_&L;%ypOCh$bAKt-Ter2$fgKw(YR3t8S zq*catvD_-C!w|lkF56e%kZMEHelv64M{K4esjH!JD674%mmAyzg69Oj!@M$9L%@%v z5|P`7r;$5vAZOAkk%?lQ*<3WJ+-B9RKDJvZUK}z|xKAr`es-bsBzCanxFM+6x@IO$ zXP+WW+gH-URG&X|engD9WENN6MxiIDEvUbLeGfYtEhz+U1Z8p0G+1c3$0aN?W*=mI zCqYB8Z@O_Hfc!wrBCt%j1y@#kx`C+cs5En-fn2E+7milvm;^~uUxJ$E`m4r8&K<;| zxcB98;)eXkR)iC@q$hb0qERsWE6#g4x=2y@#Lme25oIzy6Lmux`oo844e!sxH!d9} ztTj~~o^Pi1b?U*;^+K7hQUgH?M>a=Yyft+w&3PjjaEB8(LJuV6yV=e^uxajzl9t*u(5vVem3JZCtH?UC9t98;*Y4 zU&MzN=)F0nKgpO94)7WsXg69@t^v8U;JM&z>H1Z@^M#VeAU;m>-Snnt+xMSmB(DY4 zi)!Ls5H2|qU;-jXXx!_9mQ3&ouPaRhjDn@%quz!3MzxnD3asmqBqdM%T@cgJ!F^ zS5KNpAH=aM+|vlbjeh)j3Os!6oD^O*M+FI*pzVS!n=UUMXP#>E4FvB;)A=|K@1@2j z6~4+0@ShVl4U6;kL1ikro10!BLqjF+o5?*~RiH!NkBy{mW&7aUqu069cDc~HLGD+6 z{ku;0Jk3{4n$ZzQb=^fP{GcTzvzxR=%Z@-cBhye?I)#W;-bbKGfWU^kgg31IBq#i; zidQ$aXm9Rm_<`vcmP-uF@*??l5W4Tj`bo`Pt0T=Sl|n7`n(K2UZN>K5>J*e65*R3v z!bqn$g+tP^8y!xDVeZXKOP!{qzk={@7bsn-;@@%^qJOnSl5>PdqjOHX%JbG;-~n;E^(5zA&tzJg@G_jv)8fVy=0@9=gsNB$i%vkZIaMM1YlQ&{ z0POR(j3@3KPG=Qzj|YOc1^HL|WEZ$a84mZmJVQm8DZ=dFCn4PQabZm-=*;7QxHQGu zi~fM2k>yPN7ZG+o3bocm^pnb$vVRJ`p&7%t!Tv&z?I|Wk{O0BdQTyvH`n$Y88?0N> zInFwWsFP*Ntd@==druax_p21l!`(6#K5%k0-HW%yqla*+f^Y>0AP_Kwr?W3@EI|N5 zKA{d?iv#Q~MxXUccyj}0XFt%Nsv)Sr1iUZ52Ut3 zBiKz3@|1iye}kRce(xIt(9X+ynn+JO>_-tDD;Xbo7_%Bn@E)vnmQ8Ef#ux{C$GsK;0%n1Vn^Bk@>k{>~e{sQ`G zo}SZu2>kTGYHARkYEbW9^auS2@zf8HQwMqL1WNJJ)pmCu)JnfU?=oodcwm^fS^BSH zl;ZgNx>H!JNHaB-QCQjCJo@6nMCqYoR5LjrN%3jV6=3&#hozK|wW!O`TRg@GTN^84 z7NBJ1%aWx(<1eIvWM!l=ryi9Q6B1nQCK8VEHI;Rrk1ZbrptoR+@2Z(e5M0i&h`;^D|90PSZyMZEodB=zlJ;}>yoDU3s>TF|EqhTb?I(S!6g6j zUSt(7UQJ#PG#A0` z{0pz=1Aks%f4Uezi}lwt<-Chs=Ic|sH=wGo<<9`3fxRA@sE;IO`GKqa7Ewi~9~}`N zNg~VNS_kS)xmR1HBE>dk+&}}KS#_IR$JpVAxGSA1!&&Dw>GpGV#bj+)1Ui2p)OxrUSlwg}Uak>@QGcb+{HBei5MAjh zrBaa}7qTZ6)HVb|62tt;>EF5WI8!k5Gm{K=qoBL#WRj8M5YVT^Y1NH5>ulc~3(7q= zJH~#UU3pL@9Q4bkjCnC(P&QMY%9P?e^RqO^c=y9`xZ236Z*tMATjfeqrMI8Ch?p|1 zTi(GH#K`2pKqFhZe?|( z{D{d6F~k;Z3b3T~>nK8+gS`(xZvzr-#seqCwoRY|(oMm;>p?<|C~a4b^{fDw2a|N? zs5U%SLEXmR^#ht|RTkJ)3msR#c9xMX^V53FTHXHF9+CkEDx~m3^$T|l6h8~q;>W>V zWB-J7CMJoi_oyN!eh|b&!Yw@X5Rgg?*)r8oJd)_&Dsxz>(>YDMPd@5;n07JXLM`~f z%){w0Vn9glq-m(znw|U`=SwCrCN450O%F1Lf$*1}QQ+zO%k~SZ0?0y5Jz#z*NB&+` z;0zn>@}qC=Fl%Bf4O5;wAP*C+H$3Kq>n&{&k!oDGWaCNf!;xe!zAdV|zDdmajoM}- zK(~bXB)#kru}S%|_*x7K{(6C!1j?OBut64mEvonK2*T&&`xWIHiy1QFm?c?qPj*AWh(bvU>jgQTI=e^5m@P#ZcW+vlWMlmzefK!=^IE%if z+4m+(Z#ax2k-p^Xq9c)fD$jD?E=n8)+J$gfKz`T^R~7v6sT-I?Wg+vb+{6j3d~c>u$sEu+={zk~f2eeO z#@`N4;*{YT#!{7fy6-3F@|`pH3~dtok#0NojkVkxskmh87Snj1qJQCSYDPyw*mOQ# zsuO&q9-M@oZ@%z#=i@|aP3;66C_;f|Sh+^FwJcqlrY*BQjpjr7=0x-B^GP~y75TZ& zdb0%gsH-ZJrP&zsQo93Mz-PY-Frss+jeD+kq|WKsm{FS5;e5mHc5W6HwVj(NJPHM5 zO@X$f%0`RKC_;V&$vG3Q52*8utnZnB8|LA-_x%NLzF;#lm64QFb@!vj*FLVE`+t(# zZ7wX8)ZHH>!8^!xS@J4h9U&JYg@Tr|lF9A`aJ3xqLWDoj+t9xkygTSD#fH|-)=^w9 z#GO-Cy0N>$B-Ix7u!JfYrD>sIXy$EDzg^LLAI^I@M{#@7-J2buyE^zl802PX>aI#q zs=UM580R~sh>D(v{l=I=@!7cp#5=`Dq6OTA9UlFSM|eYat(cSYSq-;D-f3AXTL^#L zs(B-mUE^phwH1@1|98`^DB%5L!M?IH=cd2DJ}>N3@G;yL0|lJ)j+S*JX`}JcqZC!{VS7;wbYr^n^`@td zU{FQ?%2d0^W4&13DuH43N4=@nbVJ2f-&%V)8gu<*8*r=lJ5HC zT^9JyG)EG9)ad1yn^12Mbgg~>aM<_#bS=YGr*^^Lag?~W11m8u)bnm8PMi`gAI96u zSH+2YOHll{UzyKHptwiZjVkbd*=*OBYLIGnX7ToUfwO_BAAJt|{;rpMIuxA>61M;{ z)9pLn5=MrH>;&+POmY}KVm)GvT(Y^PGUXO(`KY&xLA<7`V7E=9%eWW{C$-azE3t3^VNP2^Ue@-9Li7BJsjmDR_6)|zUgRbN{kQYv1L^_mMa!|#ut2*YvK=SpQ1sjB{ zk@q-UQQOqQBppVNp{T?xJ9zsvo{+$fJdNaq6@3vEwz-by?R36UmRoL*oQD0q-{2hJ z{#9P$;Iwx^4cG%XZjid&E#po_eE?lh#EWkb_a4|fFe?m1%=%t_m2@0Y2c8cfHJvsJ zU!7%NgY==ZPT54Yu`8Jr%OGV{$g18^Nr{ZRqZ2mAi+Rtw#x(~Xv z1rx*CBq?XE=1!eCnEuMm>+8G9v^g(K`gi$Q+tsGRNQtEk>vGdj2iBlE%egi;nTa%ov_C4 z%pX&p$8~;sNUb8<#>Ucc*P3;J5elg7K0X>Tz>Qu2Ks~etQ=tVJB1`w#(W3U$cdwa;ch#K3Hif!S8u_AuCRyLP!jfM7TAMSy2y_c zRMsrgT{OPncQOMUsTQfzafg9BuCq0Qfh*h`QMw#Kr|?B619@%WObofm(|M^jc3x*N z(wA69uB86MAQV3+_fKQW^G&ouQ#*85_QhECqDQ~=<$zGJUuaYeRq?$0%+-qrp3x(v z?`OpKHnvZG1!$$}u=6joyai=?h_+Dtip~7F%dSa6wKWeEyz}5nbL-Pq!8e<~JFaBv za?p6MqJ;53X#@)($lZlw$OalB^4lOAxzRuyJ-EzChz2g4+YwNkhg!5o>JY|ZTI_&P z6OWWVS<)nR=Y~JvN*W%F5-(WhcfT?e4nBz=%mTYydxza)rr~EU(FBx%p+-i5bUgP4=ZbgqwOdeGqKh@EF}=*d(@Zb}$2T ztwC;HbkO?*?Ei)gsm{0M*Uhmf$`M}Xe^wNW`e{>z)Ai(Ayag3f*}o&vksU&E=I}MY zW#>s1S%m1#MW9suxw?@1J?(Fk5!?)fmbE1DCf{<5N~TN9@L!6>h$kRGlA)B=!=5i~ zx`+!uGp=&8daTrvgt0x4D3=73>$j?}g1#=ch=&$0d$NBnJcw~5WDI*(g5z&n)_hM0 zOvS8MKz?yJwQ|cPy7d8`p}w*;q=+t+rb733s^WKVhm&k1P#^|k$)F4#q$W69f$cX!T6m89=}}LgK~r1keCm zFLdxRhGEeGvvZ3;G1LzkD_!{eO|6vbr1OJ{rqXuG76Pg|1&rwim&L^RVWY^SKW_nK z*Gv>k$Z+Z?hT8~}=H-2Oc(?VQK#GCJhSy;Es5_U^#Cey$z2c6K{cE+FUnGU)vz`kC z1}?Vb*@)k;Y+ngKBL7>mM=T|k(PC^S&Fwr8E|yTFr94Q=$0x>7Jbpd=0i^dmHXVeK zase#YA1+6EUARV;%lZN(Na0exmfZfzIxk5sP&QDBb}a|+0!JP z{4C&j{bc3*k)zl=w#%q~E1~;nV-=*_0{*7Zl-sGa;4PrI=v?h8M^OZhpdku=fA(vY zpWI}bZrbhvM^9=;Zo)XXY~bY;zg!6U>p)z5#w30!ScH+eECsLfqc(q*>P+H-F4|k7 z_DeHCY>do0Es!lD^|q5A#Z;39EVg>=?Mukb2HNsZLH69K$MLaf$C|D9d1dl-%2;yR z9);UV4^g!-Y_vV133O*U!e4Ym5>q&d)_9jZ*(>DqSGqtU-(r?TLzGlsQ6*<=qof}% z$GzKKr4A|G0-9gWqM6&V=dluF_z29X4v=A>)bA)1SI~BH67aK@&1cqZHRhRjxzsro z&>5I)-~qhydBM@`ImmMx0P}37EO=tL>%C3E>`pYStlmcVJGbIIA^XrKM=vVj89-<< ztsQ&o**Zy{mm8I82om;OYi=4CT5wXlomRvPbwbk5H5Dx zX_x(IUyHi(8K`aCaYjM+z;+GnEgye1b!|QOWZSFrG3Ncd;609gWi`21^S4%bz16o# z&gEOg>`s+=jpC+Y=1=S4sI6$)NJaIc7$p?=Hr?Z>k1G}US`DQk+~%|iz#vcKMgaaC zi7rewIQS~C&MogJ;-~pl1Vj?wCZjm$;MkP9DUt=Hk6es^TzoKG%K6}NcOq&mW&flN z#`vx;RjSz^BbXHrvTdHP&^fJoR8hCPEPvnd%(cX=7({e8WQ7}vCryYM5W1&Sd*_yLgaJE*hq@XtOy5h-&3y1l zHwz0^00G!Rx6`5OpzSvlIm=Ce_`i8ZO`ZG%f+}Hr4FRsXzoOZ+m%oeH^A+N*$@k)?u!!O-{j4PK|}W?$FB3Ohu4?YI-sd0oDO7Q<5Yn#+<{&a*f$;Zj(?=Unr>VOpWi z^g10ymL67GQY;92q6976Xm?f2T7&g+Vcb2u)kW{*W1 z%sbIpNU2u~n~!*%5dF8tg@4(kn<5WPJlxSN<(kW8!>iGCjdzx7A$evp7kzMBi#J>F zZ{MhM2Eth)O{;n3q40B#%Ow&B?yEG3&P#xOmsIzPU6OGi9jkU&{F@^u_qfxmC#Uo5 zwRFuBs_k_m%rfnXDXqhQ_`7$y2Nlj?V=F(GW{K4ahphwY?uOzYOaF6gn)a==ThiE3 zW&l0&&W~R6mSxn;lf@mcK>}N}#P>J5JCpVHILY>|UjTnQa!2B7Ab&H?l<&qJ5oEV3 zu%>CsM9vG!%OAbxCvQ^zrZ zE%mZg7-C`MuZqG9^%E#Fo{+QJ+qpRm4LLqcT=x$VAQF_edcheN^gY85oY{U7T?TWDL< zQ4CXhdT2e3`+if=lRJtv7w-ctB8wg)RXX*7&LZqCx)%#Fm+iqA>d}nF%eABYJ%#9} z|Lq0fRd{5XneEk1Q_{N<^i6K&g+&-8dw*s(BeLK_2QasPa{-pUp2X89lJk9Tk&G2T zojZOM}tbK9O7%3>8<8#w0-DDhzZ4;YJ_5Mj9OD#fS*B#{>eCj)Tv3QMr z<`J+7ZlU^IdMNPGO`EW-Y3ONtjYw3kOc0@|vp~cPzN`qyORX4GjNt~KY~)Vl=3`VG zw`Z5Kd=LW~CUzx;Z}gYgZz%WJI{m@S!Df@;#mB9S*P@3$S= zsDGNhyA`VN=lYPn@^#J7BQ+pD4L_+js`KOp8jX49?>4yXb)B{EHCaN4(gflPyI-`0 zyA9r_OXIO9sZ9I4Fmi{Q5wYo%u=s#fOx%{DSf}mqqRX|!?%gSXF$VukjV)NbL^(g# z@AE>VRnAv+=eS_>;2^aUD^i7GmI6X*0Yir8lL_1r#_Sgigal^Qoqd1QTj#1g3(nI_ z(#2OI)aq2C>_y8qQcRUeuWYj~ST_sOEguF&paq`PX?@>CGjDZ{G+(ECj?fJlvDF3d zcbcK-CS7z5gM$NR8eJ{E0>FH~$3C84j!qQvW;Tj8qQcjU5LxA1jpxioMT};3j~Y)- zOSFCWCfGx5_(!_Zp52irvMfV?9QOQp9Wztv6I(wpqQ;QrG6U0UkQqa;V8Y0)h^JMz zH*4;kNNODzA2Pn?ln$*$D|lWR`L|k6a)?Lz06auuIp(-QW4Ytsg`i0a_c^A^lno~# z5)vNr?N~`}zv_a-!5}HBFWT#FUnxxwA0Od9Z;QhMaed;ZbHr|3B~84$|6m2@quE&t zhgqljIJ!i->J9HZ5=$mp#F*CoD6cI-k`ZcORxRnBsZw;SmmacNQOFoGE#{a1?(Ond zcQG$BH*?T$&}UW^6RFP+lr(sQo8o#cJ+3tUmgfS84>kf$mBFXRr*`1T=d~>K>s5(6 zEUEPhf5uOjs_4CS`^iTMo0}rKh5qi`KZnn!Z>6$T3VJOK`JK;EQWq@3H=Dc=1MQ@~ zQ(NVgsUnh53|*hAhhnzBQZP4ndcQ^&S5`(>G(`?p3*eLm1o#RFk-yyQmsB5#7BxnrL1-O(-lXl;@!rwvGl@2)QZrlyK;N%2b-I&T67CxzG66C>1AR%P^^;^aI zlQo(H8Ug^b(mC>O>V6~qk7M_$nBV|PgWoGw$a4Zs_zD&~h zMxItTT(=OE&?!hKyS2tLFGUAcN(-bow zTSB&-DW~Z-+$`P_77_?dd9H3^z7?ctf`g&Hh3`9q&wOfG%`=l(&Dg8)G7aV7EAA()+)+xS^30hrQB6C z`7lSUld6dg_Tz5gWMx58o>{) zI(1jWLn2Sub*=}q7t)#n591`7_Gkr(w#T2e1PKz^1J!>mk(&64yR$RuUoVMI7Pww5 zWgQhYgH`>H9JWR-EUs5@Qey^HdtG^L?R{@N7Y|OO&Qfs_V>t<=yR;h(qU+ho|J{O% zN#1Uq2N6nqp~;%qYoduRgGNn|24qVHcUvcdl`*q%|Eds$CL6%*iFC`9}2Z;aq}Dp#y&F`|B|XZ z+8~%Zhg8HB=L;xVr1E=(n*qB~>Llsvw8lZi9pk7&<{B;C@b-dK^LvY8>tz)+dHpTa zo)~z^%v?eXqEiy)^qcpm8{kxu`#y4E$4A#>)$C$TJUB09E_2xEL)}NZTbx8 zgzWQFbQ`EBASGp2`NT1+cJFzHE8PAuoNexXs`EFqvu|eQ;b~)$aeO|yE2$rhU``TO zP)*@yuAq(%loqb4dl{gs1-!~?)E#4*Iz&iEydkDYH&4I{SiX>cI1T#ea_?>*7S|K>{&>=*AtP7ctj> zDFGj)T{;a%xtt0)R?4wJXsSrZxciE@Ry*tCByLx8-|pq+(``YA!xno1s7v44pRGl=YJA^?X|;0A zwM!Gl(RI7xeR}w{PvT6uSd(3m)UR#o>9+==<>Sx7Uzo4iKZ|LTrJCv{(>_lauz9KU zScZKJHF2OEi$hO(coDT)RK>AXr;U4oD7HO6^#{`a`R_{hw$M2hYvNRMaT7Wt!tm{i zBVlYuv;BExyv*0~N?Iy$g=bdAEifJYz`nkH8j#wgxZT>3MGJsm=U| zTX&m0dIoe+)W&`jY{ugHy5(LwidvE)zeN&1U- zc9Jel++3i2DdU zc-q(jIg^Y^PR-0(IQ`+sEnZ5dWgiOGvM3#GqEF`B(1_S{8xwm2D(;cv!7%q&d%<#) z&dQ&ZHwTsygGQ8|nR7l}J}SIpw}{%8Py*#uKC)n0IGe?CVkl;RMh7rFl-kk{Ee)7p zm@ha2Hp95*-S?h4yv=^iRLvyvgAJb#yZUpY(B`Lf!Rl7c#!T^6QeEeSlTPoX>+{}; zO&aCa1vf<4i-GIoBlV8g+hjWeX@DxjM8U?yKPzt-Ux@_a!o2!4LTo#OU(;w0~^HT0QmW zTG!`~E|(my%yH~k8C$kWPvUqoCz+&zz@d}RJ{r&Cjb)Vs<*vQ<+H}b!mo(Va2C4Ja zaZSb{4}h~TP<{UMpP#<;r7ta7gS*WR^Uiu`Dg)g8KkM>5COXdJ1owSmDw9fQaCY8f zfsX}4mlXwZqoQLmQe&lFJDS+9u}-hlO&*jnKCbP$80Ci#YWzEpuKCW7(oenf%I{GE%^hNNA02liwb?Ys+-u}@w>JjvObz`yU6_ztE8tFdit#4%y_niCgqblwqliR9oaZ7TL}9H z+h=t7s`UQP+z@@6GM@IR_3>b3T(4pT!io7}AKcA5!i&YZO>U?C`Ig6QBa6xsi-une2O%ewTsHPR2M3O(mtM3n zSo5)O+>;LM+mk;1wI9X(mCwI0#?6WSyW^Gg5q6-nv!%#g_?o9um*ru$@^rH0v2SNU zv{FB2ApZUP^)yocX!I?9vXLS$^5v%iK;}+QIuvN05(GKOoa4<5-DL;y!W{`uQq4H6 zdqz%-gn#tB)6yDk&ET}r|NP}!wFn@lh%p#4Oq7BD3Lw1K0*qr#Umq5@?kdj%g6{8R=w3z4xHDPPJZa0nnO(l|Mx z1+9<#q^ot&eA?~NA+2|?XVpKghZLP}3Wp}kVj}PS zhb^>9w$REop^M0ZKD6B*`b9eFux$K0AK07D(awO2&e;-YgJGld3gt-H?b==7UBJ}e z+3kheC)XP$c`VkiWG{p&{1~51l6c@-B`e=c9={=N&{=f%?K+sg`rZ4|h!)5{`js2f z-+AHV)7htPOqV|8QRy$g|3FH-j($)NptZv{?DG)#T+dw0gL5y4(XK);dYQ3TeDtiS zG;$WXpUph7Z!y5kqXvf`+OG+S>Y;)lo+>S9kxfo_kpP}~FbB3a5F17)H)*wtv?<`5 z238@+5M;PdipEA4_}nbYxslOGfo>b{tMp4igMCXMw-lI@|VBPq;$IpuZw_ z9>7>T+|ZEnT?hGtHVEP(aC`UejrRC-a+*}P*Wvk;Pt(5UuSp-g2%Hf;@uZSr!ajtH zr~I)KK(ZQNvJgz)B$2#jFr%MoA3-cNhU^wz=Qr$f&Bn=i9IbyjZ;ij>jt1XI*bqIQ@F8u@0i84$rq;6#AyN zc?xbqP&Q0Y8)e&87L0UA*h1LsJjml!fda1=A2Jhl&x5>C>7zXdbKL!pa6q_r?{Z-H zfxTLw)VLhygz-vcu7}^6UKG~ZMVX{?^;^@|GDz7D&$m2+i{-(1wmO6{ou|A(&M&mNw0a)Q_@v8d?C)enA8FlhhXY(fjBxK zpXH?@H6WTcgX*Vd+%U*v-(o;UtH(g@4;?N|h)$g8MZ9ifkpRNwPMh;V#Be1)X)zZ9 zK(0y8%~$gD@W4UPGWSWF*NmlgE1A%z^yTkv*Cbv>8Y?qW6l^bACWXR+Kx`{Qsgzo4 zgEc;X1fz5&z$1qy(_j5a?{RC}@PX&I z+;U4=zI=JwxN)OSt>_Y8K21Lj{nD=+=-09PR(b?mh+U8IbpSW62OK-GSFX~*ZfE4(cHg~2XMFKc zPp@1~=pi1T5g%D0T@d=96hPQzQ5Se+C8SWkZ_=GtC^DQ6!hb4w;+u;mma-PJ-M*0I zaS%VnqLw`5`wLE6m7cOKFSP!zuid4^CO!SvIEbu${I70FZ+Z2WbonKZNn1CpPCNIp zP{1ebb?7|j)0G1-B>{~LAZ~-|r?#o&>3HEASyVxci_^(t-(o+il>fOoM^1)P8A379^0Nb~3Pj}vVXTtd5RCXKSZvQaIXU*SQ z=A$W@Rapj0+k&6Oqcp_eP#${vA78G}J?Z%eFAD%HFDc5eINei3CLlRkBR%)=r=@2-{;V|l@o(srI*qy7n#*El80AH$9a&p0GpnVn$wGIL z5t#{J`l{$EZku$42oEygiEpk@d%3K|Y=A=>Wbu?!PDwj&$k#J#GBl?4`*-S{nSOHTxCBy=r^7)S*?I)|H<7c%IV@pbJ zzWgkGa&RQwe*fO|*=z5W&d_W`dMdVL>$N7%_+ot^B!Hi&TmckeE~!cD=}E?|oE+H| z481f%cwou8)a>>JC67gZYIz9u5)Z%iV|jQ*IXLiZw>+Hg*j~Q;r8y>Z#XtMjz3HF* z%tUazjC#$YM!$&b0UGSR) z{ISSm-(mpu;Xud^rII^Z5q2lke#ssLCBA)M#q}Cn70}M#zOUe^LPV!(YmI3#uj z$arB@QQI9y$Mhbgm<;7Rfd?MYZjJFUzHNZl+jNIbKH*8z z56mF3s*Y<0jFOkM7!c)$e1yPd;DD>-DP*w7VqK>D+|lH#$zPS1YoX#CIvRW@pP5Fr zT<+VqFWqs+9btrHoY@$MPmg=t;{w1s$bK%RqIbpbI&Gna4;;{w8R;bqgh6dxXzdw< zFFOJrdg!6hNeA_PY%V5dw0l?#@K83KWuyjJuheb9=w_K49z{6DIBnC#g2t7Ve&FNV z--@0rd9J@}uNDJRdh(-B6|9VWZJi85NMk3j)M?<-KL7$ou4A0a$|dwqL>1#pnL_lH z9NDQ``5iYMJozZI(WkzbzZ4eEH*MOK9(+(=P}2B_9^#_!p@%x%gD@>VS1(4h`vZRa zAqhIjZdmN1u)B>^@#(X6bvXLI5>P*mukChr1i$gQr>56Fdu!Tva6G;1<3EfuSw=WJ zNkG`{EJzwY$|xi`nqA|%LJ3uM2aYpa#6&=QcOqfMZ6isB)TU_7jp&j20mt-_I~z!SD3-)!Nj|D3Wj z;QmJr#w0OL`&U6c&FA1hJ|l>Nh>5=PFV`yK%4L{6IPC_|WD18#lIzDbl3}xQG#xm! zEFF-d!9_W2f21wd4rEhMhjHr@4lML=g;Xv=Ng_eM0ypmW`D_x`l~5uLiLT`bwU zmyb+lFUxS0){_eh@gVu+o$5V%_QVAaOT}S`W1Q7b=xSb?4#)-r=t6XL zduT^}(Vlm`>s@Kxx^?NwE3aIL%{ykCk$K!3p0qE0>W9nI_aDgPO5Nw1!s(Q9O~w-s0vS;iJD6gR2*#;kO@>z;k$v z8y34L+X`roMy&;Y{(0RCFHHOQ>C0Uj6UTIZ$*2|;InY#!M=*NnNQ&vKO)Jvbx}J8< z+O$@$2j6_(zVvTCadY~Xb}7XdztmVW=5)|ud6!vkZvwBMFb@^o%$3Isi;PF}Dn z^uhjPN9CvqtJ=!^ni zqERGqJ*+vu-Ntom(p@|BDPKO7I>D9C7+@@>JeNs+YT}+MPo}{`-JQ!6SPHs_Gd-4~ zs~?iEI2nD9z9wRvOuHT0+pfrZCENX>gREyz2F3lqKXb8zGxEQF^RD#XKf5K}aNGTQ z7`sRBYU`C==_V#=V;mOl-ASTA{>!>23a9>1Lih@t1iR=zzC?!_%CTO|`}JH)!40E4 zW_if+7(8gnqh;e27w3Zm|LofBu~3x{2UMTZj`KLrLW@;b-LyOH+;b>xTDv^G=CUWI zZ+`zT;@kkv)*0o~-^*9TxNDu;^hX6O{_|<4+a|CNJ^B^{oInG7ts0xuA#G{^TONuO z_wo^+S5q7pnRxQ9%;kBzpz>ltLxy66Wa1BoH)EnoDmLR3y!`<^iI6*ElR7`2X``pb zVgTEP*b$H?@#^b=hZzz;)BMC?NqYZdREN2ZOzQAKQ7CVFgeDa*_yk^q@jv$VtDeAH zH1lWUR1bm!FU%L-r{|qj4zuFoX``=+p zQ0T+d?7pp4#Bo0Fg~6?zCvk9>AIt z=bhK+DfE?hLRfdrMjsCoZQ%Ri!-vzYx8B;PlV~4l|LNOn(s{2(LjV9E07*naR6#9J z>ux9v@Sq*z6aniL1sO|-t1q(Td;Lm(G}-&$N7C*CJzdOV03}W8T}sAHPE)3guzW?6 zaKIB8z5*w~0QX`C{f+*JF5=ZP^wBbn>mDa9ml@H8_sbY~^UXK+$s%p_>q@l0t#f?8 z@LGFxjIDc)o@;5OqF;x?E{c9%I)5R3HccAsl0Ns1+tVQqXOjzCh+VyL7-^sD5VyhlrLmUwl@!* z$0tV9XY?hz*FN{G^pa;hI{oYaZ&})-g@6ft)goz!OMJ0MpAL`lE_U=%k*=zK##vk* zA5V@vGG5p7NZVr2qHi%EFl+s+7-f~QXFPyEZHgaFM#8y)xoss93bP0x7u04V8&N%Y z=bs9`Ft1Q6SEamnKB(;&a$}kZow<2+x^0gpO3PS!(1e(G7L{h4DdKb2?@V8~{sDa; z;Gk9$VVFK!H=*qu8cY!a+;9K=XQmfD^{iOk8Q=S0nuzTn`s|wyBTVrP=3HW5`!i^d z!M=|YkFEU~cM2#pSD6t~#VbbpDu`MZ^N-)nrHvRlem9O)M=ub}t9(u)-z?@}RyQ(J znDr0d`-Sw#gWDy0hqaom0gSI-!~`ERHuGIxk*gxjdJl>3=q}^aIg)WMI&xk(DzcWn z05yD6i42wU3JOurvGmW34W%qy`K;xVMF1Zlh(h%M1BL1^_c-FLfOE{?IHwJhKd3!5 zew>{!XlKBnHrH?qDZFbJU!2Jk;-OrajDFtV+0j3tOiXAZa%9#YibszCMZVm#11QIk zR*H;Ql_FNdq86bUU!}<6q~-Vm4-w=qd2#`UpMQg=Mr)ym+j_{a1<>CH_0>WBa8TSZ z(q&M4YCNrb&O7hCJT5XePI1+63o$%sF}kgbhQTgEHK1ogcZ5dBilNpGzw)2|PkQ75 z+d_J{$5+6xg~G;BPV^L>?DX~8y8Wu>KPJ8IZ#`b0cH5Lb^7bpz8-MfD>3etYi(LU@ zveQTPsxak;4y*J~DJIGaofWw0mdt@D5x86@(UDm2pAJ6Pnc*WGb&sTf`nfgf6wNK) zdFlQ|`;cUz$YagynxA887PYU@9ruG9iQ3r^a3f}KuSBU!vE zz+%9v70c33|M=t5hrf7(77uiWjusF`4=bNOQ_q|<`T%lO30qx%BY(&Y!gB)U5nQI4 zM|u$*DHbAGf!5(wS}w>_6s6~;1;RESz&-LuLzE*rf%2--=oFM$!Zb3sKD~8fOv-^Gd^EZE+aaD&(R4yVHS3cBcK0JeUsbelYFV zb@nN%$Qp-09DI03I=ov4*X@5qiv#*(p@e~H-SOZbbSn76@T0ZA#y?N^GA-f7ARd7? z@r%i!U^qOG$=0@Q+u{>#gxyZlVOZpI5P#vLxne(f6f&wsHpx|d4o8bq69@EZ%l+E+ zCfOW6plhEl!f{X9?yP8dF9~_%^xVStx#(Ym@Q?C~P=&lkUsgobzk&~?ol8MJEtfoT zJm;Kq;;P@vgjeC zQm1<)bBtGu%q~Kv>qQ`R(dnn39t+awBDZ-M?4rg7SHEoA{Y~GDZYrIDEu@JT=Z{Qc z59w2F*hDzn=63hQgD3%dm0go|SK=^RbhKS6tX0$7;A- zmdVHz_i+Y>WRX=y4%*56FOss{m*BDVuQvV^bu{@Fl1o}XEthN8YKKP9YVcDy(#|^T ztn}dze>e`-;}rUkg-UkY_rMIRvs_PwAG>w?7(q8+*$E7tZMRZnW zWQ?#96YpyjLI=g6I`?I($Fh~{bnx+}w0sTM#w;EoZVl`42r^QayG>Tp%vf(&BhFyX4()Nc1M%4gtM z_>1fX{~P(}v4W`r?;b7Fwzy@hH-s!MLmt;|PAk`*68;K><65&>^14y^Hfo!a4kuxe zigPhkTz$FpPCiYEx%VxXF4;zo?`t1fc*;|rlCHh>T8Tmy3VdI6-eC?KcW|u3HO?5N zH{N(-qnk*p`)-|gp@uQQ;eYMwHnt7$+&6n;0&v9PY>@aqB8!X=W?+uAmhN!=nZToe zV%$?%S^6khjvLoyKjM42!Z1)8*i)z5dCn=E*sY;)d?Ib%xi8}w=@H%hG-IZUG|nVC zJQMsQ?=}gG8jtxxe>FAr)sJ+5PnMFNj&@w6lcxuM%nXa@q48o|Jsbfm;0%zXD%@X}F4TeHKgD3Dm zGXva^g<2*;ehRNJVDo1Lmuc~EZA|o+>)Zfs59s49GEhnk`45SXJb592a(Z z@@YEGy>DJmRq`{R`OI|9HP@sUyx;{>Wm$0XwVrDu?|#|#$>+E>%tE=Zc}=*!dImT< zctVXwvlu(v|01Hf!2sn(bZO$^w(rMjjVfsJ&K$l0X-dv9)-O>9iUge%Iwe1ui0lly z(V(o)o^aOch`#y0y=nge9UQ1X&PE{04vk$ToWn+;-{a)hwh-rjp*uChi1U+H z=z++xl{$c}$Mp%f+ji_p7o4>ropbtzc)fQ*4@f5UUiS!Re2r^| zfovs8?%7DiE-PYTRaDb5({Fb37Mf+f2foya-edkjcrn}U3r-%zpWZyO$G69wu_`_O zth~_wPe1rK=_}f8en_v1^QsDS$1y!Xj!$3eD*CJtK zcUtzRho>jk^}DSl3w%Nv!fh-;$(MiS)!i!bYg*zG}5bUuG?>1?L%qTNBzMI@x=Veii>uU zYjHcbv2Ob?$Y;%;CTy9h`iG)=F8U1pjMoEJ0c64{e6<4HBe)5w?%~TsBN6k17iN+1z zVbG}W>%K|aRQIGUM(6g{1~|rGTxxJO3^5^cAI-)hljxEr{hi}_dF!;}fe)Radg_7R z_1EK<>&H@ZIBrolvV4~nVifgH?DWFSXkvuq(%E`DXqKcz@r@I|(_+ET2e<|BV!!D1=L(NZk2S0!Ml2c%Hx^L%! z^jA0BqshDu0_4)1azd{J@}z%U3j>Gt>dRhxcE)u`{vmzk>wkRlI{~|H&C2wX&$%GZ zy@)d}I0J)+(^Z}A`m@XlyvUsWejF=*^MhymHWNbyN|? z-vp*qmu+H4T%n88n>@knLNgNOnzYh>)4yK)a2WjIMa>)M&!sK?U$?iW!7#~ZaMuG* z;cwtv_-hWCrkA$@Nh)&O>4tRHfcFp6z?12Muc1zOU03TG(x=`8;d} zxb=YjFklOz8RCLH7(z~nU@?}-R~azDNu|H(($nKaZ<0U#wL8M#&SNb0f^}3jDSGvG z1~~LOk*=1*I^5}=Om+9y6Ul@5EW76UFxu^n{So@6pHf**gof$aLSf@D|B!veU%V2D zj=^r1o{_FO?~Dy;9S4KR-Ks;};=Co%F9Pb`exKxo#oIV>7;KAY8M2BwhZb zjX|VOfAhXHu1P#!o?~p}^_6AXX~IJ|9u&tM6u-=mk;iZSa7ViJo}B^z>dP+~8(%MF@H6E`fb#-$E}-yqzHci$w8Yv1SCh5s9Ump-Y> zhQY8cOh$hnAWu2o`X}>}z8&$fMmomhm37y-TUMl(KO;Xq`TkwI)3rD4(739v2}oT1 zZEuWql^SDuPcLR(yKKi38#K(xQFoI^WjbFI*W?S#GAy*$h2z&@H0DxYIEN;k?y>Yh zjm70^{o|GqhZ_dFJ$h;Al+s@dl7l&-4qtW%Sl{&ezaT2z1U&70^>a=wB_=(|-Jv}C z=M@6k{BdSjXppJ(NN2Y<))Pk-uo6c_52U>_Ax_ z=T~ginH4&>g7Yi%Dob;&1?O9=+mim*)wc&7FTU`s^w@1%V^_g=`OKgMGRCKnT|zB+ z25=bUv2QWJMh%R0c)dQYuywUMnHmt2QdaAfo*UMaBj4MnbLL!mHKa#?9xQonWE#$X z4R1n4nkPpp9pmFeU%oHB^^d-l_Uu2D&Odu|`jcOINqXreThq$KXG(O1!Qushq)$RD zPp^1ivPeI>S7?MmP70_P;7$+C6NZXn!q0GLZd{h$`cr2o-tT26 z-f#ZT?-hgTpq`d64(9{F;$K@|u8$;n*cm*;VcOx%c=*xW1@6*VMDirh_-`xtz{!Uug39Q%}!}+~2r) zSGs41z6PKt`b>;@EtiAlSFJxyhgRy#9h&s7&;gd&*Bq03EflO=w>5qCI=w!iv+8iK zzWm7=o3u+MUNzvgb>@>2EXKQLT+C8t*&7CVJmLUBlqSrqug7fG=REf0tVZ4)hQ`wg z8JolBo=7?SN@|?ra%xVQ2|Rc~uo|9i8-ZD&4>GV;tm&Og1{no9?XYTJ{$5kG73+M zEHQ#razx)@03j~-Bl(4gJx)4>G-V}kDaLv1S4GcQzQFf*;DK!}A+}WI(|RfBDLPoG z`yL-x$G7iY#vng{hxCUwu@La?cfULR^$rs+w*IAE4trs8$L?(0 zas=(DR_Z3`ymIcR?ssl~?%Tu`0~9|J(E*0)$$3MR{U4PSku^51nGosH)j_MuXF>R$J91D{wC-j?Ym@WQ> z&uyFo_nc3O!nnMF0cV<2m_fucHTXT5c^Vft?yyILs4)*5jbE6y_sMl5OXeSF+mWSOs zSKu#icpzQ&$KK4CO4Y#nvQjxu z6VY$nwLDE~QZu2?`hi{^|ak|Hw~8P|mk*d05*z?oMC0_Flao z!lxkird_-Im_%jjJE!puBub^(i2!|aTf)>TlGTs5+j=DaB2fW`AcAVaXXB}?kgCq! zY9odYN`=n~_TE*g`4nX@fuFj~gojB!AwNB#;k@`um{^?o8#m>LRPr>uj_g@J2m%^@ z@((U7xmY$cFbF5A09B{?Q+%Z+=hb^z7d)gK2+|HV!pip@U7e zAmG;o8k&bZa7Ul8+#^p#MU@oiz|Xk|WvE_gYIsU-F7afk8Q_^l(6rM%rh$FaW-@%$hZaH3#|pwL8*pee}C&_pY7sS};mwL?^e8 zVY_Qiz%C`MoQ#5PF%T(wq|{oWs2K8-x9BKi){|>7WQ1VN zKp6Shtst5IYJ{UbL?|sLLPW@gQCRsANuJUXAz(w#NSq!XMojSTh46FvA#|)Dq6Bop z=(w2jho8ME{oW_8OXr=wHl2IcDe1IRHl$7K*QA|$#?$s)`_oPL?oRhTxHnz}n9yoX z95y(q#g6aoP8YxFL&01=_vW&}Ghg~G4R+j|wqG=1i39m?`H&_iU-Hl2nxD40V+DR> zE{^-f>S&u~$L%&9hDAOz*9r%wkZ*^d^L>2ZMYMCZ@j3d4iCEj^rt_a_U+r3UiPw=1zPikb5wqke`src zSsyJn1AJUtAOG85-jhD?nVVt<-S`2%<-I2k*d3FB!&8%G`qBvp@?wDFN>7kaNSAa+ zFW%|)$<-O)L7-#&aldnaZQr+x9h9$zakve4qVJ6t_0-`(_e)v#m=EyDbRx`UJv85T zktbf!yRnA~uAv*Uf^ajA>Z5-ic;sN(@vyF42h!jCXlMG&H||Lf?%ExT0EhNHqJ>Gl z->qFqJjq?gSHk$5TkH-%#}pb-=M;8X-i?%HE@#X7C?nRR@j&>_E^7E^9uu(;tVAN16W|tqJmZgbkz$Av!4ee(eCqeEsuSR0cN0m8abv;#uq#| zkmo)fxE};j$DTm(XpEmI19>A62tqSyrYowGF^dj`Q0G!vG|XvL(p2z01(P##xY{_K zF?I)tX^I{Z&KM=P-M=^e=-%6Ny~M!?;f#yz9WYd|!B{B{7#{`=iwk<+gRcpU@hKDD zhb)Sv0EeIxo-O{R`Y5Iip9L)^(f)WBQ|*I-Th~kk58y?u?9C_q#qR31Iy`Lh*=9q? zhgd`B{q#2tJ0eSq@C2Ulorm-W9XJ?pT`^SYuD+Xni`8I4A0xw zu-HXS$FIpPd_Vn-htemmzDEyDA5QxpzAqix^I%M-#`t`)bV=Tst5<*pM~{v?^*;{Azj3$2)!7P;d}kUXw0Qu)!9X+YWCZ*$o;U$V~jf$y88}J zq$_^?hFDBGBw6J3fRUK!Yo4VYCd(y{Z0(O-;+l&Y-aMym*aFDY1nHBb>DB-C+v%V_ zIe2jQ{Th#C+jDjRpS+hZ%^Y0<6!KV@kr@_woN_V1#*G>Il+zEUFYe~O2|6cTG31OQuKl*#>Kbv;`3+C8lDnR$?TVfhx;5se(7 zSdgoU!&<$@fF4s^;v`cRmx^n|lmJ(jiMKq3Qvd)U07*naRGIV-N+6$UcD(RCx$Ko2 zqDCEHrk|xQpU}KVZ|<^loc;69KR^BYum5`b&Ue0({=JUv)%Wjv-}};)S6-Q(``qUSj^OFb@b*te8(i1P@ZHYZ_c%G<3+K@(Q5h{aa|2w! z+Cr&ia&>qu_3L6RtdkDoNnBnGAU*4aez_eMgJ-eHV$fTC zukNRH9PK5o(>+jSaXIt0t%qzIVKdIB-Hz#k?DdAQtP@aA;K)NeKBqh`{V_4IUpj_w zxeG=Mjy!C{j=>q;_fe(_$F2&yU-mmYl$NhvuY2tb;K83E*>mPF4-L(Tl<+y21m2J9SNU>iv@xS zU+lWI+HyA1b+%?^x!ajOG=xRw;2px=!T>w1kUzx_WTF5pHwh@J469x{3jtm+U22W?D zjABa&N`xH-N@Z7oEFh(8n#LjnlQ@`4s@g*R7W~=ZpKH#vannEs2I4~_uHZrVq>ZGf z#$o{PWHOmViRL``OU(ss3JpvT2EAdH&k$_&5B`ubR=1zWn7cr^_z8Eb=)GKN~7N z_&UvD%A70b`s`D#!}xW$uJdhgds}nc#;3lo!wtX!4$vNGg73?E2!A0O;D`55YMkpC z;F;$=Ifh-Kdo&h~hT*+hPvSUOsTfSWBEVmzlq{ii2h%K+UddOwNx!aFLh?nC~K)1;O<(ZDaNZq^CdsbR z*HKD|B$W(}?x^%m4>0QN>>|@_=KeyEM+OsEja*HPNS5^AcXVWZ%nO%mY>Pv2HAf5~ zQBLyq8ce9O69cmhtk;_Ceocd>5Ocz7a`PXdFr0gf*{$&7Cr#=3qd{+sE{=#|6)G}Zo&5vDIgY{=pUAEx=grP_7BrY z!gNgiYby9oKFyGs`Op!54sz?)uTLNR;0M#|UiZ56)vtavop;`Oec5W*I!q?>r(c)$ zfgjUNjjO}>b$F_RjT62v@;ZJhxWy1p8_`4dLuX4L&DX*t{1O)epW1*8egwg1W`NTU zq&M_aF|K*y!K(lm+$>;X@bKhB1dy(w&dgEkuD}JEpq!kPL)@tFh*vszM!GYF9?bE8 zMLq&J;E0fq0hcES_DSm&&Y^+(ZP2Xx>2yy;j71WJ-#C1(VX=!cURf5i7641M3b-OGzLj$FwtV#@MyDI8O zM+#P*1Wa`GYMSl5rL&7n;hvxQ4tvBWUBe`g!Sdj``wU_ryPHL+3GDF=u`rciJJIVZ zG3IH`C0X@&N=vhZ5|Zpx@)U9=(O!l;>aW?MQs$R=k&Bd4e-tr1^4PZ+0B&t2+Q4DL z#j5*VJLzBkS zRUSo;uv0m{&@OF zl-rW8+4QeL@SS`@m1A>#ycAz9z4X%bj(5Bxz5L}bj|G8EoAQaa)-8i-X_sMLH7M^) z@lETI*4#$PsR?<{cE)*Pm#I z@QC=5=K8%uX{p-n!zhpDO}{*5{`m`X$fVfXJk57r+GEFLzS*U&c-u^r#N3fZavuCm zXb}L(m(+xuhU}KHvF;CspY1#teCARfk5~+F1yo{%V*eSB+MoX7rWI-b!O?VB?+=bC z4+r?sSf09jvRqGU3N12CbQ3fMgfOr~a4&d>oa5!0_NDm9(}EA;M^?&EdHRE}JW&r~ z6Fs3^w351{e@YeMM}9?R!v$P29G{hFm_?b?3m|oX7dR0|na7Dg@bMRAXQPT=7Bx8S z@)?iPL58d>N$FUzxTAput`^iBv-p1dgdUpds%ZjkhFLzd6iL3oTewb+g%Abr3B~GT z$N+K%|Bhe2I{7^L{h%$U!AbMALg(jx?&s2Tp7We^S(+o{lX-)l%SlP-!HC|W9?@snio)pWEb{6Pm~eOjQ(~Be>pshN&V$X| z=sd##Lpg>gjPfFav}2ri03UzRzdE}eD&rQKJTj?|c`G|gs>yqnJ;|8FBxDR(sbmOQ zLw?Y&^*l67pzM66o!Yfh%Ogd^Tr*@wH$>=XSmcp*^f?;tt8L({TQ!!Rezs0rRGvJ6 z#)m@CsrYJOLp&NyB60Wx6z5>XmirpMNuL3HF1Npa;G?6;2S0F)3!{xNI|el2CztWh zj8RNM&+y>egy#@{%8aI0?GL(ATfzAmI;Ua%YHfp(@#Td8ctW}_+PajS+Z{A2M!4xS z%<>t10KNu)k;A!{t8ut$5yx9}Mp@T~DqbTzNHa znLe4cI=%w1f`hm8e&h;0p(AXcPjoHU`@H$xNgc2o3(^^uGTko1)Wta3w8c0d2mbB$ zW2i(O)K};i_3EDZ-om-`s>TJ6m*$Va!_~ZV++z0Kez)6xr+XF`*K0lxquowl#i1?K zeWT9?p0a~DP^KAwu`LuS0VDpZOdRD?L2$bawK1?#DK5O6hug@HUy>vJDDcKV8d2b(J*HduGO{`Y9Li?r zepma1+n4j(cb`9+PhRT?!`ZLn+u^#*I=|zIGmPUp-D64}-8Y}fwGiy0sqAj_2bb;P z&?ow#|A+2kJdvK53Z67zxI&h6-`PbX#z}UIMIMhuo**^y7jA*cV{cX|?9D_E8f^oI zN$Dx;M$?neJfwvH85KNXmrvvIYvN^s^g3}~8KGO}lx$e!1qZ{2&mhmh75HGdxQrYg z1RwPlMGApYS{?*Cn+;^f} zy=6dD-`55@gn-f_(xHfSOSk+a1nH2F5d|rUp>qaAK&4b-q)}P~l+GChWsnZ(7+@Ib z8gj^c0R6xB-cO2h&R%=1XFn_UK5JW8^G$llxB0FHQBSnGyRgPvqlf*UU6ndA_D{Dy z%sgJPABa2MkIRSQ^iH%6D#3@6@0rc2Lp7@hna1{F!Gh39#6s#lj*n5jYE~^3?n;RS zYyto2y#A&IT@X6XqE2glC_}H{*v)_|JnRN#!uYC#hLmPh;or``?frTr71yWHV#^S& zoBVN-&0k>hsUpy`hiWqH*m9Ax**wth5s5-wjIz~BdmA$*sQz(+L;aWX@vw$XfL=y4 zn<=mmet+g$)DnrPw?96gjE>-W4A0tgP3Mr3M>_v}!ZlxpV(*V%o@i{4>*0q@IrY7j z*DKJ@#&CbU8jB@r_L?PFyzM3WL?wVl$=wtYZLq7xsitMit)lp#t|j+maeCRSdG+9< z$s}TNLt6NwPSfWE#{<>z4q=%I#>9-3h*?!UJ?Vs@=l7`owJb}?4a)eSItJp>@KS#& z`s8RSEUV{`@?CX5ExpqjX5^cq{upPT%y8@S?P;5FSz*5z1!D&P^b-{9 z2(`!n+hiEXPz=1qo?`j{LLgz1f*JQ8+v^5;Yh3JcS)vbyh?SI2x0KP-8e_&KlX-_ZZ-5Rw*1SG5ydSP`$@g)-xTO{kV;+CeNmxul7)lC|fY-ZDHN#qS3)$EHaKD<62 zR=wd8wvtyc z@eRoZqdbEU+CLij6!({}RqgWo{! z#Gf#cND={(%} zWB02PTa=CKgfkjH?&5xAh(F&tVKv3RL3? z+yAwn-AN$Y*rl29h9_6bV`wqqutQ`blRlO{ZtCY{Tc+pyi@)@k@AKpZJzBZlWRoN3 zW=c#u`2zya6#k9BE~Rj1X*I7cimx|F$6mByf0OiT#XbMwJCUSwh5qq;mD^BfW}3A* zBgKMJ4mR-V!kp*YLBip4p>prVRj>Y6K>wnSe#v1gI?Ej(c+w+j^P&sLj~|`M#-Dv$ zEhzWj3^uY3|8kniK8;q4TKUL*Ry@OdN>1m%(9^ZtF$~E_>2AwgBi|DKGi?~Nv%+wq zm-D?m!L}kvd(6cj=y1Q-?OEnUT}t%3q}5)(rrR=deD(7{lHrJ#=qhk36W5v5$M-}Z zNMleVVr~xOlI5Y9%l>Ni49OA_TMRE0gy;MhtU54ds?R0qwA4OMFEm(_ZR8haK-c^Z zcG)nCL!>*CHj4KMWf9r2VxpmthjLZ*zNJ=J%$DZdyK=B=Gr65XKieAL+we;ql>EvS zBkdy5?3H2^n)vy2@r(?p>-`S6$oe?bVI2)(v95xu1Mnu2?0v7R}RDsgZFKnN1Pv554v?x?-S5M6Pg^;XL89bP8oL*0$waku!Sa{}y4=q2g;kd0RAdU5P!*;uiz2hPuXzn~6YP!n^6&C+FlkOIDF zu6jQD{sgalI^Pu^Jb!5y*CC=OU{(@sMm>c~s?K3RY=nPo0orX>A|aSouZ7-^+r4Vk zxq)l7Jn5d@vIKh2w5>n&xp>Qg#}1SkP6{QP^h9Bf?7p?H-QWF6lOrN(;76CSKof-b z%^9r-450-;he78zoi2>#GJYG6x7`k8N-w#LeEnDo-uo5xxmqcid@_vfwI`bkQdi#M zLq#{67Qw>k=&B>9PHR7RovpgExpRJO?pzv5)L# zf=>U7eoe=$$HelBV(fsMB`2?2`r;u;%zMudWhLz&JCKPI@r`{4zVc-xcbt1e zPNdYT9245cPCkFS-+#RLJ4WE3RU64L$u0n|3zEWxzPgtD1Wt4CA{`XlqT%+HOr~uR zuc_Qu7b!=wSrS{+GPiE^N33D?g$8hW(CEFbQ4ztYbcH|PYGAP)3s$ZEF`m7=Jxqb( zQL!BmrVN7P|2nzs8zew4RY0~(UR@6py6Af3-ew#xs4b>&2vH{!L1tA`BDOve?3qIb z+N@%=1|G_`lM3$5XgG2+{CAb=bItt$#bg&O1#JWR-p9+2Uq^WoZg@+9DmsT>A=uq{ zA9?L+>YTXrKHiqmBO`KIpHR8-K`A|`yWvdh`U$%CZ{gmY{E)Re^bWG+R~FO1_HbU~ zPml8HQO&FJqLv?!p4m^!#apr>P&TOvbNlo}*ZS3}hevjipwiniU06Xo^c@Mxnn;la z_f|zOxs#+FxU&{2m3VD^PnRqz~jhlqsFjo(c!) z)qiqvD$E_x+dN|X^YEHJ0^9gP_CHV{z?tCm(M#}`-t;PtRRi~@hI_1q^JzDw$MMh>pm)L1%9+J<_zmYMFZ?@i6QJ2uTc@L8>> z?dR?K);=xNda2GwlP5ZAiY7*7K*t^3MV|laQ<8eG6ZLPCP?>FU-Avlodl08H4zwIS z)@r6bMjiiD@-Xhde_AH^jM)e0FJ4Dq;aYEzp|#bxHMHl=sAz!JgQNW^&4#g}O0f+K z*&2#V1~coUS!Bvg|wBbOmvT9wEH}nz7!7IJEOW5iIr#i||Y)%u8LI2@Fy; z;f~p;O{LCFt&Yo`BptbdD)JiD-;kLXe{#(;<%n^v#{O$L2l@P^o2^X>SJbW5|C?tq z^h@a%)aOmxdDpu2p=mDs7o)T{ay22LB-k5m)h@`P({B@?(XAFzYwOEIkLHDf%KW~j;*-D)DEgdu(8L|+kSJX%d)(FR8cL~_L2JA=f9d4o^Y349p7X?(awUtvC3*b<^6Ldelp^9{Y01TH$SbmeyH^D-@|Uv=}xjn1dt z=9PzOpFQ80{H#+bs_Cg;gWJ2UHKKh>8+}3&uNxe{pI~_4`{lv=RLrIWI)B}XVuPyU zHy~<1Bx72c3$DrBL7)Xj_;e`u% zNo#2;t7law$-WlhvnPd`heB+qP4hVI&FTGX;qY~@=mVKbWa8RK%r$m6+VQv{yIHAJ zrmNq(!VoH(o|*$(^nkl19SmvGyXs5@{RxpqQTjC|EmvC)IDcT-n2b$3)*4P>V8bm4)C~j(Oh$ z-qF~i2fhuMv9IG0D}kOgSvsGI;xx9bpQ!Cy2CdpwlX|n9c|N3 z-du~aK6*7?8*w5w{AVp6qQHK#S+gn(WJH8hj>l|65tgU;++y5r(K0sG_xh8*lz=Ba z=zRmk68v{($g5@M0zU6n727Z0441I$;#9}6@y1W)jXUkG*_{$3lpe1_J*{TqcImn7 zpi=AvSox1LrABP_?YMp`e0XVbLTagrT8KSyMJcI6bg`I9OBQEUOqfEFVtFtGd{k)9MM(gann0PW#575@lUBkAwRFd8HL5cW7Ji zp{ES~8`G-zxp=)YHX*VlN#$*oyO~YDPZteJ6s@uAktTUD&dcs6T6UJ#PowBIpo`kW z6Kf1VA>bdu@8V*@-X3FmZK{86?&G%GUdIrOnNLCFwbpg8NcX&i^?HOUBG?z9u=VLh z-3Q)p{!cgz7OmX62@9B(VQsbXT6=K9R(so~u6{_+=)z^?cet6EW`6O3}X;Z%d6*@}xjakMyb!Jbfu>z!9@ zWc&N0T2(r-O5SNsLgcYMY*a;D@pDhBMFzh^)0$hh3>rf@5>b!=Dno3=^oQ#=2JGAK zbUn1Onq-*eR4?vSVL*G#lcce@bFh$dONMEblD0K&wz4=jcv{L`mmJrakAifgoWcAs zW!;%Dg}vxyuFdSr4zRY>TaDC-ac7Y5Vr5MjUz9OURl1%YhuL(GMHNO99EY{Ycggqm zZ@%x?%uOjpe6yIU=P2S)9$soPZCj>vN_ssqu}O4KN!NM@EX60OuxxCb-Mh>RpRWb( z`LLVfO(dr~xb*>Rl6x`%6{@%NuN1(7@ng8y_!}y)@^31SZJ8RbXL%UnybXd5ai;j? z$tR)WggQ=Sno>@T)jWj$onA+CaL+XEGakvHD&aD2LwSJNy%@Zq&PdVe4b|#E)}>*@ z+i|S`7$1kULZoL4*HC`fuZKW6d>u;^59Bu2T!O3PZ)9awQywy|bx2X4WZ!E#yx;g9 z4a{kR0Rm1SZM|`{U$X^qSoDiUwe)f2OOZIB{Oot^Qis;s!X2=YZeM=#kZRW+yCe8* z&9#T#A}m}ieXD&`@`mzHlI6woaH~+>YbP_Gf+fGqbrv4YxDD7*7tGjXNHy*@YHQOY zUBLH|;{(4cx=A#%L#)Yr!8CPgxm$&Qpm-UHXNT<=+?(EQC#~E(J4M_t-mLv{ zZ3(Q%@xp!lk|o4&6>)9a?dD-6sE75diM4k3nJBWtvpWeldx=RNO&bB4rzm}Pv^sRTn-HtYc8c!DM>T;c( zt%`A10ajnGE9lw%Az7+DP|4NE*4wZvY4$^NDyhWT+PX~~CK*)Uu$1u8YFKMNxuT)e zMi|Z9Z2~U|Y=5eDFpMa(K53>dg-d7;x|b|JtN&V33a`{j?_ZM=tiQqaA|zIfiLgQ@ z3qknl5>ap|LDqm)mLdnR2a==6u{f1}=(9FXje)(2Q`I_@We3Ny7aqP0jY|J0aPp&j zlzVkA_PL($F@JC~WS!c(0h*Ok zu^nj)c2a6?=Fz(jFYKW-gE>DmNG)D+QT?_QHhnYZ!M+=!%H#wWSDe(r@TZlT(92-= zPu)ab?kpsbXZmT5VyujktkNFzKLZa|f@6*W`ClnFvWBxT~*>PzY# z3G@o*`~4hj_^cd6>w|Y8GHZO7R9okPt8dWOnkm(jU2-O=Dig>&Vo1`)*K%0#DF+l; z&>tJC>B(#xtJz@V(RLHA;Ai&x{x*b4N*0SxqcNUJaayrG3`(5^oMa29gFwTPc=9Vt z-k~DFi`rpZOOB+!GhizWB5raFmBX4nIW6wiX!s9zrO8vyV9DvJ#z24MM<#+4rjW3Z z%loXq@2^GGxb1C1M#>ba2CJG`<01nUTnAo1CCyn=SPh+iQk;O6mU}Nnrdc&U-!s%c zjASypiIm_drTC#r^9Gss-+f=bLZa;StLB4;3J=RNmmty2F7{>W^qb-wIqxT9KeK$p z$v*&Ja%lOiM*@Pw!Tu7+emSJ8eu9*;vJv!X$O%8nF~K{Z6luibLh%;2-X$$rqV}{c zfTd7o;)HhNTi<=WtWuF!6Dsc(YW{@HH~-shzHeuxH=}RCJA-DPSlmFH&F>H6TXFrt zRPTR?zP9oBEmjVf37)|K=bO!qGil}oI08TV;X0z_IPht6fn00j%Z>5?L!6$50wLql zyL2ZjU$2#DQ~J1kl$*~HagHQe+v#ThGB)$HF9;t-&CtxN-|-u_lUF!o zxom`TdU!vGk?O82wcf7TClbYk85%Yb>^PQw9%sS_@6R;xR#sCBnQ}=Oc&+NHgy7s5 zWO2sYBVgH9NaM_-aS*2?w>G}mVU(;_8YGiOzg50YymYb)kx zofmnyR_`C%2t5v{1%g?=yl=q^m~J%$K56cZ5mz<3=04`@Wl|wnn@$^l)us=nVE_a( z43ct9RCBDIQpBR*`5LZ9floCahD{C7PR|1NOr<0|W?}z1E|cZ>4!z%f7oWb~ohVr> z#HeWKpLiL}kc5;V&gs-KDKsnjE_x@;=M|NidE$w%V*7$5NLJv_{b6PT55omE>ks46 zW&~0l!w7q!;Y(i$RCuUZ2yVGNFb;AiW*h>K`^d1&&Nq?m*Er-JP01)lMu@BubHI~G ztDY9@|Dc^-oZEjimH|uXnCH-XG#|wN{&{H(Lj8%1Tj?T3HnZxQkW%5Wmsh7%ofjXH z)qRXnbZYVPikFgMN)NC?-H_mUgu}s8m-km$<3u##CVWg{(m93j;ps(=xkqxz+tSLH z9NmKEU`Jk!M@4&2@|q4kH)Wgk+p-(v;H4Bf2Ap@_6a^e}D$1j=JJTfQ2@5KITgRuu zE0v&xdr-#{KKed980KAm)N9Ft)j%{@3#S$9ofIEw&r}27&V`sR{g{c@0DrzbF+=MR zr(OA3^vr$Du!ZJl=k7LQQ>F3ED)qyt^spe|S%7YA0~=QU3~%043!xyK?@kv8P;A;` zs!!x*BfKfPmi8NI?-oM{u+9qWHG){w%+0MKZfe803PjqW#I&~L6vO;y3qGV&x{hG* z04DxTOYytu_#UZ_^+ zoM&`7ijVWGl)#iAaIkZuRbZ}ee$}k<{VMqf(80wA!7LTDn>BEaELbP+N({!+FAdS4 z_+JMqJ>*0gPflc!g9GT%QB-jaGuCqwCr=F_vM!CY67f|Y^P*U!dl7K&v7;lO@{NK( zI#%L5Zqc=%96x|kL{fJB0{V_v?7)o=ZU_`$$abROo4TDssObTT#VDQ$dK#3Sk~cgnW6aI_PxhmGw_2TvC%A;-6^+6BfDD_q4Y$Cqid_;Cv>|{0}He= z^VjoO9x6Uc+jM-|i-#$%3!w z+n=Vffc@d8cf|UCiY6@wpN8W%dfXDDIt+KEF^i|B&vUJ6YfQQ`3zHc6wpVXsVO9_2 zf02N+H{}=7t1}N5?hIow1{8C?BdDj~rIBThxMNY&yra@5#z1WqFk|nvXI;Cq-Gozdn#|LVF_M!9iHIdhDz*E4wmd&D9qH}%tvOTi7o?q z1IIUeQm*;@la5Kk1Mrf8)%#1g9xu|}1s@QL>GC57tcJX+%GS*oeYYFa4vwxw zWgKI2I=1b`)ukAfFyG?hGuV6Uc^@-UP0}`Cc3ZMuWLIP_r z&z`8?&?8X{hGa|4L(Zfep4`1ZPbHhFc3?Qqm}$Sl!EMrww?HnlbJIt%=w(TkJgWC8 zwrULU4ti$D`IVa3Yw9g9ye^Zkt6|-M;G;Z|ajCNJ`+fduB`_+d79~`4{Aw)U$#<%o>g$lIkq9(KmFhio{J3{~#d&%C7HTi|v*`qH^1CBe zOG3G%Idv@#dko;ezfG=s9f2-ygoiFp$I9&$MC>*UWH_&I@QiyDqrIydolBjMmor)r zy0P)?^LO9|EV$~j^`8Hr>Q^1W_5Q&1o9(G?2w8ECcQi6&gz;(dW1iZ}d;WznMjUKa zFk|Gnr#gDwQ~T5kvyU)7c42ezGamoJ_J~l<=S%`KKmv?A5@Gx_q`nXR`j~AzFK(=` z8~tZ_c=f9u=*svyT{wo-)h_iJGRMkNFJ@-%7VXGl4Q{hwa|{i4NjZM+QBofffqLJ( z3K`t1MmkHFKm6ONx!_WAax2@!tfP8UGhli--mSSW7$kp}X!fp6WRV6c#((hz~Y?2k>hx~EJ z#m|qkaTO3;bxF7jOu!P^zSDx49h2I~)FC{^=w`2=_!Rg{y2keep3h?**c@7p9%x{h z81nsCiLIpw`0fPRJa!BG2(va2aav}<_JP)4=(^oBoBxk26o4!`*-QU# zS+xI8mVNk4Y?@eBcrV&OmxX>^2J3h%EO9@;p3fx#8RNXQjnNoyEATCF(i`6k!t8#1vNW}G3VdoolxfAwtI zCM6u~48v8SFmXcPwF$mV=z$}`2O^+!gEtIBMW_*@avFp=iFxCjhO2f*E-|m$#8xHvGR2PI>b3iYAAgAm<#vK?_lfE_`u_wo?;WGKq}+j!ia=B9P%kz8taopG!m zRYtB{&!~dpG6f0i*6)txnQ;oSkWJm{l}tCEC||j&gxO@#@@uU>_m*Y&P?&F0B-_4MY{*nVgfueN z#QV)n-`7ha(o(=!igJ9?3daz(ov+ug{C*K?VUKA~6u2kh;7$_)BIX2z9^VQPd#vne z$N>C6K&JCRg*;mkdQ>AQC^*;b0N~W=Y%hc%^S7Y{a@Z3A%`TPl=9Uy{A#q=pZ_^ zCasCS=*wr%(sG7~O_Qw<*8zj8Oy8mVke)8&*0^KZ7Yz~;-0#!iNPor}eSD4X?F}6i z@`PjIv<2ucpAM#x`Kie$cP=t3CqXm;DH;6pjk8xg7AFLnKa&tNW}b}NXC@x*t?QY- zwW5(xC9a5wxIUB+suKaSPL@@bhkra*)(c`$64vkOcj0HVPg5uC=;)B;ws?vR+;c7A za%Ywm{R5=qk}rT7b>q#DurrMjkg?u68&L~hcQGPAA%lpuAh6@nAztl<7Pc^8BMpt$ zon9Qs4;y}30!CVQXDY+`BRluco>iG9qiJGvC-xzHbfNznp+#n;*O@L=yiGxTD@h!L z(8v@Lo0^%ClG|G&f@&RFRL^t_BBlxhwT850>9CyJO_V+qXcNYjueB8(TKM%H{sZmE z$cSbbJm(ov!|s?f>N&9&!}(*e5`e5I|MfoKGwbjqsMLorgBfK$%HtELwZT*X#w8_y z=(5lu9VC?X`ribv0}n;sOYu3E_%T1>Nd@rGhbt_}ahC~#@9H~W+LwV7P>=%CIRl&y zp_M;Xyp+qiq)o=e6`u|OAChbbz|eT1E$HRtiF#iV<&S)7K!=b*J?Gi;w}J>GAAp`V z+0f-Hp1q1p)VWvizm|-ZoAOWBMJG7S-2|jWJi{6 z>R(+Dz-(03&CgWP`wmY=+k3!pV4IhAojlVZ@~~+vU*q5;_MUEyzn2 zFE67wU_t#6ZGO7E1WY9g+1QoQ@_P|NtmhC>nYN>AV%#X(1Ffo^zvl;W1FjA!niW3c zP9xJ!yzLYKzS(>AMm13I@PoAQhF3yK{Jj) z#p(U1Vf^vymOGH);O2hCi%UzJO+gP=6L>sxqOV-oo8`>jE^&R}bFK0+2wSyX8hQCb zgu-2g)#RMq#^$!)L_H&!{%Zz}aA3gfGM)v}bIEn?sgQqll=MM}iqcDob6h3pBH5_B ze(*AD=t|RHD8^m|#?Wwd>t*IdVNn>SgEgYa2B0{<%Vf*=*vB;JvKl=8(j)LVivOV(X9) zWluh$5TzRhOae0Ja|mfNxDCL9cb)BkTJpI2Q2zxlW}wigFx!v(;w;~kHaBfOD% z$jk*6A!k^uS(81WJRg+nvcvbP>Zt$$^!U)>xan{8XSc5sil{vH_O1-t_)cox0_^a0 z|$nIO$;Qh?`}X1 z2?5_P0N|y2#X?ZOH0vKy{(qX#(a8!%)(1DsLoV4WR07NTHG(xOoCplm%|4mw{qwPy zGZjG4VcQ(%;J$r(IuUK6vJM3p>vdoHSoG?7q+w|dSlp=NMd$@%{JIHD$aYmvo#@;I z^)wF1%cYL!l#`gSCSJmn%NsM197&0ZiDfl44=WyXX40_Udvl=-nb!3ftx`#)8XzJBhBZb80^G9IW@IIkL>!Ag7`8XrvFjFK%?YpU~q6F zu<8nX)8BNS#c#kq*>}KTXXVWcf0%ZQkda8Yf67+y99E)Y1P}Ix=4Y;7zupd106cU3 zx~?^;PXCJMxmQj&w{3o(DdwMI;?)7gXpLt_lm1mqFVtpw6N9QPnEny|%kSppHqjh! z!nqRbMhj~oH=&c&B{}0^07*@ZO=4&66d=r$Ge8a2!^Wp~+FvQ%sHv?r7q)JA2;^;5 zLg^P0s!ZRa1-$P2-P=z?&q1kp<^TZSoM8uSfn?CIAKn!Qwb5{fHQ{g`z{#N%(x0He z^e7Ju^_7VA{rQj6+cY&ncyVAn#rnW;%}Udk(JDZ=;jFnp_oBRf;-*X3e-|j%WhE%_ z;&FwGbDm2(B0vBH$z~p?`U#%QAj67hFDXE7$YY;Ai>v?et96FooOPSl3;fd2op~tZ z4$ z)@-ltA8qw?XmA84gbjRo>j}7x&QNjwbCE)t$N;OUtK7Ko8@keGJQUz`1Vl~guADsX z+eJl1*f46Pws($=ue9O9=}iXWxBfoq2RzwYe&f=)(R9VmpA0wx{GX8y#+EYp>C^0n zGe*Cxx;hWk6;^F2-}ar^3|QQ^fpG6Jp{DJH6@btRSfonw)m2qKa;j-r2Fo1;DYRL^ zi%gsr0)+p<3l{**X;q5de*eO)lje|&+Z~))>wTD55rQt9-@&#cR7~ZZMo_-se$R6; z{low%t)n?|FBZW|ItQfa*Rrzr=Qwd6I06(b`M<%6&juS(?5VjJ z>FNB<<+20<}>qwXP zeUq00Ths>g15Fkg9XaT$2RPM*zDffrubj1M%JnNpKqn>3ia#yDX<3UYKTHgK zkbW%u0`IJRz}#bhuVqPG5Irl|vlpCd|IPvg9c`kpz?XOTN~wY456f*0fm8hWVzAEP z3mGzFYE>hH9?I*)$bz%8!E!Jz~dyDSti*iyj!+-iZVb z+BFCY3;$7ize)1f)*&i@iwof*IL8X-S5m@&r6Zu-d&1!0;F&A5(#tY=v!{>!@1Dga z#JX=yBCgs-vY$H6c?846O)o7xwIlmzW@vpoEMawplF8YY=3flLDSDRWv=MTguM-s@ zq7^4z2;T1M>hi$0Cw}_5NYx%=IDPmA(NeFq12GPEK^Hev8=h!M4Pt^y%8OADQtr6% zIx5nip#gu*ClBz+S30)-zbsKt(|gCOQJiz{PpVrI5JlR-ug3Gl8IFwCm@dLqn`a=< ze9O-~xHykuoa;|SVtIKvQ8+9OjW~HzN%stt&B+MjQ;dcLz5{}vo=ZfMLqN`vGpUvRj3cI_DJlOc22b%9OOHEsup`H<2AJH-_^kB{GKS zjXZC5NNbTEnbWf~zVGhVum8)!AmUaaY!}U-3LNZN!20)CaCxdV4VJP!Lw_N(u z^ttr>RBUS8d`ESKqB*Z`X3urSL*GU*ymIqk?_*>x=@<7p8eCyR<}8>R)3gVE80FG< zr1E^Yr#p|JDIx^Y9}#N$Ip*X!0ZMD_&z-!poX4R!Bs@IaW9v_sxUw?RQmp?k%XG|BOh(LcC*y}7DwgU1)DZR>r|#mr^cXO|V7#Z3^ueCa7VpRAKb))?f^{~he{d-4$ko+c4 zDQg11U=x<9a`SWfupnexFx_R&(v5n^S` z%~4IIXr;rAD+HT8*oA|^6u?l-Ksat~Ap8zzO}=G1EobS9Q_(Ib$0qgB$I+s$-uaf% zbxR`jRWDCsaih5h&MHSfc4fBv_Rp3oj#B5P+Vvip3n&K$7h74SJqQblCY(BGC1AN^ z5f%~yitQ+g+xG|jDbD!)byo#nVL{seS^xIUX#wc7SQbJso`+xO_Jy)R#Oib#3?vB$ zSP@ulnqpwpSsF5d2nH(dd%@_ex0Imw;)I%u)!PFwfm0w*b)@Na2A!f-!*M%-+F1O- zE)$9W4LGh;JHTPZ(jebTlJtiiG78S>KMGGORhW6UJrz6;r&J*c{Ujk(Q_pLiHK4Ua)3}4>QaIqA2Z^D7 zAw%=wm5TL1x`gW$Q4&Uys~1$d@!||HKK`?Y{O5}|cdpwonzbaCO0XJpKcbIhQjKY2xC?ERnywigO%`W>z)lXfa9dytA%-5o z6^CnIRhuM${=t<{vHqJe;<14LzM*_$!KaMq=r1*MiqeJ#*sYeeUP9KId?-*z{_72B zMnXd}o*;o3h)6A@Y-!S`n7fym;5pgb_v5aoJsJ{NSRkxc!7Az=f!gSBxSX9;nec{^?Nw`85)mAqPYT0KOCn*a6jCPrc83~vTG%4 z8@7V%sMSBHPyD5=M{+z%AN^=9f(r5yc*S=Fyk46vM8wYYH>Z4t+ulNQC zG4sVhnw%*LzGMyw=z>|>j?z_CFvIqjPs@Qr-VIPh37{V0xoR<${vF50OF)51J!AeA7trEsvnTrdIL#%r-Q zSr3n*Xs5|aCXkZ3MFjn4{||3~z`W4?9n;0?6J-hwde3eSlmV@`?FfRknjoKNfOblJ z+S=NH2BSfl%3vYbdObOIcujC#C&Y?VG~x6p{`Ygx@!}#Zzhe2Wb(qPb*`!aqy#99X z%SfZN2~k8unlxQK2g;m3;609rTQNl^UWI4tGSZrbEc+!TvFFEy)A4UfP2k2#)9!Th zqZbA$By>!s+gj>Om(D6i9)LRzz45r$bR2T@H<}|`xt}$~oiJ8b2o2v(5fRys?fEk9N?^Q&fS0}raM3{GnN=&e)IyFNS>M`l z9)WAW8k|IN_*s->SZaoa*k5@SLrE}P3Kjv%t1JoSH=4v#{x7Dx5$fVg2r-(GkGvce zPkEtgP^ePq=O4q`PKad=xET|uK`A7|Ynd+0(66#-Ix=p!19nJi-;r)_L$)wPG6J+v z_cH0Ov!x)LkJ_#Z2N``g`()JOmWxcK6xDRzYA+QBdW60kpT{9a3tB@8tRJ*7(s)!^ z(y1(8u>wyCo(`I2Cb2O58F=`q1jrukPJz-o9t$eH!AbNYjq(c9^#9QN`ttrk+9gpE zoz9r(zoErD;^$`7sw(Y`Qe6~Z#Z;V$7hYFW5TK!e0*0h}5+en?NkVY#g$>7ixmmtf zkmHP_p^$eE@DOvKX!KKmzU3u1X2M%0dNKlA?@br5FzzAReG%A(BYsNTvv%sm^(Vgt z=+92Oc@tM8h2Sc&#+l2B9RDvTElRS|{fhOkEU(EPYQcZs=C)3a2Cd!=l_EZxYK*~E zBfSv{kBXxRoghtu(th`{BFZ6MS6?Y3myyM4Dc7Uc<>*Vp;Vjw>8*OFbApTzmuU z8x4$Ch+!w)8gFJnG?RP2*syLK5WIf_w&ukupN95yP=nJuBpHOkh%|DNDsu8CG@3#G z=yX>nByMYsiS5!~Po_pF&qVl~;;X7kE2xWXe_V`fdeLcbRE33bb_c{ngEIl~joKLZ z<{=Xv$yiSP@dX?6TIHFmr(N^=`K!`nx>W1kL&ZZY1he>WuiqT0sD+-}LofcSbHn9GjW_ zEMh}a5I@r|;mYn!kTTxz+UnJnQy++HRF2o7}B{2#&5z7=<}|M@!t)N|$S z&7ucj%Kdfd>2v=Y=KJo_M2{t+L59rub_6e_UCAtjk(eL_*^tt*f+B)v5OMPuCGjAAl|=@seh3o{FZ#y zpORp((06SBILXMDS>^u5sGU>qM-MNInM%Avw3sB$*q@2yPLj`&rL5Xb84|8w8`aJq zC2lmL^GS`7uU`JG(*S`0BazP^HU7Veng*fie~G`Ke1*^#E|*LPCQgUf1rvNTUf$sB z2TnnEkrZHaXo`Zoy`P0t%&h{2QVI|Y!DrblX5zZ$8+FMbjhcoS&>9j2W{QAtQWDNq zBzFGjF1_-qkW%QEcujKp-uJhc{x^4ZARiHZbLHPA1CY{Q-ideO`#ff$*5o$HhaFZ@ zl4y%>FG_gGL^8WfP{^vymE;^vC{895kR>y0Wr!!m_M`2-K)Ks}D>~+#iTY40j(hZ% zVvzhhDPlBHms%+ZzR|THR8`Wt6qB7s{^yB(fSmWS*?+d@`#}U`(YN*KO$AC!?u+dh zId@U@`po|dz;hsjtN7RYyF8z%FjC9q`ev2N3#Al^TmrVF1BSuv9+R`u?bsrMN7XAm zR-d;%B<}5dvVK>$5*LoSwb&&V_3>d!kv+JgoMxqE3Id)rq4A~-TkJ)cL%u5dp)>-{M(ISflg`__R+;a8ufTE!MM%ByDY2t6 zSvZ&Ep-jgzs+S!;pQT}OhW_6QFt4n=p^Z1+zJXRFnlXdLINuWo|E9q&U4!bO22Wig zVlH;40KX0tghE;_ibH1!zv&n5MlGhqCu2&rCkK%dSj)^V`wU}`N|mhkmtJg!?K`&` zB#4Y*)u6{QMQp5pitu_C__Y3MM?&T&WmijLP1@WP(}1qnUvE8SvTW_5q7mDA_kXPC z1Awb)i<9^wv;FWYg#LlRXCR+$1p>;Ub~MblHc&{#nMhN$Se(V+Kbhgg(1U1Ym~#z+ z{cU)Ch43>=_W5E*@4OJqXIfR4U-pX@X&q&Meu+Q&ZIe{0dDXZo3AZKP8A2B$^b26vb~{;2A`y28a+L_%AKGee*H5kzs0rJ`2&8tfj)8(uVqn>?JaK zCJtNQtM5!re7?~nL73xZ#%+~(r2^buZ`FlOCNxoXGx|(0eJeqlUA3Tr#8 z+*_7CKwv`pn5F9BMsR|L1$ltYVNN6gfD3GNrF>Qu6CnHidQI{yqaEknt!Vha( z612pkwc0HyiJ1lxzkO7JSeQ2HPN^Qh>8whjKz$8D^z~AHMck%oxDz`vBwf{Bim3m( zY7*)L_B-|ZNqaFJ0BvZJO+n?r@s~UwI^T((w54 z+}5X1^T^MOOJ)6*CcXk2H?T25u@I%o%o8JhD68_%)1k(nH-}bNK&w$c=v(}r_uc@`<{vTSLD~oa%4o7VIXPpv<2I zrrTu+fIHHTa71dagt){`e$Xo~i*3!G2Ed@>0?cEiQe=MBRfnAr&A%zV<5>Z?f`cIX zVv`{6cCV8E?r*Q2EBh;Wb^HN}{NvaAjxi4^m-ETp+;k#BZs&Cj|9Cm6as?)Y6jCDM zK?YGhBB9xuxGmkGeCYS>Rjoa#AB~9;=E+k{U(oI zPYW2tzIwjBJwWh)0m6Oq^XbyZ=1jdEHDopPv)5qZ*MVkv{S#Dz@6;DqW(V~_b^!S9 zp)4I|N>bYEd-V37I4tBt|6c*%tnwQ|Nc|Ta z9)zZMEY(7;^`jeD$_a6UA;7o1m%180Hg)gwAyeLYu*@VU*IbMIlX~)es)HoY;>%0b z=ax!||4&_C9u9RMwQo-%6lJN1$}&%sB_hTW%5E$p+bCQ1%FbX^QY6$2NsO{(9sAfv zNos6m-zg2*8S601`}Kt9z258h^B;3DpZR>xa-aL$=OAzsOsCt?3j`B0mNEDdQNSEWIkozD(uncdY>ZU4y70E;rn&@<^vtexfX`z{Yz!>< zaKG@JhyfBFJmK)=Jr7A>DzLYzG2|iD*lPpKan+~8Zic^Yu(D^ji!yx z*Ie*luy};{RS*2bQ8vHWd|X9rWgYLldi4hd%1Xus2$s{xvuM1fh{R2SUQL3A7J}yj zejr1Z!V9zA^<#ZghU1=|ihEChZimE8eY7-EuITe^Ya^kpkr`2YeA#4*YU>N2G8}=Tx9jOF+u->bY6kMyw7oh*^+KTPFwER?k=fJ7Ze`fh7E^@hM&eyi9IrFBp>yQ2gEa1v|LoGz|wvGljH3z_x_ zOC-Pz-ota!J>mTKRIj-2qQ05E%vx@lxhV^;`EPl)Wh8$f~8hgT=Ps2J%`$yDcB+Hm1XxWIAN$SHsrA zd$!zL+=NEMMx}>1tqIng(=5BGJM+ia-n!n-z4$s&e*C+n&H;+JaL)dGSt_qE4M+88 z=r+&q!b!v`UZLEg8J@p7WMZj{4VyLFnH}%SS^LeA;QSM2MwIo@b zlHwhlEz}J+xAVygKi_iUv^CEBXlPUUo0h8&tit)2km0Xeax+?8j@u@URb7Hn+n4I& zO{9sd`Lrc5C-;S0gIyUrgCrScWvFwxpD6#vLh}x$y^yuRluCo;?BH zO_iO|ojdCmXOqFC`M{ib-PbLNJy%nd*tIOA$~VP-PdQ=CdSeCqekP|g&Y}`8w$c$i zIK(Hwlv(GmQYw;Hj3j(i4(5(Gyf|IgAxh|Tm2djab?GLlOQKUUf5UoHA)KJ>`W9O+(WGNTS_;s#|fcTu`_Ctyg)Naz) zeNaC3z7$*l=7GMJ0vbB0rLQX-bA{;()S0n}w8F})Y+bY`DjY<3v~?KY|Hj;i?`X}&sC0$H zW-D%Gl(=?ZeBB-NW6^o-d@wH?B|Q`s*XCE3=N>|Ho;uk&@+(=UmM`^9cN-l)&y<GcE5l8!r6EJ5DyaCZn}|M^aF73u04cY z_zsnlSDxsYHc2ht7z zHdKD+{fgiTx>%%8VEQAa6OD}R6iO^y@0HVU>u8^TMcwZGXh>SZ>&!t`q83QiaO96R z2J2D=X0 zeK!X=)kFJW@blxRhvo#MEb})HVw3oVdxzEG?q*+)=63kn8Zzr3F){4gR+;(QNqE(p zL6tFL+re)zk+iAA4it$HbD2Q%5MRCAi<^rs(>?m$k@ZF-xZ!Lu*AqmwrL-_RE2Ai4 z?57c-P9lTt!hE=Y)!bHky9uGz|FdHM3D#ulQ(V#)y240f7_agD{~zG+5RJ$Cp6r%P z-T()lSsD&t3!E*Lq4UT;tC1|sZ0j2K5SCSXl%3(H_po55fw5};;Ty3veF<8n(zZK! z|GfQ`)!6y@eXvgn>%>SGX~a2mV7dU|rnnqJ%j?n&JJGkp^e%L6{h=WTMwZDesH%Pg zmzY*i_a)wN@>POH=<}~ppf~)*s(&7B{_@GuOe8-kuR~pY6-_(8)yoQ4e+zd7WLybEh{$X=uaR;ZfJft7PLp zE)GMuG-4~}p%vL!s#oPHY`5d#9`S=t3B$$5xRU1lh3Ufx;4 zj0Y2cPPvqn3U>#4#UEdSyTf3j+c)ZsA+5|e0{Wyleg??K>T1ZX^cOjE9@$PP$?o2t zb@+xQ%E_mvzhE3I28(Lre{LmQP9N0?rP)_%Dja3{3(qFXj{&L9YV+FUzf3^HNfF`m z-G*NfThG-}^Ufqd^l=k4^N-5IESUk${zrOl78)lsmPJk<{hCBAqLGT~C)cFHtj+~H zJgrL&vRAB>^0r=5-x&7`>C>3$<1pSCVEJMS}3!G%z9RWK5XN_!it)NdqXeY#9?HT~cUebM`JIb^N+0U?A z{xHlkC1e#uSqkydX*@e>D+j(T?x!?Wu2)#JJ+pim-Vh>0khv&X(ouBJLV(+xG*%6- z6w)$N-N}CN6MB8&B_wahoB9P^5+F%Z2q%pae^7E}joB?QC)%7w2XP-7cH-6gu6okM zPU9py@5{?5zuPDyc#(f@s?NSYD1+pHo0Ge=yr=Q<7qw?LWh=(Y`dB-8svtWpr5mGX zFQI=;9u7XIC?%`))-eDxL!+Y;YPxw{-RwipJB*&mrWu^XTObpA=;wGvoqz>uQ4Okf z5j>xiM@?OVL$37oxjBljZGTR$T!=);+CVYC#n7t%2zH)n?q{m)WRv>*{?U27M2djh zFuIav8Vepc6370}iW8%Js`Z-o&ZnL8J3NCdBfTaEf#D{d1Kpv&0ywn^=EMJVdgy%6f4Chqjn1qSI4q`O<2U8P;N<(K8n>cH z%e>YQuJD{^&dKX`b(>Tw$S0;X_j>Kmp&ao6UZZ9uB?VboSmlW%eA}<;`PDX?&);6B zzt|k=tQo1>mTT);7Z=&M6lR`bfCJcdqLyt;nb#%R<-K+YqLB~C3xGcg{AK9AA6qtc z{?|W95y%{Q^kN@`WYhRUTO<2wikd6R@hGbs5gSj5r>z2H|-E$Edg@95o z`LJ8jc4mn0q@m!4vR8>q+e6;qIUiFKsO0*OSJkpB(T&MRoWf=bL{deOp6%WrC}xfv zpjNo8tbJH+TYGaGU7gAKFu_`Wxp1FcWFb>ZS0t1&LEaJbdIv+NAAK9Kp|=xxYw zr_mwj97pm8%IZ8~P#m^nrgydeB1ZA*=0|Pb<^?CRWYQ#_=yxGcXp z#=XKAsM2>8@keevJ)i>tckW7YX0cJOO=jEB{`0OYG|R*L34@9;yfzS2@@Po89oL;C z8$1zLz6SJ&CWJCUn@@?+!)ZDsH(Y7N*8Oe$YZP?v@vw^`RrfADr@wEVUe`B)QXq#J z2)70}B*6%%@E#;Z9h^On+qqIyC(uT9eLJg1K6Q!(yqA=^ceRYJ zU>3B1qqR8-%co&NeR=6(hHw5`L@=h znBx@Nos?7_pM=X0u?3OC~35~nd~fl;sy2r#Rigz=m5 zUaf8#>iDzKo9)c}+#NRaEqkMmdd8C04Kcp@d_wFIf?6(JZuG!LM#3-OX6a!obSEZw`>U#&>2M4r1S*B*DUwmm!nSY=Ax+?aV{0Ib)WrYAM+SvVFO z*783%#OYnp7~7sHNwD6i_D}$&Vl}uaNkUS{!a>Vx#-PKxCWe}bp5BF}mYg%ASuS}< zo6Xx?+-YW|QLY`PPKDv|wuPCMZ`-xbqWWJ^+kOt2Qqi>aPVXZ|$G6Z8)cYhM>P)J* z$kE`7cVugGF-J26eQzuTJV6viNa-l_NpFN>iGF>WwxYr(!95L6HMY&)u*+=gKHRuj z8|T6GrzT?0@6?1@m*3Du(eLq93=D-DKw0pF?%Ptm6{;&Q4G&W=_l%UMSdcYMEZ{{w z<5RiYK9XI!Nh#%U+cIHd&e`-n4AIZ6-rclqY_xCNZO}3Z0!Wq;3oB5U-~~D~a!}uSw_RTG?$ToukMmOJt(OV z`Rgj6|Cd6?dXKc|ZUmfx60D7;6dY1G2=ZyEIU3_vUM~$qX@i(8qdhxL$496|d~scA zP`5e`E~?{k5acsUCgzecL?n~hCY>$f{Chfee4r_d$+59T}{_Xa`0@SeWh*=^1D!VgH_nVxA$k-qae1u zaEV{J^8rTX@;(icJqgFCaO4?&PuMz?5Gchn z6F&qR%bhL3jJ6l_ z;(Ra^@79nJ&B67uqrg9f&l$5EcV=ByEj8PGS$k+dhP-{iP

f(LUYB;~_IS9e}`! zZ`r{+GH=AS8xpM&Y8zek-c-Ae*ZG@Nh}-yZ8a#j`V+2+8UHWGZP9d^E3Cb$T*o#8- zQa8T7MzssM8Ox6Q-_||#GL_dJGbZYw8pX#i+o}1a8 z+i+hTg2jbnsUbEd=G~jDcSo*P;pM9H!b>FVLMgA5`54WF4p6}YQ=oypGN;Bt5y282 zCi%|%R^skb7f+0>B}hTaE0Obz#rYuFGv1(39|zId`SQV{4W)MQFCur<%P3h+x5V!V z(Tl@oX@^_&)P=6adgFDxGuYPsa*>3XpRv>1g6<_$*qVWDKDfe_l6UiJK<%s9PW=}O zb|A~={Y=g$MPrVYy>t`)`Y>b;$M14zh}ycGUThVu`e*%_qtFWQ{>Xk9|KcIRyQP@X z27$hm)X6;*aqs^S999e#Kz7N74DmC`*;1OKE_>Rs<+Ibzl{BXDbQwFY+ts7r=MAjIJ)b>wbu>BQ;$GN(}4F$Yd3Ur~8C*S-|eb!83) zNNHit5IP!*oV`&7b20@%N0v<^{G8SMJMBnsy@c+&>CF0Eh$2c6z~d`%<{am-cU+r% z@q*#OpU|P*Ao@p2P2Ck}3M8#fP<}d}Q9yeuX$7lr=S!1Ds>FAGrn>N1I(a74r+qr!9>BKx410y=>v*6Ob`$(;BcEZqvke6f zb*Lc61l$n_6`v4%cS_H-7sQzBp?9ljC`fXwt0wRue6)=Mx+gu*fanH||@0${CQf4!|-?LPt3vAZXT;Ccm z{P3batA@Y)ks}oDUT9;PCRCUAlk#IRWB$S(+Ms6zI0KeV&&z+KFwiFC(^J#UPUSHY z1Y|g`<6O6fsIeFiWTbH=%10_KZB829r!xRytLWlQ{FGnJ>)PxBO4e~reLsr2A2#i? zOV@Z7Leq*2HpPZxgf`7rqPv@DN({V1c6KvE7d(MwQe+BIzC`9*m{u5eDa;vH)6m{x z+txXf2jk`nz+IRKeNV+#>=b-i-e>9o%=_XmM~!IY*{#>DX4XIqS6FRS!Pa40v0H0) zd`Fxo>$l|ut_!||d@5wp(@jl$XpQ}fj`lLhjrNvT6n8c@N&?S4%bA?S*HY)phGHLN z40Q&ViuKY7Pd*y)B6O#7RewPNl~vwNC+!N%m1|-5dpATsS{4BP6QZN) zO(xxZgC3v`f=g6i$DZ9-XhVo68ipQpeU`NVJ8Ss*m~XEvxXuLmxX91w!4*hBRTE7W zpr%2o9L>GD1-{)8pH3IvFmDeYI^lFY{#emxv)}nTOj!`Z|ugxK*PP+Gwq|rf2%(3~pE<{~hwDR<0~T>`@_KTnaCOyHS~O&&K(7 zrw#8)ecr2*zZ@%m@9Kc~{b7(EycsG;K*>VNo!CEECq8?VFJjAU4I)Z7F?~&RV5Zy2 zS_%kj9z;KRSWs`;VBX;*a{moca>#r9{w$$*EIam(?&*&)SWLrE)x=7EsL#uueEMHv z*))q{)Y--_2!9L2rrrjaIJrkJa&3w1#tugf_B~($Wq>FW3Ig$b6S=c#r|+p+RMULM zX>|f@BH;q;H38%@n{M6I6XO!U^AxVMUx*hJU;l|E7H(Oz7h-q^;*Gl@&&J=?@xYZU z#wE#lJ6f%{8?-Dkr;8~&hlEcFOB0$FffA`KJe%&#{=KR$?sv<604PocxTb|`tb5-B zooG=+rVI0c@Pt?&Fdj!R3a>_5CnT#hdHLK#+ zC|8b&t5Onlil$=5y%N=)=pcor+V3zDf^c}FZraeJBB8T#R_hr8%pNzAZzkT6teo=D zL~Lrx^aLuO@f_Xye^Ngi5eOb5P(~WRu#>s&9E?=b`@r7>&0hc-4XLR`^(l?wOc8?2 zm4*p%TT+Jjk`Xl}?`yr#v4WQS2}_Eo{?}b}Toi>Q&nF1`04YA|4!B09kIUZh&g1=- z-~W}E{6h~aEL@wX+l?qy4{kS*$!MG0FATYLkiZNJ&|L1<9LZA&VbkDD=aso#GE?jA#S_zbg}WwXb@Rn2 z+qP4;dEyJI^N%eMdjpM4S+0uUY_m^A5U+0VMc7bK5t%P)WupD~UiNyxRqPAN-(!;_ z&Z^mXvU2&XZt^Akg#z@f^ZI}fbii-yO^M9kNH^CY0NZwq3fFmthhgM*>%_kNl+nDc zcGKnWk11=lS59!ZAE&k@|GZ$g#MGlhGs)~=Db`SF!<@TCZ_@!OtQ#RlxLMOwsQ7zQ zv5YT5<{-ejieg!vn=2^w#yWGuqER9Kj|Y{lC{P^nJ&WA$5BM?_JhpJEv=&TH`LZ}j z-|BLr6}I!N_W@zvA~{BEqWX!Nc!`g!MOcQ-hmW%F8uhq4YR)MGmBFZ4(7GAS`efQ5 z8+z5z>LP~Q@l4L?A4I<%KMO?FpD9Pxftx>Y54`fp6_`Svl2_b-hC;iYjZ69Ra;W zscst=36J0t-TJsn*BdOFmU)0|96s&>-Af;{Kr=623iU{7NFuQY&Nxu zCuNz^l4N62i9*_Rs05MuQix~1iM^tpFeKkZxC^=1%q&j`|ASf4#Z=caGuPfTPlg$i z%&5v)qbw2gF67KRLZNbHWqM^F?`AzUALp95=FHpNCGq^g8DxO?d|~ps)IoeMNVl5( zhI3)H%WYll&7Aax1OsAl~t46W+Xr{EH}72 zo-w{7bCQ1$o|{?)KIp#mdkOFR@pdxh;!;|F1C%kcCMOWIeg@~F^Yoka=X`o^SSsj_ zlUEwIaN4YQtt;PueN3@*_tnN`)5tm}zUvbQ7bVHifYfyx9vNx#1jftmTEk${fYg4s z*@oR*Dommg6grSCotj|iSoU)Pi=gTPsNWOV$tBN|^Y>RI2NaXad|x&tzF5-Bu~r)W z8Nse)OEF;7Op@RYAZ2~llQSjzXkQmlbH+yohWPoG#Ba@YLQxtpb zq{=&2BIHA1Gbs7)gMw>bDD26kfVCIe_QSk#UKc7yMf5G0s}CNkR%4R+&7B<#``1gm5^uJz zv_nE3sPL^?J;Jxuz)wQZso2*N!pSU=cg4|U9YyIO+fm}&`$hjAAqirzlJ~92S?^ek z)2St*#guJF7s<>A2fw>H*g&kR6L~=YIbOFN%A}Nm_|+-Zj>N5HfC%~C?H< zmfvUgv)&sm8i%YLxJI^U`W&^fndABxlwD+bU&Xhq!31|jm5;MT_$R833op-1#u7Qh zLQ3jZzj@eN8p|GeFXqbihdgU(?ac&P!A37_@Svr5R(#opVGeHsgRrA#J|w(Zl(8i> z$fN;lR%Gc*FG$czBrqpA%(kZ7tqIb!0k>-S?VTJ(0B^?9xn4KM)CAjjFYp}P-h5<#)MAkOlGL16KfbnA!tTC2W|kApvsgr~OnLm&Cn(AEa8iJTH0uYY ztTk+O-T&U^*unBWpKi#5DIP|uOGJ>pfHk+O2la40_ke$DTujIq=X{~0?mqmv^a3GW zQiy~_S1SlIof}?Qb#W)mjX$qoYS>-8_!xg?j>W?*p|Q#>=ef2|i8_d8Uf+3(aZt#1 zbTWFWQntE#Yn2yaJX9-Eu`@avLCj$|L#Wt-30j28&{fru@hK!!AIUU1uAD?lCQf24 zT05x|+E=kQCt1qRsy*m&r!Ue>#Rm)CQpWG8Ie`B7`R}%i?kiPa9oPpzH}QK1^*=k& z>H#cFya0&d58rTe=U^a)lV^fS)8TS3>O@oIB{p^!)<3fg!I{0-bv-AcFxzyyDHM1; zJgAGq`_L(ALT#TYGgzkgv7yA$ubM8mrsxpcbH?an1xG0Do$ZO3F-s75l_E&k&Z-9V z>3G(OLAtcV6WcvZPZ0EXukCRNp`w@SwslbVe{9-*Y>-#e+V@`e!&?ww=B$2DSa=e_ zPIraPQqpklFnm(^2qBU_+Zs(K-s18ZWnriqqGgL4^z=r5I;|(0HNQQ&n zFis|zL(S_$dMj))PXyL<#3yA%vJn_R+bkf|&|RUs%YtxWxRLjqgZ>JgrI&Zkf`fm= zK{L({0mNeW5xzh8xc!7S0c_lS37m<7R|9$S#H%-I(|>C*>TksbOFOgGfuUYt_Q-4a zIgK<7d*BnqFP804gU)w;LBJ>c`u_T%Pbug_^PgrqKgp{W&?Tptq|(-*3ay;=6((W7 zTjvO$29+q|cpR{5GZt9eZlx5B{B8NvfY-i?HtjT8F0%nYObhP97gEZVxlA>JMYIlP3V zU6Gp1m{4=8L{POxlRBY{jnHu`>3+N17}~8Bq4S<}#)EyVmp&-}BAn`=J4gldyGs;y zqF>?nLRmL=@yj9zQQ3ngo}uQ>$43ZXcFeEa`KBL4quK<~#%zL?tDn;kYM^SiTlv1% z>!u^0#~({0J^2}E0qfT2Tc>lzXBkt+8|qBP8fr`$;GUpQvek6yJRu-jD6YwZda-a> zZwS+8jr(*Etu}UdO;89oJ@<@Ag}azrEz*;<7$?svZg5+WeqM;zf9Tb;HGw*r{Q9zS z*QMo&ZyrHpcx6Ems#L)JXhC6AxQbOw?Zf73$XWyK6MSnLi*I(4*hyJg3kOv~ifl$C zr#pMZwjqCp4S=!A+Y+0Ajkv-2=&#f74LT^o0eEv1ef8phRiBAH68XSeGlF8VwNolD zponR$gq$|1@(Yvn)~c5-Tn3F%O}b{J#uXSo7-Ut*{5R`$Zvtquo>IIcuuowcHL`7s zyE-+P6BmTUfIV=%ve=(?m25SM@i1hMIfVAbf9&sdDu8 zXM?7VBo=L+ip>XAY#<4^j5>0t z8t0yY?W%jWY=3+RY>_aU3rIHU(#3UYlZLCV7=gpgMnCU|-DM)6Yzz#Vcio$eD#kEv zvr&PyT7pZUt=h<(LNs|lxSTozm~d3fo5a0$h%!v~=DK(2OSAH5DQ0=;#FFoUs9Gll zFtv>O@IR*yc%pq4-+ugX+hRzp+5RWon={xIoHr)yE^)cBbVh0aV*h(_ARI894YY*( z{*CP+bl}gvkOR)86TJET>VBfzhYN^CD)3;6Y5KW)59YsL626zY+ZpPrjq2FHivM2R zK<|2ri*D@xN8eBFbP=MvtVP0@itm0p+vA#mlNQsMsNvLa%-& Date: Mon, 28 Jan 2019 14:39:26 -0600 Subject: [PATCH 65/69] Doc: all: update revision history for 2.0.1 release --- .../en-US/Pacemaker_Administration.ent | 2 +- .../en-US/Revision_History.xml | 26 +++++++++++----------- .../en-US/Pacemaker_Explained.ent | 2 +- doc/Pacemaker_Explained/en-US/Revision_History.xml | 2 +- doc/Pacemaker_Remote/en-US/Book_Info.xml | 2 +- doc/Pacemaker_Remote/en-US/Revision_History.xml | 11 +++++++++ 6 files changed, 28 insertions(+), 17 deletions(-) diff --git a/doc/Pacemaker_Administration/en-US/Pacemaker_Administration.ent b/doc/Pacemaker_Administration/en-US/Pacemaker_Administration.ent index d9e752b..f67f950 100644 --- a/doc/Pacemaker_Administration/en-US/Pacemaker_Administration.ent +++ b/doc/Pacemaker_Administration/en-US/Pacemaker_Administration.ent @@ -1,4 +1,4 @@ - + diff --git a/doc/Pacemaker_Administration/en-US/Revision_History.xml b/doc/Pacemaker_Administration/en-US/Revision_History.xml index eaaacd6..dae578f 100644 --- a/doc/Pacemaker_Administration/en-US/Revision_History.xml +++ b/doc/Pacemaker_Administration/en-US/Revision_History.xml @@ -10,33 +10,33 @@ - 1-1 - Tue Dec 4 2018 + 1-0 + Tue Jan 23 2018 KenGaillot kgaillot@redhat.com - - JanPokorný - jpokorny@redhat.com - - Add "Troubleshooting" chapter, minor - clarifications and reformatting + Move administration-oriented information from + Pacemaker Explained into its own + book - 1-0 - Tue Jan 23 2018 + 1-1 + Mon Jan 28 2019 KenGaillot kgaillot@redhat.com + + JanPokorný + jpokorny@redhat.com + - Move administration-oriented information from - Pacemaker Explained into its own - book + Add "Troubleshooting" chapter, minor + clarifications and reformatting diff --git a/doc/Pacemaker_Explained/en-US/Pacemaker_Explained.ent b/doc/Pacemaker_Explained/en-US/Pacemaker_Explained.ent index a767f5f..a79fcf3 100644 --- a/doc/Pacemaker_Explained/en-US/Pacemaker_Explained.ent +++ b/doc/Pacemaker_Explained/en-US/Pacemaker_Explained.ent @@ -1,4 +1,4 @@ - + diff --git a/doc/Pacemaker_Explained/en-US/Revision_History.xml b/doc/Pacemaker_Explained/en-US/Revision_History.xml index 3da97ac..71c5a5b 100644 --- a/doc/Pacemaker_Explained/en-US/Revision_History.xml +++ b/doc/Pacemaker_Explained/en-US/Revision_History.xml @@ -141,7 +141,7 @@ 12-0 - Fri Dec 7 2018 + Mon Jan 28 2019 KenGaillotkgaillot@redhat.com ReidWahlnwahl@redhat.com JanPokornýjpokorny@redhat.com diff --git a/doc/Pacemaker_Remote/en-US/Book_Info.xml b/doc/Pacemaker_Remote/en-US/Book_Info.xml index bf11f23..8132402 100644 --- a/doc/Pacemaker_Remote/en-US/Book_Info.xml +++ b/doc/Pacemaker_Remote/en-US/Book_Info.xml @@ -13,7 +13,7 @@ simple textual changes (corrections, translations, etc.). --> 7 - 1 + 2 The document exists as both a reference and deployment guide for the Pacemaker Remote service. diff --git a/doc/Pacemaker_Remote/en-US/Revision_History.xml b/doc/Pacemaker_Remote/en-US/Revision_History.xml index b636049..9950f4c 100644 --- a/doc/Pacemaker_Remote/en-US/Revision_History.xml +++ b/doc/Pacemaker_Remote/en-US/Revision_History.xml @@ -56,6 +56,17 @@ KenGaillotkgaillot@redhat.com Update banner for Pacemaker 2.0 and content for CentOS 7.4 with Pacemaker 1.1.16 + + + 7-2 + Mon Jan 29 2019 + + JanPokorný + jpokorny@redhat.com + + Minor reformatting + + -- 1.8.3.1 From 9958bc9fde3dc4e9447a4e72c4ccbd9699ff4b58 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Mon, 28 Jan 2019 12:04:41 -0600 Subject: [PATCH 66/69] Fix: controller: clear election dampening when DC is lost same issue as 5c80a964 but for controller --- daemons/controld/controld_election.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/daemons/controld/controld_election.c b/daemons/controld/controld_election.c index 9fbf1e1..5d6858c 100644 --- a/daemons/controld/controld_election.c +++ b/daemons/controld/controld_election.c @@ -1,5 +1,5 @@ /* - * Copyright 2004-2018 Andrew Beekhof + * Copyright 2004-2019 Andrew Beekhof * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. @@ -41,6 +41,13 @@ void controld_remove_voter(const char *uname) { election_remove(fsa_election, uname); + + if (safe_str_eq(uname, fsa_our_dc)) { + /* Clear any election dampening in effect. Otherwise, if the lost DC had + * just won, an immediate new election could fizzle out with no new DC. + */ + election_clear_dampening(fsa_election); + } } void -- 1.8.3.1 From 0904789e74c59a33479b550befc82118a600fe36 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 29 Jan 2019 11:56:00 -0600 Subject: [PATCH 67/69] Fix: controller: really avoid closing attrd IPC for temporary failures Since 35d69f2f (1.0.3), the controller attempts to connect to the attribute manager and update an attribute multiple times before giving up. b7c0e7f0 (1.1.9) attempted to avoid closing the IPC connection if the error was "try again". However, rather than testing the errno from crm_ipc_connect(), it tested the return value from attrd_update_delegate(), which would never return that. Refactor so we can check the right function's result. Also tweak log messages. --- daemons/controld/controld_attrd.c | 41 ++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/daemons/controld/controld_attrd.c b/daemons/controld/controld_attrd.c index 0ce8be8..0653804 100644 --- a/daemons/controld/controld_attrd.c +++ b/daemons/controld/controld_attrd.c @@ -81,31 +81,44 @@ update_attrd_helper(const char *host, const char *name, const char *value, } for (int attempt = 1; attempt <= 4; ++attempt) { + rc = pcmk_ok; + + // If we're not already connected, try to connect if (crm_ipc_connected(attrd_ipc) == FALSE) { - crm_ipc_close(attrd_ipc); - crm_info("Connecting to attribute manager (attempt %d of 4)", - attempt); + if (attempt == 1) { + // Start with a clean slate + crm_ipc_close(attrd_ipc); + } if (crm_ipc_connect(attrd_ipc) == FALSE) { - crm_perror(LOG_INFO, "Connection to attribute manager failed"); + rc = errno; } + crm_debug("Attribute manager connection attempt %d of 4: %s (%d)", + attempt, pcmk_strerror(rc), rc); } - if (command) { - rc = attrd_update_delegate(attrd_ipc, command, host, name, value, + if (rc == pcmk_ok) { + rc = command? + attrd_update_delegate(attrd_ipc, command, host, name, value, XML_CIB_TAG_STATUS, NULL, NULL, - user_name, attrd_opts); - } else { - /* (ab)using name/value as resource/operation */ - rc = attrd_clear_delegate(attrd_ipc, host, name, value, - interval_spec, user_name, attrd_opts); + user_name, attrd_opts) + + /* No command means clear fail count (name/value is really + * resource/operation) + */ + : attrd_clear_delegate(attrd_ipc, host, name, value, + interval_spec, user_name, attrd_opts); + crm_debug("Attribute manager request attempt %d of 4: %s (%d)", + attempt, pcmk_strerror(rc), rc); } if (rc == pcmk_ok) { + // Success, we're done break; - } else if (rc != -EAGAIN && rc != -EALREADY) { - crm_info("Disconnecting from attribute manager: %s (%d)", - pcmk_strerror(rc), rc); + } else if ((rc != EAGAIN) && (rc != EALREADY)) { + /* EAGAIN or EALREADY indicates a temporary block, so just try + * again. Otherwise, close the connection for a clean slate. + */ crm_ipc_close(attrd_ipc); } -- 1.8.3.1 From 77f6bf4a5762ae0b2192335a74894da7306c9674 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 29 Jan 2019 14:59:29 -0600 Subject: [PATCH 68/69] Test: CTS: really don't require nodes to be specified if listing tests 9c993a7c was incomplete --- cts/CIB.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cts/CIB.py b/cts/CIB.py index 2e606b2..3b3c2ac 100644 --- a/cts/CIB.py +++ b/cts/CIB.py @@ -441,7 +441,8 @@ class ConfigFactory(object): self.register("pacemaker20", CIB20, CM, self) self.register("pacemaker30", CIB30, CM, self) # self.register("hae", HASI, CM, self) - self.target = self.CM.Env["nodes"][0] + if self.CM.Env["ListTests"] == 0: + self.target = self.CM.Env["nodes"][0] self.tmpfile = None def log(self, args): -- 1.8.3.1 From 8de16b8a745e5b9d54c8441730d70c479ad84550 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Mon, 28 Jan 2019 14:53:58 -0600 Subject: [PATCH 69/69] Doc: ChangeLog: update for 2.0.1-rc4 release --- ChangeLog | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ChangeLog b/ChangeLog index ea65fbe..c51c439 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,16 @@ +* Tue Jan 29 2019 Ken Gaillot Pacemaker-2.0.1-rc4 +- Changesets: 42 + 15 files changed, 216 insertions(+), 137 deletions(-) + +- Changes since Pacemaker-2.0.1-rc3 + + attrd: clear election dampening when the writer leaves + + controller: clear election dampening when DC is lost + + scheduler: don't order non-DC shutdowns before DC fencing + + libcrmservice: cancel DBus call when cancelling systemd/upstart actions + + tools: remove duplicate fence history state in crm_mon XML output + + build: offer configure option to disable tests broken with glib 2.59.0 + + build: minor logging fixes to allow compatibility with GCC 9 -Werror + * Thu Jan 10 2019 Ken Gaillot Pacemaker-2.0.1-rc3 - Changesets: 27 - Diff: 20 files changed, 375 insertions(+), 195 deletions(-) -- 1.8.3.1