From af918f795bf00f1a25775c47552c5d56138ffbd7 Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Mar 31 2020 09:37:46 +0000 Subject: import pacemaker-1.1.21-4.el7 --- diff --git a/.gitignore b/.gitignore index 86d121d..bcfcb86 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ SOURCES/nagios-agents-metadata-105ab8a.tar.gz -SOURCES/pacemaker-3c4c782.tar.gz +SOURCES/pacemaker-f14e36f.tar.gz diff --git a/.pacemaker.metadata b/.pacemaker.metadata index 4a34c74..ec07f17 100644 --- a/.pacemaker.metadata +++ b/.pacemaker.metadata @@ -1,2 +1,2 @@ ea6c0a27fd0ae8ce02f84a11f08a0d79377041c3 SOURCES/nagios-agents-metadata-105ab8a.tar.gz -1a5dd220a120eba50048913807dd40c4fcde967c SOURCES/pacemaker-3c4c782.tar.gz +1affd72b4a9a8190e5e87761b16c32935120e65c SOURCES/pacemaker-f14e36f.tar.gz diff --git a/SOURCES/001-constraint-fix.patch b/SOURCES/001-constraint-fix.patch deleted file mode 100644 index 0461b3f..0000000 --- a/SOURCES/001-constraint-fix.patch +++ /dev/null @@ -1,3135 +0,0 @@ -From 31ff280494d415b0f4f599b4a3551065daf5f7b1 Mon Sep 17 00:00:00 2001 -From: "Gao,Yan" -Date: Fri, 22 Feb 2019 11:49:30 +0100 -Subject: [PATCH 1/3] Fix: scheduler: cl#5301 - respect order constraints when - relevant resources are being probed - -This fixes violations of order constraints introduced by faf44d811 and -8f76b7821. - -Given the typical scenario with an order constraint "A.stop -> B.stop", -if A has been cleaned up and is being reprobed while B is stopping, -B.stop should wait for A.probe to complete. Since the pseudo action -"probe_complete" has been dropped by 8f76b7821, the solution here is to -optionally order "A.probe -> B.stop" as the possible alternative of -"A.stop -> B.stop". - -This also addresses the cases where actions of B are other actions -than "stop", including order constraints like "A.stop -> B.start" -implied by anti-colocations. ---- - pengine/allocate.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++++++++- - 1 file changed, 164 insertions(+), 1 deletion(-) - -diff --git a/pengine/allocate.c b/pengine/allocate.c -index 9177fb1..578db2f 100644 ---- a/pengine/allocate.c -+++ b/pengine/allocate.c -@@ -2267,8 +2267,164 @@ apply_remote_node_ordering(pe_working_set_t *data_set) - } - } - -+static gboolean -+order_first_probe_unneeded(pe_action_t * probe, pe_action_t * rh_action) -+{ -+ /* No need to probe the resource on the node that is being -+ * unfenced. Otherwise it might introduce transition loop -+ * since probe will be performed after the node is -+ * unfenced. -+ */ -+ if (safe_str_eq(rh_action->task, CRM_OP_FENCE) -+ && probe->node && rh_action->node -+ && probe->node->details == rh_action->node->details) { -+ const char *op = g_hash_table_lookup(rh_action->meta, "stonith_action"); -+ -+ if (safe_str_eq(op, "on")) { -+ return TRUE; -+ } -+ } -+ -+ // Shutdown waits for probe to complete only if it's on the same node -+ if ((safe_str_eq(rh_action->task, CRM_OP_SHUTDOWN)) -+ && probe->node && rh_action->node -+ && probe->node->details != rh_action->node->details) { -+ return TRUE; -+ } -+ return FALSE; -+} -+ -+ -+static void -+order_first_probes(pe_working_set_t * data_set) -+{ -+ GListPtr gIter = NULL; -+ -+ for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) { -+ pe__ordering_t *order = gIter->data; -+ enum pe_ordering order_type = pe_order_optional; -+ -+ pe_resource_t *lh_rsc = order->lh_rsc; -+ pe_resource_t *rh_rsc = order->rh_rsc; -+ pe_action_t *lh_action = order->lh_action; -+ pe_action_t *rh_action = order->rh_action; -+ const char *lh_action_task = order->lh_action_task; -+ const char *rh_action_task = order->rh_action_task; -+ -+ char *key = NULL; -+ GListPtr probes = NULL; -+ GListPtr rh_actions = NULL; -+ -+ GListPtr pIter = NULL; -+ -+ if (lh_rsc == NULL) { -+ continue; -+ -+ } else if (rh_rsc && lh_rsc == rh_rsc) { -+ continue; -+ } -+ -+ if (lh_action == NULL && lh_action_task == NULL) { -+ continue; -+ } -+ -+ if (rh_action == NULL && rh_action_task == NULL) { -+ continue; -+ } -+ -+ /* Technically probe is expected to return "not running", which could be -+ * the alternative of stop action if the status of the resource is -+ * unknown yet. -+ */ -+ if (lh_action && safe_str_neq(lh_action->task, RSC_STOP)) { -+ continue; -+ -+ } else if (lh_action == NULL -+ && lh_action_task -+ && crm_ends_with(lh_action_task, "_" RSC_STOP "_0") == FALSE) { -+ continue; -+ } -+ -+ /* Do not probe the resource inside of a stopping container. Otherwise -+ * it might introduce transition loop since probe will be performed -+ * after the container starts again. -+ */ -+ if (rh_rsc && lh_rsc->container == rh_rsc) { -+ if (rh_action && safe_str_eq(rh_action->task, RSC_STOP)) { -+ continue; -+ -+ } else if (rh_action == NULL && rh_action_task -+ && crm_ends_with(rh_action_task,"_" RSC_STOP "_0")) { -+ continue; -+ } -+ } -+ -+ if (order->type == pe_order_none) { -+ continue; -+ } -+ -+ // Preserve the order options for future filtering -+ if (is_set(order->type, pe_order_apply_first_non_migratable)) { -+ set_bit(order_type, pe_order_apply_first_non_migratable); -+ } -+ -+ if (is_set(order->type, pe_order_same_node)) { -+ set_bit(order_type, pe_order_same_node); -+ } -+ -+ // Keep the order types for future filtering -+ if (order->type == pe_order_anti_colocation -+ || order->type == pe_order_load) { -+ order_type = order->type; -+ } -+ -+ key = generate_op_key(lh_rsc->id, RSC_STATUS, 0); -+ probes = find_actions(lh_rsc->actions, key, NULL); -+ free(key); -+ -+ if (probes == NULL) { -+ continue; -+ } -+ -+ if (rh_action) { -+ rh_actions = g_list_prepend(rh_actions, rh_action); -+ -+ } else if (rh_rsc && rh_action_task) { -+ rh_actions = find_actions(rh_rsc->actions, rh_action_task, NULL); -+ } -+ -+ if (rh_actions == NULL) { -+ g_list_free(probes); -+ continue; -+ } -+ -+ crm_trace("Processing for LH probe based on ordering constraint %s -> %s" -+ " (id=%d, type=%.6x)", -+ lh_action ? lh_action->uuid : lh_action_task, -+ rh_action ? rh_action->uuid : rh_action_task, -+ order->id, order->type); -+ -+ for (pIter = probes; pIter != NULL; pIter = pIter->next) { -+ pe_action_t *probe = (pe_action_t *) pIter->data; -+ GListPtr rIter = NULL; -+ -+ for (rIter = rh_actions; rIter != NULL; rIter = rIter->next) { -+ pe_action_t *rh_action_iter = (pe_action_t *) rIter->data; -+ -+ if (order_first_probe_unneeded(probe, rh_action_iter)) { -+ continue; -+ } -+ order_actions(probe, rh_action_iter, order_type); -+ } -+ } -+ -+ g_list_free(rh_actions); -+ g_list_free(probes); -+ } -+} -+ - static void --order_probes(pe_working_set_t * data_set) -+order_then_probes(pe_working_set_t * data_set) - { - #if 0 - GListPtr gIter = NULL; -@@ -2389,6 +2545,13 @@ order_probes(pe_working_set_t * data_set) - #endif - } - -+static void -+order_probes(pe_working_set_t * data_set) -+{ -+ order_first_probes(data_set); -+ order_then_probes(data_set); -+} -+ - gboolean - stage7(pe_working_set_t * data_set) - { --- -1.8.3.1 - - -From 62c73cf7c115d79aa72b8533ee8a820c90d2618b Mon Sep 17 00:00:00 2001 -From: "Gao,Yan" -Date: Fri, 1 Mar 2019 16:34:23 +0100 -Subject: [PATCH 2/3] Test: scheduler: cl#5301 - respect order constraints when - relevant resources are being probed (update tests) - ---- - .../11-a-then-bm-b-move-a-clone-starting.dot | 1 + - .../11-a-then-bm-b-move-a-clone-starting.exp | 3 ++ - pengine/test10/594.dot | 2 + - pengine/test10/594.exp | 6 +++ - pengine/test10/662.dot | 9 ++++ - pengine/test10/662.exp | 27 ++++++++++++ - pengine/test10/797.dot | 4 ++ - pengine/test10/797.exp | 12 ++++++ - pengine/test10/829.dot | 6 +++ - pengine/test10/829.exp | 18 ++++++++ - pengine/test10/bug-cl-5247.dot | 3 ++ - pengine/test10/bug-cl-5247.exp | 9 ++++ - pengine/test10/bug-lf-2435.dot | 1 + - pengine/test10/bug-lf-2435.exp | 3 ++ - pengine/test10/bug-n-387749.dot | 3 ++ - pengine/test10/bug-n-387749.exp | 9 ++++ - pengine/test10/bug-rh-1097457.dot | 8 ++++ - pengine/test10/bug-rh-1097457.exp | 24 +++++++++++ - pengine/test10/bundle-replicas-change.dot | 2 + - pengine/test10/bundle-replicas-change.exp | 6 +++ - pengine/test10/clone-no-shuffle.dot | 2 + - pengine/test10/clone-no-shuffle.exp | 6 +++ - pengine/test10/group5.dot | 6 +++ - pengine/test10/group5.exp | 18 ++++++++ - pengine/test10/group6.dot | 10 +++++ - pengine/test10/group6.exp | 30 ++++++++++++++ - pengine/test10/group9.dot | 9 ++++ - pengine/test10/group9.exp | 27 ++++++++++++ - pengine/test10/inc2.dot | 5 +++ - pengine/test10/inc2.exp | 15 +++++++ - pengine/test10/inc3.dot | 10 +++++ - pengine/test10/inc3.exp | 30 ++++++++++++++ - pengine/test10/inc4.dot | 13 ++++++ - pengine/test10/inc4.exp | 39 ++++++++++++++++++ - pengine/test10/inc5.dot | 16 ++++++++ - pengine/test10/inc5.exp | 48 ++++++++++++++++++++++ - pengine/test10/master-7.dot | 13 ++++++ - pengine/test10/master-7.exp | 39 ++++++++++++++++++ - pengine/test10/master-8.dot | 14 +++++++ - pengine/test10/master-8.exp | 42 +++++++++++++++++++ - pengine/test10/master-9.dot | 5 +++ - pengine/test10/master-9.exp | 15 +++++++ - pengine/test10/notify-0.dot | 1 + - pengine/test10/notify-0.exp | 3 ++ - pengine/test10/notify-1.dot | 1 + - pengine/test10/notify-1.exp | 3 ++ - pengine/test10/notify-2.dot | 1 + - pengine/test10/notify-2.exp | 3 ++ - pengine/test10/notify-3.dot | 5 +++ - pengine/test10/notify-3.exp | 15 +++++++ - pengine/test10/novell-252693-3.dot | 2 + - pengine/test10/novell-252693-3.exp | 6 +++ - pengine/test10/order3.dot | 3 ++ - pengine/test10/order3.exp | 9 ++++ - pengine/test10/rec-node-11.dot | 3 ++ - pengine/test10/rec-node-11.exp | 9 ++++ - pengine/test10/reload-becomes-restart.dot | 1 + - pengine/test10/reload-becomes-restart.exp | 3 ++ - pengine/test10/remote-connection-unrecoverable.dot | 2 + - pengine/test10/remote-connection-unrecoverable.exp | 6 +++ - pengine/test10/rsc_dep1.dot | 2 + - pengine/test10/rsc_dep1.exp | 6 +++ - pengine/test10/rsc_dep5.dot | 4 ++ - pengine/test10/rsc_dep5.exp | 12 ++++++ - pengine/test10/unfence-definition.dot | 4 ++ - pengine/test10/unfence-definition.exp | 12 ++++++ - pengine/test10/unfence-parameters.dot | 4 ++ - pengine/test10/unfence-parameters.exp | 12 ++++++ - pengine/test10/unrunnable-1.dot | 3 ++ - pengine/test10/unrunnable-1.exp | 9 ++++ - pengine/test10/whitebox-imply-stop-on-fence.dot | 6 +++ - pengine/test10/whitebox-imply-stop-on-fence.exp | 18 ++++++++ - pengine/test10/whitebox-migrate1.dot | 1 + - pengine/test10/whitebox-migrate1.exp | 6 ++- - pengine/test10/whitebox-migrate1.summary | 6 +-- - pengine/test10/whitebox-move.dot | 1 + - pengine/test10/whitebox-move.exp | 3 ++ - pengine/test10/whitebox-ms-ordering.dot | 3 ++ - pengine/test10/whitebox-ms-ordering.exp | 9 ++++ - pengine/test10/whitebox-orphaned.dot | 1 + - pengine/test10/whitebox-orphaned.exp | 3 ++ - pengine/test10/whitebox-stop.dot | 1 + - pengine/test10/whitebox-stop.exp | 3 ++ - 83 files changed, 769 insertions(+), 4 deletions(-) - -diff --git a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.dot b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.dot -index 2b45d58..4a89db6 100644 ---- a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.dot -+++ b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.dot -@@ -11,6 +11,7 @@ - "myclone-clone_stopped_0" -> "myclone-clone_start_0" [ style = bold] - "myclone-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] - "myclone_monitor_0 f20node2" -> "myclone-clone_start_0" [ style = bold] -+"myclone_monitor_0 f20node2" -> "myclone-clone_stopped_0" [ style = bold] - "myclone_monitor_0 f20node2" [ style=bold color="green" fontcolor="black"] - "myclone_start_0 f20node2" -> "myclone-clone_running_0" [ style = bold] - "myclone_start_0 f20node2" [ style=bold color="green" fontcolor="black"] -diff --git a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp -index b391b42..4eeb086 100644 ---- a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp -+++ b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp -@@ -45,6 +45,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/594.dot b/pengine/test10/594.dot -index 4f3ea64..8a24440 100644 ---- a/pengine/test10/594.dot -+++ b/pengine/test10/594.dot -@@ -12,10 +12,12 @@ digraph "g" { - "DoFencing_stop_0" -> "child_DoFencing:2_stop_0 hadev1" [ style = bold] - "DoFencing_stop_0" [ style=bold color="green" fontcolor="orange" ] - "DoFencing_stopped_0" [ style=bold color="green" fontcolor="orange" ] -+"child_DoFencing:0_monitor_0 hadev1" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:0_monitor_0 hadev1" [ style=bold color="green" fontcolor="black" ] - "child_DoFencing:0_stop_0 hadev2" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:0_stop_0 hadev2" -> "do_shutdown hadev2" [ style = bold] - "child_DoFencing:0_stop_0 hadev2" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:2_monitor_0 hadev2" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_monitor_0 hadev2" [ style=bold color="green" fontcolor="black" ] - "child_DoFencing:2_stop_0 hadev1" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_stop_0 hadev1" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/594.exp b/pengine/test10/594.exp -index 0bbf746..c0025f0 100644 ---- a/pengine/test10/594.exp -+++ b/pengine/test10/594.exp -@@ -160,6 +160,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/662.dot b/pengine/test10/662.dot -index 9848322..173103c 100644 ---- a/pengine/test10/662.dot -+++ b/pengine/test10/662.dot -@@ -6,17 +6,26 @@ - "DoFencing_stop_0" -> "child_DoFencing:0_stop_0 c001n02" [ style = bold] - "DoFencing_stop_0" [ style=bold color="green" fontcolor="orange" ] - "DoFencing_stopped_0" [ style=bold color="green" fontcolor="orange" ] -+"child_DoFencing:0_monitor_0 c001n04" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:0_monitor_0 c001n04" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:0_monitor_0 c001n09" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:0_monitor_0 c001n09" [ style=bold color="green" fontcolor="black" ] - "child_DoFencing:0_stop_0 c001n02" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:0_stop_0 c001n02" -> "do_shutdown c001n02" [ style = bold] - "child_DoFencing:0_stop_0 c001n02" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:1_monitor_0 c001n02" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:1_monitor_0 c001n02" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:1_monitor_0 c001n04" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:1_monitor_0 c001n04" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:2_monitor_0 c001n03" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:2_monitor_0 c001n09" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_monitor_0 c001n09" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:3_monitor_0 c001n02" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 c001n02" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:3_monitor_0 c001n03" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:3_monitor_0 c001n04" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 c001n04" [ style=bold color="green" fontcolor="black" ] - "do_shutdown c001n02" [ style=bold color="green" fontcolor="black" ] - "rsc_c001n02_monitor_0 c001n03" -> "rsc_c001n02_start_0 c001n03" [ style = bold] -diff --git a/pengine/test10/662.exp b/pengine/test10/662.exp -index 3751565..f0a0560 100644 ---- a/pengine/test10/662.exp -+++ b/pengine/test10/662.exp -@@ -271,6 +271,33 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/797.dot b/pengine/test10/797.dot -index 9ef868a..ff049d9 100644 ---- a/pengine/test10/797.dot -+++ b/pengine/test10/797.dot -@@ -36,12 +36,16 @@ - "child_DoFencing:1_stop_0 c001n02" -> "do_shutdown c001n02" [ style = bold] - "child_DoFencing:1_stop_0 c001n02" [ style=bold color="green" fontcolor="black" ] - "child_DoFencing:2_monitor_0 c001n01" -> "DoFencing_start_0" [ style = bold] -+"child_DoFencing:2_monitor_0 c001n01" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_monitor_0 c001n01" [ style=bold color="green" fontcolor="black" ] - "child_DoFencing:3_monitor_0 c001n01" -> "DoFencing_start_0" [ style = bold] -+"child_DoFencing:3_monitor_0 c001n01" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 c001n01" [ style=bold color="green" fontcolor="black" ] - "child_DoFencing:3_monitor_0 c001n02" -> "DoFencing_start_0" [ style = bold] -+"child_DoFencing:3_monitor_0 c001n02" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 c001n02" [ style=bold color="green" fontcolor="black" ] - "child_DoFencing:3_monitor_0 c001n03" -> "DoFencing_start_0" [ style = bold] -+"child_DoFencing:3_monitor_0 c001n03" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "do_shutdown c001n02" [ style=bold color="green" fontcolor="black" ] - "rsc_c001n01_monitor_0 c001n03" -> "rsc_c001n01_start_0 c001n01" [ style = dashed] -diff --git a/pengine/test10/797.exp b/pengine/test10/797.exp -index 62a01ae..62d82a9 100644 ---- a/pengine/test10/797.exp -+++ b/pengine/test10/797.exp -@@ -253,6 +253,18 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/829.dot b/pengine/test10/829.dot -index 16ba24c..f01f1ad 100644 ---- a/pengine/test10/829.dot -+++ b/pengine/test10/829.dot -@@ -5,13 +5,19 @@ digraph "g" { - "DoFencing_stop_0" -> "child_DoFencing:0_stop_0 c001n02" [ style = bold] - "DoFencing_stop_0" [ style=bold color="green" fontcolor="orange" ] - "DoFencing_stopped_0" [ style=bold color="green" fontcolor="orange" ] -+"child_DoFencing:0_monitor_0 c001n01" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:0_monitor_0 c001n01" [ style=bold color="green" fontcolor="black" ] - "child_DoFencing:0_stop_0 c001n02" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:0_stop_0 c001n02" [ style=bold color="green" fontcolor="orange" ] -+"child_DoFencing:1_monitor_0 c001n01" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:1_monitor_0 c001n01" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:2_monitor_0 c001n03" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:2_monitor_0 c001n08" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_monitor_0 c001n08" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:3_monitor_0 c001n01" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 c001n01" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:3_monitor_0 c001n03" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "rsc_c001n01_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "rsc_c001n01_monitor_0 c001n08" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/829.exp b/pengine/test10/829.exp -index 74afbcc..247486a 100644 ---- a/pengine/test10/829.exp -+++ b/pengine/test10/829.exp -@@ -225,6 +225,24 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/bug-cl-5247.dot b/pengine/test10/bug-cl-5247.dot -index a978467..c125dac 100644 ---- a/pengine/test10/bug-cl-5247.dot -+++ b/pengine/test10/bug-cl-5247.dot -@@ -115,6 +115,7 @@ digraph "g" { - "stonith 'off' pgsr02" -> "vip-rep_start_0 pgsr01" [ style = bold] - "stonith 'off' pgsr02" -> "vip-rep_stop_0 pgsr02" [ style = bold] - "stonith 'off' pgsr02" [ style=bold color="green" fontcolor="orange"] -+"vip-master_monitor_0 pgsr01" -> "master-group_stopped_0" [ style = bold] - "vip-master_monitor_0 pgsr01" -> "vip-master_start_0 pgsr01" [ style = bold] - "vip-master_monitor_0 pgsr01" [ style=bold color="green" fontcolor="black"] - "vip-master_monitor_10000 pgsr01" [ style=bold color="green" fontcolor="black"] -@@ -125,6 +126,8 @@ digraph "g" { - "vip-master_stop_0 pgsr02" -> "master-group_stopped_0" [ style = bold] - "vip-master_stop_0 pgsr02" -> "vip-master_start_0 pgsr01" [ style = bold] - "vip-master_stop_0 pgsr02" [ style=bold color="green" fontcolor="orange"] -+"vip-rep_monitor_0 pgsr01" -> "master-group_stopped_0" [ style = bold] -+"vip-rep_monitor_0 pgsr01" -> "vip-master_stop_0 pgsr02" [ style = bold] - "vip-rep_monitor_0 pgsr01" -> "vip-rep_start_0 pgsr01" [ style = bold] - "vip-rep_monitor_0 pgsr01" [ style=bold color="green" fontcolor="black"] - "vip-rep_monitor_10000 pgsr01" [ style=bold color="green" fontcolor="black"] -diff --git a/pengine/test10/bug-cl-5247.exp b/pengine/test10/bug-cl-5247.exp -index 14c9d91..d08214d 100644 ---- a/pengine/test10/bug-cl-5247.exp -+++ b/pengine/test10/bug-cl-5247.exp -@@ -210,6 +210,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -@@ -308,6 +314,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/bug-lf-2435.dot b/pengine/test10/bug-lf-2435.dot -index 0407439..76d8f48 100644 ---- a/pengine/test10/bug-lf-2435.dot -+++ b/pengine/test10/bug-lf-2435.dot -@@ -6,5 +6,6 @@ digraph "g" { - "dummy3_stop_0 c21.chepkov.lan" [ style=bold color="green" fontcolor="black" ] - "dummy4_monitor_0 c19.chepkov.lan" [ style=bold color="green" fontcolor="black" ] - "dummy4_monitor_0 c20.chepkov.lan" [ style=bold color="green" fontcolor="black" ] -+"dummy4_monitor_0 c21.chepkov.lan" -> "dummy2_start_0 c21.chepkov.lan" [ style = bold] - "dummy4_monitor_0 c21.chepkov.lan" [ style=bold color="green" fontcolor="black" ] - } -diff --git a/pengine/test10/bug-lf-2435.exp b/pengine/test10/bug-lf-2435.exp -index 387d266..69a4a18 100644 ---- a/pengine/test10/bug-lf-2435.exp -+++ b/pengine/test10/bug-lf-2435.exp -@@ -8,6 +8,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/bug-n-387749.dot b/pengine/test10/bug-n-387749.dot -index 4a978ce..5095351 100644 ---- a/pengine/test10/bug-n-387749.dot -+++ b/pengine/test10/bug-n-387749.dot -@@ -41,6 +41,7 @@ digraph "g" { - "group_nfs_stop_0" [ style=bold color="green" fontcolor="orange" ] - "group_nfs_stopped_0" -> "group_nfs_start_0" [ style = bold] - "group_nfs_stopped_0" [ style=bold color="green" fontcolor="orange" ] -+"resource_ipaddr1_single_monitor_0 power720-1" -> "group_nfs_stopped_0" [ style = bold] - "resource_ipaddr1_single_monitor_0 power720-1" -> "resource_ipaddr1_single_start_0 power720-1" [ style = bold] - "resource_ipaddr1_single_monitor_0 power720-1" [ style=bold color="green" fontcolor="black" ] - "resource_ipaddr1_single_monitor_5000 power720-1" [ style=bold color="green" fontcolor="black" ] -@@ -51,6 +52,8 @@ digraph "g" { - "resource_ipaddr1_single_stop_0 power720-2" -> "group_nfs_stopped_0" [ style = bold] - "resource_ipaddr1_single_stop_0 power720-2" -> "resource_ipaddr1_single_start_0 power720-1" [ style = bold] - "resource_ipaddr1_single_stop_0 power720-2" [ style=bold color="green" fontcolor="black" ] -+"resource_nfsserver_single_monitor_0 power720-1" -> "group_nfs_stopped_0" [ style = bold] -+"resource_nfsserver_single_monitor_0 power720-1" -> "resource_ipaddr1_single_stop_0 power720-2" [ style = bold] - "resource_nfsserver_single_monitor_0 power720-1" -> "resource_nfsserver_single_start_0 power720-1" [ style = bold] - "resource_nfsserver_single_monitor_0 power720-1" [ style=bold color="green" fontcolor="black" ] - "resource_nfsserver_single_monitor_15000 power720-1" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/bug-n-387749.exp b/pengine/test10/bug-n-387749.exp -index 9f1f22d..d6fe8e4 100644 ---- a/pengine/test10/bug-n-387749.exp -+++ b/pengine/test10/bug-n-387749.exp -@@ -178,6 +178,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -@@ -270,6 +276,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/bug-rh-1097457.dot b/pengine/test10/bug-rh-1097457.dot -index 3dc24b6..9658ef6 100644 ---- a/pengine/test10/bug-rh-1097457.dot -+++ b/pengine/test10/bug-rh-1097457.dot -@@ -6,8 +6,12 @@ digraph "g" { - "FAKE3-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] - "FAKE3_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] - "FAKE4-IP_monitor_0 lamaVM1" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] -+"FAKE4-IP_monitor_0 lamaVM1" -> "FAKE4_stop_0 lamaVM2" [ style = bold] -+"FAKE4-IP_monitor_0 lamaVM1" -> "lamaVM2-G4_stopped_0" [ style = bold] - "FAKE4-IP_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] - "FAKE4-IP_monitor_0 lamaVM3" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] -+"FAKE4-IP_monitor_0 lamaVM3" -> "FAKE4_stop_0 lamaVM2" [ style = bold] -+"FAKE4-IP_monitor_0 lamaVM3" -> "lamaVM2-G4_stopped_0" [ style = bold] - "FAKE4-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] - "FAKE4-IP_monitor_30000 lamaVM2" [ style=bold color="green" fontcolor="black"] - "FAKE4-IP_start_0 lamaVM2" -> "FAKE4-IP_monitor_30000 lamaVM2" [ style = bold] -@@ -18,8 +22,10 @@ digraph "g" { - "FAKE4-IP_stop_0 lamaVM2" -> "lamaVM2-G4_stopped_0" [ style = bold] - "FAKE4-IP_stop_0 lamaVM2" [ style=bold color="green" fontcolor="orange"] - "FAKE4_monitor_0 lamaVM1" -> "FAKE4_start_0 lamaVM2" [ style = bold] -+"FAKE4_monitor_0 lamaVM1" -> "lamaVM2-G4_stopped_0" [ style = bold] - "FAKE4_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] - "FAKE4_monitor_0 lamaVM3" -> "FAKE4_start_0 lamaVM2" [ style = bold] -+"FAKE4_monitor_0 lamaVM3" -> "lamaVM2-G4_stopped_0" [ style = bold] - "FAKE4_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] - "FAKE4_monitor_30000 lamaVM2" [ style=bold color="green" fontcolor="black"] - "FAKE4_start_0 lamaVM2" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] -@@ -53,8 +59,10 @@ digraph "g" { - "FSlun1_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] - "FSlun2_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] - "FSlun3_monitor_0 lamaVM1" -> "FSlun3_start_0 lama2" [ style = bold] -+"FSlun3_monitor_0 lamaVM1" -> "VM2_stop_0 lama3" [ style = bold] - "FSlun3_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] - "FSlun3_monitor_0 lamaVM3" -> "FSlun3_start_0 lama2" [ style = bold] -+"FSlun3_monitor_0 lamaVM3" -> "VM2_stop_0 lama3" [ style = bold] - "FSlun3_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] - "FSlun3_monitor_10000 lama2" [ style=bold color="green" fontcolor="black"] - "FSlun3_monitor_10000 lamaVM2" [ style=bold color="green" fontcolor="black"] -diff --git a/pengine/test10/bug-rh-1097457.exp b/pengine/test10/bug-rh-1097457.exp -index 677c24b..0f60fa6 100644 ---- a/pengine/test10/bug-rh-1097457.exp -+++ b/pengine/test10/bug-rh-1097457.exp -@@ -42,6 +42,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -@@ -259,6 +265,18 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -353,6 +371,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/bundle-replicas-change.dot b/pengine/test10/bundle-replicas-change.dot -index 23264d9..c991371 100644 ---- a/pengine/test10/bundle-replicas-change.dot -+++ b/pengine/test10/bundle-replicas-change.dot -@@ -44,6 +44,7 @@ digraph "g" { - "httpd-bundle-docker-0_stop_0 rh74-test" [ style=bold color="green" fontcolor="black"] - "httpd-bundle-docker-1_monitor_0 rh74-test" -> "httpd-bundle-clone_start_0" [ style = bold] - "httpd-bundle-docker-1_monitor_0 rh74-test" -> "httpd-bundle-docker-1_start_0 rh74-test" [ style = bold] -+"httpd-bundle-docker-1_monitor_0 rh74-test" -> "httpd-bundle_stopped_0" [ style = bold] - "httpd-bundle-docker-1_monitor_0 rh74-test" [ style=bold color="green" fontcolor="black"] - "httpd-bundle-docker-1_monitor_60000 rh74-test" [ style=bold color="green" fontcolor="black"] - "httpd-bundle-docker-1_start_0 rh74-test" -> "httpd-bundle-1_monitor_0 rh74-test" [ style = bold] -@@ -54,6 +55,7 @@ digraph "g" { - "httpd-bundle-docker-1_start_0 rh74-test" [ style=bold color="green" fontcolor="black"] - "httpd-bundle-docker-2_monitor_0 rh74-test" -> "httpd-bundle-clone_start_0" [ style = bold] - "httpd-bundle-docker-2_monitor_0 rh74-test" -> "httpd-bundle-docker-2_start_0 rh74-test" [ style = bold] -+"httpd-bundle-docker-2_monitor_0 rh74-test" -> "httpd-bundle_stopped_0" [ style = bold] - "httpd-bundle-docker-2_monitor_0 rh74-test" [ style=bold color="green" fontcolor="black"] - "httpd-bundle-docker-2_monitor_60000 rh74-test" [ style=bold color="green" fontcolor="black"] - "httpd-bundle-docker-2_start_0 rh74-test" -> "httpd-bundle-2_monitor_0 rh74-test" [ style = bold] -diff --git a/pengine/test10/bundle-replicas-change.exp b/pengine/test10/bundle-replicas-change.exp -index 8d1c67f..2e8042e 100644 ---- a/pengine/test10/bundle-replicas-change.exp -+++ b/pengine/test10/bundle-replicas-change.exp -@@ -537,6 +537,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/clone-no-shuffle.dot b/pengine/test10/clone-no-shuffle.dot -index 9ac9e13..5174ecb 100644 ---- a/pengine/test10/clone-no-shuffle.dot -+++ b/pengine/test10/clone-no-shuffle.dot -@@ -11,6 +11,7 @@ digraph "g" { - "drbd1:0_stop_0 dktest2sles10" -> "ms-drbd1_stopped_0" [ style = bold] - "drbd1:0_stop_0 dktest2sles10" [ style=bold color="green" fontcolor="black" ] - "drbd1:1_monitor_0 dktest1sles10" -> "ms-drbd1_start_0" [ style = bold] -+"drbd1:1_monitor_0 dktest1sles10" -> "ms-drbd1_stopped_0" [ style = bold] - "drbd1:1_monitor_0 dktest1sles10" [ style=bold color="green" fontcolor="black" ] - "drbd1:1_monitor_11000 dktest1sles10" [ style=bold color="green" fontcolor="black" ] - "drbd1:1_post_notify_start_0 dktest1sles10" -> "ms-drbd1_confirmed-post_notify_running_0" [ style = bold] -@@ -73,6 +74,7 @@ digraph "g" { - "stonith-1_monitor_0 dktest2sles10" -> "stonith-1_start_0 dktest1sles10" [ style = bold] - "stonith-1_monitor_0 dktest2sles10" [ style=bold color="green" fontcolor="black" ] - "stonith-1_start_0 dktest1sles10" [ style=bold color="green" fontcolor="black" ] -+"testip_monitor_0 dktest1sles10" -> "ms-drbd1_demote_0" [ style = bold] - "testip_monitor_0 dktest1sles10" [ style=bold color="green" fontcolor="black" ] - "testip_stop_0 dktest2sles10" -> "ms-drbd1_demote_0" [ style = bold] - "testip_stop_0 dktest2sles10" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/clone-no-shuffle.exp b/pengine/test10/clone-no-shuffle.exp -index 615709c..554d8e2 100644 ---- a/pengine/test10/clone-no-shuffle.exp -+++ b/pengine/test10/clone-no-shuffle.exp -@@ -228,6 +228,9 @@ - - - -+ -+ -+ - - - -@@ -297,6 +300,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/group5.dot b/pengine/test10/group5.dot -index 6f524bd..3fe0193 100644 ---- a/pengine/test10/group5.dot -+++ b/pengine/test10/group5.dot -@@ -1,5 +1,6 @@ - digraph "g" { - "child_rsc1_monitor_0 node2" -> "child_rsc1_start_0 node2" [ style = bold] -+"child_rsc1_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] - "child_rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1_start_0 node2" -> "child_rsc2_start_0 node2" [ style = bold] - "child_rsc1_start_0 node2" -> "rsc2_running_0" [ style = bold] -@@ -7,7 +8,9 @@ - "child_rsc1_stop_0 node1" -> "child_rsc1_start_0 node2" [ style = bold] - "child_rsc1_stop_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc1_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"child_rsc2_monitor_0 node2" -> "child_rsc1_stop_0 node1" [ style = bold] - "child_rsc2_monitor_0 node2" -> "child_rsc2_start_0 node2" [ style = bold] -+"child_rsc2_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc2_start_0 node2" -> "child_rsc3_start_0 node2" [ style = bold] - "child_rsc2_start_0 node2" -> "rsc2_running_0" [ style = bold] -@@ -16,7 +19,9 @@ - "child_rsc2_stop_0 node1" -> "child_rsc2_start_0 node2" [ style = bold] - "child_rsc2_stop_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"child_rsc3_monitor_0 node2" -> "child_rsc2_stop_0 node1" [ style = bold] - "child_rsc3_monitor_0 node2" -> "child_rsc3_start_0 node2" [ style = bold] -+"child_rsc3_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] - "child_rsc3_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc3_start_0 node2" -> "rsc2_running_0" [ style = bold] - "child_rsc3_start_0 node2" [ style=bold color="green" fontcolor="black" ] -@@ -45,6 +50,7 @@ - "rsc2_stopped_0" -> "rsc1_stop_0 node1" [ style = bold] - "rsc2_stopped_0" -> "rsc2_start_0" [ style = bold] - "rsc2_stopped_0" [ style=bold color="green" fontcolor="orange" ] -+"rsc3_monitor_0 node2" -> "rsc2_stop_0" [ style = bold] - "rsc3_monitor_0 node2" -> "rsc3_start_0 node2" [ style = bold] - "rsc3_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc3_start_0 node2" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/group5.exp b/pengine/test10/group5.exp -index cb3480d..4ea2b08 100644 ---- a/pengine/test10/group5.exp -+++ b/pengine/test10/group5.exp -@@ -45,6 +45,15 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -66,6 +75,9 @@ - - - -+ -+ -+ - - - -@@ -134,6 +146,9 @@ - - - -+ -+ -+ - - - -@@ -181,6 +196,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/group6.dot b/pengine/test10/group6.dot -index 101763f..a563e05 100644 ---- a/pengine/test10/group6.dot -+++ b/pengine/test10/group6.dot -@@ -1,5 +1,6 @@ - digraph "g" { - "child_rsc1_monitor_0 node2" -> "child_rsc1_start_0 node2" [ style = bold] -+"child_rsc1_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1_start_0 node2" -> "child_rsc2_start_0 node2" [ style = bold] - "child_rsc1_start_0 node2" -> "rsc1_running_0" [ style = bold] -@@ -7,7 +8,9 @@ - "child_rsc1_stop_0 node1" -> "child_rsc1_start_0 node2" [ style = bold] - "child_rsc1_stop_0 node1" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"child_rsc2_monitor_0 node2" -> "child_rsc1_stop_0 node1" [ style = bold] - "child_rsc2_monitor_0 node2" -> "child_rsc2_start_0 node2" [ style = bold] -+"child_rsc2_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc2_start_0 node2" -> "child_rsc3_start_0 node2" [ style = bold] - "child_rsc2_start_0 node2" -> "rsc1_running_0" [ style = bold] -@@ -16,7 +19,9 @@ - "child_rsc2_stop_0 node1" -> "child_rsc2_start_0 node2" [ style = bold] - "child_rsc2_stop_0 node1" -> "rsc1_stopped_0" [ style = bold] - "child_rsc2_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"child_rsc3_monitor_0 node2" -> "child_rsc2_stop_0 node1" [ style = bold] - "child_rsc3_monitor_0 node2" -> "child_rsc3_start_0 node2" [ style = bold] -+"child_rsc3_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc3_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc3_start_0 node2" -> "rsc1_running_0" [ style = bold] - "child_rsc3_start_0 node2" [ style=bold color="green" fontcolor="black" ] -@@ -25,6 +30,7 @@ - "child_rsc3_stop_0 node1" -> "rsc1_stopped_0" [ style = bold] - "child_rsc3_stop_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc4_monitor_0 node2" -> "child_rsc4_start_0 node2" [ style = bold] -+"child_rsc4_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] - "child_rsc4_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc4_start_0 node2" -> "child_rsc5_start_0 node2" [ style = bold] - "child_rsc4_start_0 node2" -> "rsc2_running_0" [ style = bold] -@@ -32,7 +38,9 @@ - "child_rsc4_stop_0 node1" -> "child_rsc4_start_0 node2" [ style = bold] - "child_rsc4_stop_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc4_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"child_rsc5_monitor_0 node2" -> "child_rsc4_stop_0 node1" [ style = bold] - "child_rsc5_monitor_0 node2" -> "child_rsc5_start_0 node2" [ style = bold] -+"child_rsc5_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] - "child_rsc5_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc5_start_0 node2" -> "child_rsc6_start_0 node2" [ style = bold] - "child_rsc5_start_0 node2" -> "rsc2_running_0" [ style = bold] -@@ -41,7 +49,9 @@ - "child_rsc5_stop_0 node1" -> "child_rsc5_start_0 node2" [ style = bold] - "child_rsc5_stop_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc5_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"child_rsc6_monitor_0 node2" -> "child_rsc5_stop_0 node1" [ style = bold] - "child_rsc6_monitor_0 node2" -> "child_rsc6_start_0 node2" [ style = bold] -+"child_rsc6_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] - "child_rsc6_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc6_start_0 node2" -> "rsc2_running_0" [ style = bold] - "child_rsc6_start_0 node2" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/group6.exp b/pengine/test10/group6.exp -index a74b155..cddd6f4 100644 ---- a/pengine/test10/group6.exp -+++ b/pengine/test10/group6.exp -@@ -7,6 +7,15 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -93,6 +102,9 @@ - - - -+ -+ -+ - - - -@@ -140,6 +152,9 @@ - - - -+ -+ -+ - - - -@@ -208,6 +223,15 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -293,6 +317,9 @@ - - - -+ -+ -+ - - - -@@ -340,6 +367,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/group9.dot b/pengine/test10/group9.dot -index d3c3d0e..610fe93 100644 ---- a/pengine/test10/group9.dot -+++ b/pengine/test10/group9.dot -@@ -25,7 +25,9 @@ - "foo_stopped_0" [ style=bold color="green" fontcolor="orange" ] - "rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] -+"rsc3_monitor_0 node2" -> "foo_stopped_0" [ style = bold] - "rsc3_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] -+"rsc4_monitor_0 node2" -> "foo_stopped_0" [ style = bold] - "rsc4_monitor_0 node2" -> "rsc4_start_0 node1" [ style = bold] - "rsc4_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc4_start_0 node1" -> "foo_running_0" [ style = bold] -@@ -34,6 +36,8 @@ - "rsc4_stop_0 node1" -> "foo_stopped_0" [ style = bold] - "rsc4_stop_0 node1" -> "rsc4_start_0 node1" [ style = bold] - "rsc4_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"rsc5_monitor_0 node2" -> "foo_stopped_0" [ style = bold] -+"rsc5_monitor_0 node2" -> "rsc4_stop_0 node1" [ style = bold] - "rsc5_monitor_0 node2" -> "rsc5_start_0 node1" [ style = bold] - "rsc5_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc5_start_0 node1" -> "foo_running_0" [ style = bold] -@@ -42,6 +46,7 @@ - "rsc5_stop_0 node1" -> "rsc4_stop_0 node1" [ style = bold] - "rsc5_stop_0 node1" -> "rsc5_start_0 node1" [ style = bold] - "rsc5_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"rsc6_monitor_0 node2" -> "bar_stopped_0" [ style = bold] - "rsc6_monitor_0 node2" -> "rsc6_start_0 node2" [ style = bold] - "rsc6_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc6_start_0 node2" -> "bar_running_0" [ style = bold] -@@ -50,6 +55,8 @@ - "rsc6_stop_0 node1" -> "bar_stopped_0" [ style = bold] - "rsc6_stop_0 node1" -> "rsc6_start_0 node2" [ style = bold] - "rsc6_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"rsc7_monitor_0 node2" -> "bar_stopped_0" [ style = bold] -+"rsc7_monitor_0 node2" -> "rsc6_stop_0 node1" [ style = bold] - "rsc7_monitor_0 node2" -> "rsc7_start_0 node2" [ style = bold] - "rsc7_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc7_start_0 node2" -> "bar_running_0" [ style = bold] -@@ -59,6 +66,8 @@ - "rsc7_stop_0 node1" -> "rsc6_stop_0 node1" [ style = bold] - "rsc7_stop_0 node1" -> "rsc7_start_0 node2" [ style = bold] - "rsc7_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"rsc8_monitor_0 node2" -> "bar_stopped_0" [ style = bold] -+"rsc8_monitor_0 node2" -> "rsc7_stop_0 node1" [ style = bold] - "rsc8_monitor_0 node2" -> "rsc8_start_0 node2" [ style = bold] - "rsc8_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc8_start_0 node2" -> "bar_running_0" [ style = bold] -diff --git a/pengine/test10/group9.exp b/pengine/test10/group9.exp -index cf026e6..f05c2c2 100644 ---- a/pengine/test10/group9.exp -+++ b/pengine/test10/group9.exp -@@ -28,6 +28,15 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -119,6 +128,9 @@ - - - -+ -+ -+ - - - -@@ -181,6 +193,15 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -263,6 +284,9 @@ - - - -+ -+ -+ - - - -@@ -316,6 +340,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/inc2.dot b/pengine/test10/inc2.dot -index 456f21f..357536f 100644 ---- a/pengine/test10/inc2.dot -+++ b/pengine/test10/inc2.dot -@@ -1,9 +1,12 @@ - digraph "g" { - "child_rsc1:0_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:0_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:0_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:1_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:1_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:2_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:2_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:2_start_0 node2" -> "rsc1_running_0" [ style = bold] - "child_rsc1:2_start_0 node2" [ style=bold color="green" fontcolor="black" ] -@@ -11,6 +14,7 @@ - "child_rsc1:2_stop_0 node1" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:2_stop_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:3_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:3_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:3_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:3_start_0 node2" -> "rsc1_running_0" [ style = bold] - "child_rsc1:3_start_0 node2" [ style=bold color="green" fontcolor="black" ] -@@ -18,6 +22,7 @@ - "child_rsc1:3_stop_0 node1" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:3_stop_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:4_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:4_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:4_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:4_stop_0 node1" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:4_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/inc2.exp b/pengine/test10/inc2.exp -index 19b9b1a..10c6f43 100644 ---- a/pengine/test10/inc2.exp -+++ b/pengine/test10/inc2.exp -@@ -123,6 +123,21 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/inc3.dot b/pengine/test10/inc3.dot -index 8e5b063..36ab9b4 100644 ---- a/pengine/test10/inc3.dot -+++ b/pengine/test10/inc3.dot -@@ -1,9 +1,12 @@ - digraph "g" { - "child_rsc1:0_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:0_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:0_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:1_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:1_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:2_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:2_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:2_start_0 node2" -> "rsc1_running_0" [ style = bold] - "child_rsc1:2_start_0 node2" [ style=bold color="green" fontcolor="black" ] -@@ -11,6 +14,7 @@ - "child_rsc1:2_stop_0 node1" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:2_stop_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:3_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:3_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:3_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:3_start_0 node2" -> "rsc1_running_0" [ style = bold] - "child_rsc1:3_start_0 node2" [ style=bold color="green" fontcolor="black" ] -@@ -18,16 +22,21 @@ - "child_rsc1:3_stop_0 node1" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:3_stop_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:4_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:4_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:4_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:4_stop_0 node1" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:4_stop_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:0_monitor_0 node1" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:0_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:0_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:1_monitor_0 node1" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:1_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:2_monitor_0 node1" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:2_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:2_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:3_monitor_0 node1" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:3_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:3_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:3_start_0 node1" -> "rsc2_running_0" [ style = bold] - "child_rsc2:3_start_0 node1" [ style=bold color="green" fontcolor="black" ] -@@ -35,6 +44,7 @@ - "child_rsc2:3_stop_0 node2" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:3_stop_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:4_monitor_0 node1" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:4_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:4_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:4_start_0 node1" -> "rsc2_running_0" [ style = bold] - "child_rsc2:4_start_0 node1" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/inc3.exp b/pengine/test10/inc3.exp -index 55f01b6..790a934 100644 ---- a/pengine/test10/inc3.exp -+++ b/pengine/test10/inc3.exp -@@ -123,6 +123,21 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -304,6 +319,21 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/inc4.dot b/pengine/test10/inc4.dot -index 250052f..5c2ec9c 100644 ---- a/pengine/test10/inc4.dot -+++ b/pengine/test10/inc4.dot -@@ -1,9 +1,12 @@ - digraph "g" { - "child_rsc1:0_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:0_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:0_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:1_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:1_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:2_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:2_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:2_start_0 node2" -> "child_rsc1:3_start_0 node2" [ style = bold] - "child_rsc1:2_start_0 node2" -> "rsc1_running_0" [ style = bold] -@@ -11,7 +14,9 @@ - "child_rsc1:2_stop_0 node1" -> "child_rsc1:2_start_0 node2" [ style = bold] - "child_rsc1:2_stop_0 node1" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:2_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"child_rsc1:3_monitor_0 node2" -> "child_rsc1:2_stop_0 node1" [ style = bold] - "child_rsc1:3_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:3_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:3_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:3_start_0 node2" -> "rsc1_running_0" [ style = bold] - "child_rsc1:3_start_0 node2" [ style=bold color="green" fontcolor="black" ] -@@ -19,18 +24,24 @@ - "child_rsc1:3_stop_0 node1" -> "child_rsc1:3_start_0 node2" [ style = bold] - "child_rsc1:3_stop_0 node1" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:3_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"child_rsc1:4_monitor_0 node2" -> "child_rsc1:3_stop_0 node1" [ style = bold] - "child_rsc1:4_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:4_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:4_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:4_stop_0 node1" -> "child_rsc1:3_stop_0 node1" [ style = bold] - "child_rsc1:4_stop_0 node1" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:4_stop_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:0_monitor_0 node1" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:0_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:0_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:1_monitor_0 node1" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:1_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:2_monitor_0 node1" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:2_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:2_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:3_monitor_0 node1" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:3_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:3_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:3_start_0 node1" -> "child_rsc2:4_start_0 node1" [ style = bold] - "child_rsc2:3_start_0 node1" -> "rsc2_running_0" [ style = bold] -@@ -38,7 +49,9 @@ - "child_rsc2:3_stop_0 node2" -> "child_rsc2:3_start_0 node1" [ style = bold] - "child_rsc2:3_stop_0 node2" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:3_stop_0 node2" [ style=bold color="green" fontcolor="black" ] -+"child_rsc2:4_monitor_0 node1" -> "child_rsc2:3_stop_0 node2" [ style = bold] - "child_rsc2:4_monitor_0 node1" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:4_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:4_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:4_start_0 node1" -> "rsc2_running_0" [ style = bold] - "child_rsc2:4_start_0 node1" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/inc4.exp b/pengine/test10/inc4.exp -index ccc77d4..a5e9a09 100644 ---- a/pengine/test10/inc4.exp -+++ b/pengine/test10/inc4.exp -@@ -42,6 +42,9 @@ - - - -+ -+ -+ - - - -@@ -86,6 +89,9 @@ - - - -+ -+ -+ - - - -@@ -132,6 +138,21 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -254,6 +275,9 @@ - - - -+ -+ -+ - - - -@@ -319,6 +343,21 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/inc5.dot b/pengine/test10/inc5.dot -index dad7dd3..2a3d9f5 100644 ---- a/pengine/test10/inc5.dot -+++ b/pengine/test10/inc5.dot -@@ -4,8 +4,10 @@ - "child_rsc1:2_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:0_monitor_0 node2" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:0_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:0_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:1_monitor_0 node2" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:1_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:1_start_0 node2" -> "rsc2_running_0" [ style = bold] - "child_rsc2:1_start_0 node2" [ style=bold color="green" fontcolor="black" ] -@@ -13,16 +15,20 @@ - "child_rsc2:1_stop_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:1_stop_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:2_monitor_0 node1" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:2_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:2_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:2_monitor_0 node2" -> "rsc2_start_0" [ style = bold] -+"child_rsc2:2_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc3:0_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc3:1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc3:2_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc3:2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc4:0_monitor_0 node2" -> "rsc4_start_0" [ style = bold] -+"child_rsc4:0_monitor_0 node2" -> "rsc4_stopped_0" [ style = bold] - "child_rsc4:0_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc4:1_monitor_0 node2" -> "rsc4_start_0" [ style = bold] -+"child_rsc4:1_monitor_0 node2" -> "rsc4_stopped_0" [ style = bold] - "child_rsc4:1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc4:1_start_0 node2" -> "rsc4_running_0" [ style = bold] - "child_rsc4:1_start_0 node2" [ style=bold color="green" fontcolor="black" ] -@@ -30,12 +36,16 @@ - "child_rsc4:1_stop_0 node1" -> "rsc4_stopped_0" [ style = bold] - "child_rsc4:1_stop_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc4:2_monitor_0 node1" -> "rsc4_start_0" [ style = bold] -+"child_rsc4:2_monitor_0 node1" -> "rsc4_stopped_0" [ style = bold] - "child_rsc4:2_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc4:2_monitor_0 node2" -> "rsc4_start_0" [ style = bold] -+"child_rsc4:2_monitor_0 node2" -> "rsc4_stopped_0" [ style = bold] - "child_rsc4:2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc5:0_monitor_0 node1" -> "rsc5_start_0" [ style = bold] -+"child_rsc5:0_monitor_0 node1" -> "rsc5_stopped_0" [ style = bold] - "child_rsc5:0_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc5:1_monitor_0 node1" -> "rsc5_start_0" [ style = bold] -+"child_rsc5:1_monitor_0 node1" -> "rsc5_stopped_0" [ style = bold] - "child_rsc5:1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc5:1_start_0 node1" -> "rsc5_running_0" [ style = bold] - "child_rsc5:1_start_0 node1" [ style=bold color="green" fontcolor="black" ] -@@ -43,16 +53,20 @@ - "child_rsc5:1_stop_0 node2" -> "rsc5_stopped_0" [ style = bold] - "child_rsc5:1_stop_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc5:2_monitor_0 node1" -> "rsc5_start_0" [ style = bold] -+"child_rsc5:2_monitor_0 node1" -> "rsc5_stopped_0" [ style = bold] - "child_rsc5:2_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc5:2_monitor_0 node2" -> "rsc5_start_0" [ style = bold] -+"child_rsc5:2_monitor_0 node2" -> "rsc5_stopped_0" [ style = bold] - "child_rsc5:2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc6:0_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc6:1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc6:2_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc6:2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc7:0_monitor_0 node1" -> "rsc7_start_0" [ style = bold] -+"child_rsc7:0_monitor_0 node1" -> "rsc7_stopped_0" [ style = bold] - "child_rsc7:0_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc7:1_monitor_0 node1" -> "rsc7_start_0" [ style = bold] -+"child_rsc7:1_monitor_0 node1" -> "rsc7_stopped_0" [ style = bold] - "child_rsc7:1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc7:1_start_0 node1" -> "rsc7_running_0" [ style = bold] - "child_rsc7:1_start_0 node1" [ style=bold color="green" fontcolor="black" ] -@@ -60,8 +74,10 @@ - "child_rsc7:1_stop_0 node2" -> "rsc7_stopped_0" [ style = bold] - "child_rsc7:1_stop_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc7:2_monitor_0 node1" -> "rsc7_start_0" [ style = bold] -+"child_rsc7:2_monitor_0 node1" -> "rsc7_stopped_0" [ style = bold] - "child_rsc7:2_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc7:2_monitor_0 node2" -> "rsc7_start_0" [ style = bold] -+"child_rsc7:2_monitor_0 node2" -> "rsc7_stopped_0" [ style = bold] - "child_rsc7:2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc8:0_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc8:1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/inc5.exp b/pengine/test10/inc5.exp -index 0d19405..ecf8d3d 100644 ---- a/pengine/test10/inc5.exp -+++ b/pengine/test10/inc5.exp -@@ -108,6 +108,18 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -271,6 +283,18 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -398,6 +422,18 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -561,6 +597,18 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/master-7.dot b/pengine/test10/master-7.dot -index cd89c08..6cf865b 100644 ---- a/pengine/test10/master-7.dot -+++ b/pengine/test10/master-7.dot -@@ -10,9 +10,13 @@ digraph "g" { - "DoFencing_stopped_0" [ style=bold color="green" fontcolor="orange" ] - "child_DoFencing:0_stop_0 c001n01" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:0_stop_0 c001n01" [ style=bold color="green" fontcolor="orange" ] -+"child_DoFencing:2_monitor_0 c001n03" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:2_monitor_0 c001n08" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_monitor_0 c001n08" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:3_monitor_0 c001n02" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 c001n02" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:3_monitor_0 c001n03" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "group-1_running_0" [ style=bold color="green" fontcolor="orange" ] - "group-1_start_0" -> "group-1_running_0" [ style = bold] -@@ -72,16 +76,25 @@ digraph "g" { - "ocf_msdummy:0_demote_0 c001n01" [ style=bold color="green" fontcolor="orange" ] - "ocf_msdummy:0_stop_0 c001n01" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:0_stop_0 c001n01" [ style=bold color="green" fontcolor="orange" ] -+"ocf_msdummy:4_monitor_0 c001n02" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:4_monitor_0 c001n02" [ style=bold color="green" fontcolor="black" ] -+"ocf_msdummy:4_monitor_0 c001n03" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:4_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] -+"ocf_msdummy:4_monitor_0 c001n08" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:4_monitor_0 c001n08" [ style=bold color="green" fontcolor="black" ] - "ocf_msdummy:4_stop_0 c001n01" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:4_stop_0 c001n01" [ style=bold color="green" fontcolor="orange" ] -+"ocf_msdummy:5_monitor_0 c001n02" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:5_monitor_0 c001n02" [ style=bold color="green" fontcolor="black" ] -+"ocf_msdummy:5_monitor_0 c001n08" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:5_monitor_0 c001n08" [ style=bold color="green" fontcolor="black" ] -+"ocf_msdummy:6_monitor_0 c001n03" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:6_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] -+"ocf_msdummy:6_monitor_0 c001n08" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:6_monitor_0 c001n08" [ style=bold color="green" fontcolor="black" ] -+"ocf_msdummy:7_monitor_0 c001n02" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:7_monitor_0 c001n02" [ style=bold color="green" fontcolor="black" ] -+"ocf_msdummy:7_monitor_0 c001n03" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:7_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "rsc_c001n01_monitor_5000 c001n03" [ style=bold color="green" fontcolor="black" ] - "rsc_c001n01_start_0 c001n03" -> "rsc_c001n01_monitor_5000 c001n03" [ style = bold] -diff --git a/pengine/test10/master-7.exp b/pengine/test10/master-7.exp -index 05abae6..d93ebbf 100644 ---- a/pengine/test10/master-7.exp -+++ b/pengine/test10/master-7.exp -@@ -393,6 +393,18 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -572,6 +584,33 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/master-8.dot b/pengine/test10/master-8.dot -index 58909fe..067f5da 100644 ---- a/pengine/test10/master-8.dot -+++ b/pengine/test10/master-8.dot -@@ -10,9 +10,13 @@ digraph "g" { - "DoFencing_stopped_0" [ style=bold color="green" fontcolor="orange" ] - "child_DoFencing:0_stop_0 c001n01" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:0_stop_0 c001n01" [ style=bold color="green" fontcolor="orange" ] -+"child_DoFencing:2_monitor_0 c001n03" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:2_monitor_0 c001n08" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_monitor_0 c001n08" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:3_monitor_0 c001n02" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 c001n02" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:3_monitor_0 c001n03" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "group-1_running_0" [ style=bold color="green" fontcolor="orange" ] - "group-1_start_0" -> "group-1_running_0" [ style = bold] -@@ -84,24 +88,34 @@ digraph "g" { - "ocf_msdummy:0_stop_0 c001n01" -> "ocf_msdummy:0_start_0 c001n03" [ style = bold] - "ocf_msdummy:0_stop_0 c001n01" [ style=bold color="green" fontcolor="orange" ] - "ocf_msdummy:4_monitor_0 c001n02" -> "master_rsc_1_start_0" [ style = bold] -+"ocf_msdummy:4_monitor_0 c001n02" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:4_monitor_0 c001n02" [ style=bold color="green" fontcolor="black" ] - "ocf_msdummy:4_monitor_0 c001n03" -> "master_rsc_1_start_0" [ style = bold] -+"ocf_msdummy:4_monitor_0 c001n03" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:4_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "ocf_msdummy:4_monitor_0 c001n08" -> "master_rsc_1_start_0" [ style = bold] -+"ocf_msdummy:4_monitor_0 c001n08" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:4_monitor_0 c001n08" [ style=bold color="green" fontcolor="black" ] - "ocf_msdummy:5_monitor_0 c001n02" -> "master_rsc_1_start_0" [ style = bold] -+"ocf_msdummy:5_monitor_0 c001n02" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:5_monitor_0 c001n02" [ style=bold color="green" fontcolor="black" ] - "ocf_msdummy:5_monitor_0 c001n03" -> "master_rsc_1_start_0" [ style = bold] -+"ocf_msdummy:5_monitor_0 c001n03" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:5_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "ocf_msdummy:5_monitor_0 c001n08" -> "master_rsc_1_start_0" [ style = bold] -+"ocf_msdummy:5_monitor_0 c001n08" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:5_monitor_0 c001n08" [ style=bold color="green" fontcolor="black" ] - "ocf_msdummy:6_monitor_0 c001n03" -> "master_rsc_1_start_0" [ style = bold] -+"ocf_msdummy:6_monitor_0 c001n03" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:6_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "ocf_msdummy:6_monitor_0 c001n08" -> "master_rsc_1_start_0" [ style = bold] -+"ocf_msdummy:6_monitor_0 c001n08" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:6_monitor_0 c001n08" [ style=bold color="green" fontcolor="black" ] - "ocf_msdummy:7_monitor_0 c001n02" -> "master_rsc_1_start_0" [ style = bold] -+"ocf_msdummy:7_monitor_0 c001n02" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:7_monitor_0 c001n02" [ style=bold color="green" fontcolor="black" ] - "ocf_msdummy:7_monitor_0 c001n03" -> "master_rsc_1_start_0" [ style = bold] -+"ocf_msdummy:7_monitor_0 c001n03" -> "master_rsc_1_stopped_0" [ style = bold] - "ocf_msdummy:7_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "rsc_c001n01_monitor_5000 c001n03" [ style=bold color="green" fontcolor="black" ] - "rsc_c001n01_start_0 c001n03" -> "rsc_c001n01_monitor_5000 c001n03" [ style = bold] -diff --git a/pengine/test10/master-8.exp b/pengine/test10/master-8.exp -index 477dbf8..fb584c5 100644 ---- a/pengine/test10/master-8.exp -+++ b/pengine/test10/master-8.exp -@@ -393,6 +393,18 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -601,6 +613,36 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/master-9.dot b/pengine/test10/master-9.dot -index 16e756e..370c214 100644 ---- a/pengine/test10/master-9.dot -+++ b/pengine/test10/master-9.dot -@@ -4,13 +4,18 @@ - "DoFencing_stop_0" -> "child_DoFencing:1_stop_0 ibm1" [ style = bold] - "DoFencing_stop_0" [ style=bold color="green" fontcolor="orange" ] - "DoFencing_stopped_0" [ style=bold color="green" fontcolor="orange" ] -+"child_DoFencing:1_monitor_0 va1" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:1_monitor_0 va1" [ style=bold color="green" fontcolor="black" ] - "child_DoFencing:1_stop_0 ibm1" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:1_stop_0 ibm1" -> "do_shutdown ibm1" [ style = bold] - "child_DoFencing:1_stop_0 ibm1" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:2_monitor_0 ibm1" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_monitor_0 ibm1" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:2_monitor_0 va1" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_monitor_0 va1" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:3_monitor_0 ibm1" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 ibm1" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:3_monitor_0 va1" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 va1" [ style=bold color="green" fontcolor="black" ] - "do_shutdown ibm1" [ style=bold color="green" fontcolor="black" ] - "heartbeat_127.0.0.12_monitor_5000 va1" [ style=dashed color="red" fontcolor="black" ] -diff --git a/pengine/test10/master-9.exp b/pengine/test10/master-9.exp -index f2b0ba5..436284d 100644 ---- a/pengine/test10/master-9.exp -+++ b/pengine/test10/master-9.exp -@@ -65,6 +65,21 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/notify-0.dot b/pengine/test10/notify-0.dot -index b0a1355..691cc68 100644 ---- a/pengine/test10/notify-0.dot -+++ b/pengine/test10/notify-0.dot -@@ -5,6 +5,7 @@ - "child_rsc1:1_start_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:0_stop_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:0_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"child_rsc2:1_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "rsc1_running_0" [ style=bold color="green" fontcolor="orange" ] - "rsc1_start_0" -> "child_rsc1:1_start_0 node1" [ style = bold] -diff --git a/pengine/test10/notify-0.exp b/pengine/test10/notify-0.exp -index e5abf68..bb9955f 100644 ---- a/pengine/test10/notify-0.exp -+++ b/pengine/test10/notify-0.exp -@@ -78,6 +78,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/notify-1.dot b/pengine/test10/notify-1.dot -index d2f6183..ecdf8ba 100644 ---- a/pengine/test10/notify-1.dot -+++ b/pengine/test10/notify-1.dot -@@ -13,6 +13,7 @@ - "child_rsc2:0_pre_notify_stop_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:0_stop_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:0_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"child_rsc2:1_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "rsc1_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange" ] - "rsc1_confirmed-pre_notify_start_0" -> "rsc1_post_notify_running_0" [ style = bold] -diff --git a/pengine/test10/notify-1.exp b/pengine/test10/notify-1.exp -index ee763cc..ca7d42a 100644 ---- a/pengine/test10/notify-1.exp -+++ b/pengine/test10/notify-1.exp -@@ -239,6 +239,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/notify-2.dot b/pengine/test10/notify-2.dot -index d2f6183..ecdf8ba 100644 ---- a/pengine/test10/notify-2.dot -+++ b/pengine/test10/notify-2.dot -@@ -13,6 +13,7 @@ - "child_rsc2:0_pre_notify_stop_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:0_stop_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:0_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"child_rsc2:1_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "rsc1_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange" ] - "rsc1_confirmed-pre_notify_start_0" -> "rsc1_post_notify_running_0" [ style = bold] -diff --git a/pengine/test10/notify-2.exp b/pengine/test10/notify-2.exp -index ee763cc..ca7d42a 100644 ---- a/pengine/test10/notify-2.exp -+++ b/pengine/test10/notify-2.exp -@@ -239,6 +239,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/notify-3.dot b/pengine/test10/notify-3.dot -index 66341d0..a0e1c0b 100644 ---- a/pengine/test10/notify-3.dot -+++ b/pengine/test10/notify-3.dot -@@ -1,5 +1,6 @@ - digraph "g" { - "child_rsc1:0_monitor_0 node2" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:0_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:0_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:0_post_notify_start_0 node1" -> "rsc1_confirmed-post_notify_running_0" [ style = bold] - "child_rsc1:0_post_notify_start_0 node1" [ style=bold color="green" fontcolor="black" ] -@@ -10,6 +11,7 @@ - "child_rsc1:0_pre_notify_stop_0 node1" -> "rsc1_confirmed-pre_notify_stop_0" [ style = bold] - "child_rsc1:0_pre_notify_stop_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:1_monitor_0 node1" -> "rsc1_start_0" [ style = bold] -+"child_rsc1:1_monitor_0 node1" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc1:1_post_notify_start_0 node1" -> "rsc1_confirmed-post_notify_running_0" [ style = bold] - "child_rsc1:1_post_notify_start_0 node1" [ style=bold color="green" fontcolor="black" ] -@@ -20,12 +22,15 @@ - "child_rsc1:1_stop_0 node2" -> "child_rsc1:1_start_0 node1" [ style = bold] - "child_rsc1:1_stop_0 node2" -> "rsc1_stopped_0" [ style = bold] - "child_rsc1:1_stop_0 node2" [ style=bold color="green" fontcolor="black" ] -+"child_rsc2:0_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:0_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:0_pre_notify_stop_0 node1" -> "rsc2_confirmed-pre_notify_stop_0" [ style = bold] - "child_rsc2:0_pre_notify_stop_0 node1" [ style=bold color="green" fontcolor="black" ] - "child_rsc2:0_stop_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:0_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"child_rsc2:1_monitor_0 node1" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] -+"child_rsc2:1_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] - "child_rsc2:1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc1_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange" ] - "rsc1_confirmed-post_notify_stopped_0" -> "rsc1_pre_notify_start_0" [ style = bold] -diff --git a/pengine/test10/notify-3.exp b/pengine/test10/notify-3.exp -index 388e009..7948b38 100644 ---- a/pengine/test10/notify-3.exp -+++ b/pengine/test10/notify-3.exp -@@ -188,6 +188,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -@@ -414,6 +420,15 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/novell-252693-3.dot b/pengine/test10/novell-252693-3.dot -index 6d225f9..ed1d276 100644 ---- a/pengine/test10/novell-252693-3.dot -+++ b/pengine/test10/novell-252693-3.dot -@@ -70,6 +70,7 @@ - "evmsdcloneset_start_0" -> "evmsdcloneset_running_0" [ style = bold] - "evmsdcloneset_start_0" [ style=bold color="green" fontcolor="orange" ] - "imagestoreclone:0_monitor_0 node1" -> "imagestorecloneset_start_0" [ style = bold] -+"imagestoreclone:0_monitor_0 node1" -> "imagestorecloneset_stopped_0" [ style = bold] - "imagestoreclone:0_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "imagestoreclone:0_monitor_20000 node1" [ style=bold color="green" fontcolor="black" ] - "imagestoreclone:0_post_notify_start_0 node1" -> "imagestorecloneset_confirmed-post_notify_running_0" [ style = bold] -@@ -129,6 +130,7 @@ - "sles10_migrate_from_0 node1" [ style=bold color="green" fontcolor="black"] - "sles10_migrate_to_0 node2" -> "sles10_migrate_from_0 node1" [ style = bold] - "sles10_migrate_to_0 node2" [ style=bold color="green" fontcolor="black"] -+"sles10_monitor_0 node1" -> "imagestorecloneset_stop_0" [ style = bold] - "sles10_monitor_0 node1" -> "sles10_migrate_to_0 node2" [ style = bold] - "sles10_monitor_0 node1" -> "sles10_start_0 node1" [ style = bold] - "sles10_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/novell-252693-3.exp b/pengine/test10/novell-252693-3.exp -index e7cc1c4..41b43c8 100644 ---- a/pengine/test10/novell-252693-3.exp -+++ b/pengine/test10/novell-252693-3.exp -@@ -456,6 +456,9 @@ - - - -+ -+ -+ - - - -@@ -468,6 +471,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/order3.dot b/pengine/test10/order3.dot -index dd4e427..9c94c40 100644 ---- a/pengine/test10/order3.dot -+++ b/pengine/test10/order3.dot -@@ -5,7 +5,9 @@ - "rsc1_start_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc1_stop_0 node1" -> "rsc1_start_0 node2" [ style = bold] - "rsc1_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"rsc2_monitor_0 node2" -> "rsc1_stop_0 node1" [ style = bold] - "rsc2_monitor_0 node2" -> "rsc2_start_0 node2" [ style = bold] -+"rsc2_monitor_0 node2" -> "rsc4_stop_0 node1" [ style = bold] - "rsc2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc2_start_0 node2" -> "rsc3_start_0 node2" [ style = bold] - "rsc2_start_0 node2" [ style=bold color="green" fontcolor="black" ] -@@ -13,6 +15,7 @@ - "rsc2_stop_0 node1" -> "rsc2_start_0 node2" [ style = bold] - "rsc2_stop_0 node1" -> "rsc4_stop_0 node1" [ style = bold] - "rsc2_stop_0 node1" [ style=bold color="green" fontcolor="black" ] -+"rsc3_monitor_0 node2" -> "rsc2_stop_0 node1" [ style = bold] - "rsc3_monitor_0 node2" -> "rsc3_start_0 node2" [ style = bold] - "rsc3_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc3_start_0 node2" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/order3.exp b/pengine/test10/order3.exp -index c6ee38e..038e006 100644 ---- a/pengine/test10/order3.exp -+++ b/pengine/test10/order3.exp -@@ -24,6 +24,9 @@ - - - -+ -+ -+ - - - -@@ -68,6 +71,9 @@ - - - -+ -+ -+ - - - -@@ -143,6 +149,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/rec-node-11.dot b/pengine/test10/rec-node-11.dot -index 6749d23..3f48ea8 100644 ---- a/pengine/test10/rec-node-11.dot -+++ b/pengine/test10/rec-node-11.dot -@@ -11,6 +11,7 @@ digraph "g" { - "group1_stopped_0" -> "group1_start_0" [ style = bold] - "group1_stopped_0" -> "rsc3_stop_0 node2" [ style = bold] - "group1_stopped_0" [ style=bold color="green" fontcolor="orange" ] -+"rsc1_monitor_0 node2" -> "group1_stopped_0" [ style = bold] - "rsc1_monitor_0 node2" -> "rsc1_start_0 node2" [ style = bold] - "rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc1_start_0 node2" -> "group1_running_0" [ style = bold] -@@ -19,6 +20,8 @@ digraph "g" { - "rsc1_stop_0 node1" -> "group1_stopped_0" [ style = bold] - "rsc1_stop_0 node1" -> "rsc1_start_0 node2" [ style = bold] - "rsc1_stop_0 node1" [ style=bold color="green" fontcolor="orange" ] -+"rsc2_monitor_0 node2" -> "group1_stopped_0" [ style = bold] -+"rsc2_monitor_0 node2" -> "rsc1_stop_0 node1" [ style = bold] - "rsc2_monitor_0 node2" -> "rsc2_start_0 node2" [ style = bold] - "rsc2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc2_start_0 node2" -> "group1_running_0" [ style = bold] -diff --git a/pengine/test10/rec-node-11.exp b/pengine/test10/rec-node-11.exp -index eca2455..d68f392 100644 ---- a/pengine/test10/rec-node-11.exp -+++ b/pengine/test10/rec-node-11.exp -@@ -32,6 +32,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -@@ -120,6 +126,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/reload-becomes-restart.dot b/pengine/test10/reload-becomes-restart.dot -index ed3720a..a6616f9 100644 ---- a/pengine/test10/reload-becomes-restart.dot -+++ b/pengine/test10/reload-becomes-restart.dot -@@ -37,6 +37,7 @@ digraph "g" { - "rsc1:1_start_0 node1" -> "rsc2_start_0 node1" [ style = bold] - "rsc1:1_start_0 node1" [ style=bold color="green" fontcolor="black"] - "rsc2:1_monitor_0 node2" -> "cl-rsc2_start_0" [ style = bold] -+"rsc2:1_monitor_0 node2" -> "cl-rsc2_stopped_0" [ style = bold] - "rsc2:1_monitor_0 node2" [ style=bold color="green" fontcolor="black"] - "rsc2:1_monitor_200000 node2" [ style=bold color="green" fontcolor="black"] - "rsc2:1_start_0 node2" -> "cl-rsc2_running_0" [ style = bold] -diff --git a/pengine/test10/reload-becomes-restart.exp b/pengine/test10/reload-becomes-restart.exp -index 63ebff0..c3e3721 100644 ---- a/pengine/test10/reload-becomes-restart.exp -+++ b/pengine/test10/reload-becomes-restart.exp -@@ -240,6 +240,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/remote-connection-unrecoverable.dot b/pengine/test10/remote-connection-unrecoverable.dot -index 4cc243b..0360cd0 100644 ---- a/pengine/test10/remote-connection-unrecoverable.dot -+++ b/pengine/test10/remote-connection-unrecoverable.dot -@@ -7,7 +7,9 @@ digraph "g" { - "remote1_stop_0 node1" [ style=bold color="green" fontcolor="orange"] - "rsc1_delete_0 remote1" -> "rsc1_start_0 node2" [ style = dashed] - "rsc1_delete_0 remote1" [ style=dashed color="red" fontcolor="black"] -+"rsc1_monitor_0 node2" -> "remote1_stop_0 node1" [ style = bold] - "rsc1_monitor_0 node2" -> "rsc1_start_0 node2" [ style = bold] -+"rsc1_monitor_0 node2" -> "rsc2-master_demote_0" [ style = bold] - "rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black"] - "rsc1_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] - "rsc1_start_0 node2" -> "rsc1_monitor_10000 node2" [ style = bold] -diff --git a/pengine/test10/remote-connection-unrecoverable.exp b/pengine/test10/remote-connection-unrecoverable.exp -index 59132fd..73fa7a1 100644 ---- a/pengine/test10/remote-connection-unrecoverable.exp -+++ b/pengine/test10/remote-connection-unrecoverable.exp -@@ -12,6 +12,9 @@ - - - -+ -+ -+ - - - -@@ -166,6 +169,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/rsc_dep1.dot b/pengine/test10/rsc_dep1.dot -index 49d686c..da1da7d 100644 ---- a/pengine/test10/rsc_dep1.dot -+++ b/pengine/test10/rsc_dep1.dot -@@ -1,11 +1,13 @@ - digraph "g" { - "rsc1_monitor_0 node1" -> "rsc1_start_0 node2" [ style = bold] -+"rsc1_monitor_0 node1" -> "rsc2_start_0 node1" [ style = bold] - "rsc1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "rsc1_monitor_0 node2" -> "rsc1_start_0 node2" [ style = bold] - "rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc1_start_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc2_monitor_0 node1" -> "rsc2_start_0 node1" [ style = bold] - "rsc2_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] -+"rsc2_monitor_0 node2" -> "rsc1_start_0 node2" [ style = bold] - "rsc2_monitor_0 node2" -> "rsc2_start_0 node1" [ style = bold] - "rsc2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc2_start_0 node1" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/rsc_dep1.exp b/pengine/test10/rsc_dep1.exp -index 50b7506..80a96e9 100644 ---- a/pengine/test10/rsc_dep1.exp -+++ b/pengine/test10/rsc_dep1.exp -@@ -11,6 +11,9 @@ - - - -+ -+ -+ - - - -@@ -45,6 +48,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/rsc_dep5.dot b/pengine/test10/rsc_dep5.dot -index ec7c307..8bdc4ed 100644 ---- a/pengine/test10/rsc_dep5.dot -+++ b/pengine/test10/rsc_dep5.dot -@@ -1,13 +1,17 @@ - digraph "g" { -+"rsc1_monitor_0 node1" -> "rsc3_start_0 node1" [ style = bold] - "rsc1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] -+"rsc1_monitor_0 node2" -> "rsc2_start_0 node2" [ style = bold] - "rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc2_monitor_0 node1" -> "rsc2_start_0 node2" [ style = bold] -+"rsc2_monitor_0 node1" -> "rsc3_start_0 node1" [ style = bold] - "rsc2_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] - "rsc2_monitor_0 node2" -> "rsc2_start_0 node2" [ style = bold] - "rsc2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc2_start_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc3_monitor_0 node1" -> "rsc3_start_0 node1" [ style = bold] - "rsc3_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] -+"rsc3_monitor_0 node2" -> "rsc2_start_0 node2" [ style = bold] - "rsc3_monitor_0 node2" -> "rsc3_start_0 node1" [ style = bold] - "rsc3_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] - "rsc3_start_0 node1" [ style=bold color="green" fontcolor="black" ] -diff --git a/pengine/test10/rsc_dep5.exp b/pengine/test10/rsc_dep5.exp -index 5647d46..6944272 100644 ---- a/pengine/test10/rsc_dep5.exp -+++ b/pengine/test10/rsc_dep5.exp -@@ -11,6 +11,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -@@ -45,8 +51,14 @@ - - - -+ -+ -+ - - -+ -+ -+ - - - -diff --git a/pengine/test10/unfence-definition.dot b/pengine/test10/unfence-definition.dot -index e2dc564..e899ff3 100644 ---- a/pengine/test10/unfence-definition.dot -+++ b/pengine/test10/unfence-definition.dot -@@ -12,11 +12,14 @@ digraph "g" { - "clvmd-clone_stopped_0" -> "dlm-clone_stop_0" [ style = bold] - "clvmd-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] - "clvmd:1_monitor_0 virt-2" -> "clvmd-clone_start_0" [ style = bold] -+"clvmd:1_monitor_0 virt-2" -> "clvmd-clone_stopped_0" [ style = bold] -+"clvmd:1_monitor_0 virt-2" -> "clvmd_stop_0 virt-1" [ style = bold] - "clvmd:1_monitor_0 virt-2" [ style=bold color="green" fontcolor="black"] - "clvmd:1_start_0 virt-2" -> "clvmd-clone_running_0" [ style = bold] - "clvmd:1_start_0 virt-2" -> "clvmd:2_start_0 virt-3" [ style = bold] - "clvmd:1_start_0 virt-2" [ style=bold color="green" fontcolor="black"] - "clvmd:2_monitor_0 virt-3" -> "clvmd-clone_start_0" [ style = bold] -+"clvmd:2_monitor_0 virt-3" -> "clvmd-clone_stopped_0" [ style = bold] - "clvmd:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] - "clvmd:2_start_0 virt-3" -> "clvmd-clone_running_0" [ style = bold] - "clvmd:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] -@@ -40,6 +43,7 @@ digraph "g" { - "dlm-clone_stopped_0" -> "dlm-clone_start_0" [ style = bold] - "dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] - "dlm:2_monitor_0 virt-3" -> "dlm-clone_start_0" [ style = bold] -+"dlm:2_monitor_0 virt-3" -> "dlm-clone_stopped_0" [ style = bold] - "dlm:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] - "dlm:2_start_0 virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold] - "dlm:2_start_0 virt-3" -> "dlm-clone_running_0" [ style = bold] -diff --git a/pengine/test10/unfence-definition.exp b/pengine/test10/unfence-definition.exp -index 019c03d..840a8d2 100644 ---- a/pengine/test10/unfence-definition.exp -+++ b/pengine/test10/unfence-definition.exp -@@ -137,6 +137,9 @@ - - - -+ -+ -+ - - - -@@ -223,6 +226,9 @@ - - - -+ -+ -+ - - - -@@ -301,6 +307,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/unfence-parameters.dot b/pengine/test10/unfence-parameters.dot -index ab3ce37..a1ee969 100644 ---- a/pengine/test10/unfence-parameters.dot -+++ b/pengine/test10/unfence-parameters.dot -@@ -12,11 +12,14 @@ digraph "g" { - "clvmd-clone_stopped_0" -> "dlm-clone_stop_0" [ style = bold] - "clvmd-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] - "clvmd:1_monitor_0 virt-2" -> "clvmd-clone_start_0" [ style = bold] -+"clvmd:1_monitor_0 virt-2" -> "clvmd-clone_stopped_0" [ style = bold] -+"clvmd:1_monitor_0 virt-2" -> "clvmd_stop_0 virt-1" [ style = bold] - "clvmd:1_monitor_0 virt-2" [ style=bold color="green" fontcolor="black"] - "clvmd:1_start_0 virt-2" -> "clvmd-clone_running_0" [ style = bold] - "clvmd:1_start_0 virt-2" -> "clvmd:2_start_0 virt-3" [ style = bold] - "clvmd:1_start_0 virt-2" [ style=bold color="green" fontcolor="black"] - "clvmd:2_monitor_0 virt-3" -> "clvmd-clone_start_0" [ style = bold] -+"clvmd:2_monitor_0 virt-3" -> "clvmd-clone_stopped_0" [ style = bold] - "clvmd:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] - "clvmd:2_start_0 virt-3" -> "clvmd-clone_running_0" [ style = bold] - "clvmd:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] -@@ -40,6 +43,7 @@ digraph "g" { - "dlm-clone_stopped_0" -> "dlm-clone_start_0" [ style = bold] - "dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] - "dlm:2_monitor_0 virt-3" -> "dlm-clone_start_0" [ style = bold] -+"dlm:2_monitor_0 virt-3" -> "dlm-clone_stopped_0" [ style = bold] - "dlm:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] - "dlm:2_start_0 virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold] - "dlm:2_start_0 virt-3" -> "dlm-clone_running_0" [ style = bold] -diff --git a/pengine/test10/unfence-parameters.exp b/pengine/test10/unfence-parameters.exp -index fc3317d..3e70cb8 100644 ---- a/pengine/test10/unfence-parameters.exp -+++ b/pengine/test10/unfence-parameters.exp -@@ -121,6 +121,9 @@ - - - -+ -+ -+ - - - -@@ -207,6 +210,9 @@ - - - -+ -+ -+ - - - -@@ -285,6 +291,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/unrunnable-1.dot b/pengine/test10/unrunnable-1.dot -index 7ead826..6164046 100644 ---- a/pengine/test10/unrunnable-1.dot -+++ b/pengine/test10/unrunnable-1.dot -@@ -11,10 +11,13 @@ - "child_192.168.100.182_monitor_5000 c001n03" [ style=dashed color="red" fontcolor="black" ] - "child_192.168.100.183_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "child_192.168.100.183_monitor_5000 c001n03" [ style=dashed color="red" fontcolor="black" ] -+"child_DoFencing:1_monitor_0 c001n03" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:1_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "child_DoFencing:1_stop_0 c001n02" -> "DoFencing_stopped_0" [ style = dashed] - "child_DoFencing:1_stop_0 c001n02" [ style=dashed color="red" fontcolor="black" ] -+"child_DoFencing:2_monitor_0 c001n03" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:2_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] -+"child_DoFencing:3_monitor_0 c001n03" -> "DoFencing_stopped_0" [ style = bold] - "child_DoFencing:3_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "rsc_c001n01_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] - "rsc_c001n01_monitor_5000 c001n03" [ style=dashed color="red" fontcolor="black" ] -diff --git a/pengine/test10/unrunnable-1.exp b/pengine/test10/unrunnable-1.exp -index 56fb4c0..94ab05c 100644 ---- a/pengine/test10/unrunnable-1.exp -+++ b/pengine/test10/unrunnable-1.exp -@@ -106,6 +106,15 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/whitebox-imply-stop-on-fence.dot b/pengine/test10/whitebox-imply-stop-on-fence.dot -index a58e7c7..62ba699 100644 ---- a/pengine/test10/whitebox-imply-stop-on-fence.dot -+++ b/pengine/test10/whitebox-imply-stop-on-fence.dot -@@ -20,7 +20,9 @@ - "clvmd-clone_stop_0" [ style=bold color="green" fontcolor="orange"] - "clvmd-clone_stopped_0" -> "dlm-clone_stop_0" [ style = bold] - "clvmd-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"clvmd_monitor_0 lxc-01_kiff-02" -> "clvmd-clone_stopped_0" [ style = bold] - "clvmd_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] -+"clvmd_monitor_0 lxc-02_kiff-02" -> "clvmd-clone_stopped_0" [ style = bold] - "clvmd_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] - "clvmd_stop_0 kiff-01" -> "clvmd-clone_stopped_0" [ style = bold] - "clvmd_stop_0 kiff-01" -> "dlm_stop_0 kiff-01" [ style = bold] -@@ -29,7 +31,9 @@ - "dlm-clone_stop_0" -> "dlm_stop_0 kiff-01" [ style = bold] - "dlm-clone_stop_0" [ style=bold color="green" fontcolor="orange"] - "dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"dlm_monitor_0 lxc-01_kiff-02" -> "dlm-clone_stopped_0" [ style = bold] - "dlm_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] -+"dlm_monitor_0 lxc-02_kiff-02" -> "dlm-clone_stopped_0" [ style = bold] - "dlm_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] - "dlm_stop_0 kiff-01" -> "dlm-clone_stopped_0" [ style = bold] - "dlm_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] -@@ -57,7 +61,9 @@ - "shared0-clone_stop_0" [ style=bold color="green" fontcolor="orange"] - "shared0-clone_stopped_0" -> "clvmd-clone_stop_0" [ style = bold] - "shared0-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"shared0_monitor_0 lxc-01_kiff-02" -> "shared0-clone_stopped_0" [ style = bold] - "shared0_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] -+"shared0_monitor_0 lxc-02_kiff-02" -> "shared0-clone_stopped_0" [ style = bold] - "shared0_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] - "shared0_stop_0 kiff-01" -> "clvmd_stop_0 kiff-01" [ style = bold] - "shared0_stop_0 kiff-01" -> "shared0-clone_stopped_0" [ style = bold] -diff --git a/pengine/test10/whitebox-imply-stop-on-fence.exp b/pengine/test10/whitebox-imply-stop-on-fence.exp -index cdba621..3b24768 100644 ---- a/pengine/test10/whitebox-imply-stop-on-fence.exp -+++ b/pengine/test10/whitebox-imply-stop-on-fence.exp -@@ -83,6 +83,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -@@ -149,6 +155,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -@@ -212,6 +224,12 @@ - - - -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/whitebox-migrate1.dot b/pengine/test10/whitebox-migrate1.dot -index ae88207..85e465f 100644 ---- a/pengine/test10/whitebox-migrate1.dot -+++ b/pengine/test10/whitebox-migrate1.dot -@@ -23,6 +23,7 @@ - "rhel7-node1_migrate_from_0 rhel7-node3" [ style=bold color="green" fontcolor="black"] - "rhel7-node1_migrate_to_0 rhel7-node2" -> "rhel7-node1_migrate_from_0 rhel7-node3" [ style = bold] - "rhel7-node1_migrate_to_0 rhel7-node2" [ style=bold color="green" fontcolor="black"] -+"rhel7-node1_monitor_0 rhel7-node3" -> "remote-rsc_migrate_to_0 rhel7-node2" [ style = bold] - "rhel7-node1_monitor_0 rhel7-node3" -> "rhel7-node1_migrate_to_0 rhel7-node2" [ style = bold] - "rhel7-node1_monitor_0 rhel7-node3" -> "rhel7-node1_start_0 rhel7-node3" [ style = bold] - "rhel7-node1_monitor_0 rhel7-node3" [ style=bold color="green" fontcolor="black"] -diff --git a/pengine/test10/whitebox-migrate1.exp b/pengine/test10/whitebox-migrate1.exp -index 81c35ec..48c2550 100644 ---- a/pengine/test10/whitebox-migrate1.exp -+++ b/pengine/test10/whitebox-migrate1.exp -@@ -89,7 +89,11 @@ - - - -- -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/whitebox-migrate1.summary b/pengine/test10/whitebox-migrate1.summary -index 58ef985..c45f360 100644 ---- a/pengine/test10/whitebox-migrate1.summary -+++ b/pengine/test10/whitebox-migrate1.summary -@@ -22,14 +22,14 @@ Transition Summary: - Executing cluster transition: - * Resource action: shooter1 stop on rhel7-node3 - * Resource action: FAKE3 stop on rhel7-node3 -- * Resource action: remote-rsc migrate_to on rhel7-node2 - * Resource action: rhel7-node1 monitor on rhel7-node3 - * Resource action: shooter1 start on rhel7-node2 - * Resource action: FAKE3 start on rhel7-node2 -- * Resource action: remote-rsc migrate_from on rhel7-node3 -- * Resource action: rhel7-node1 migrate_to on rhel7-node2 -+ * Resource action: remote-rsc migrate_to on rhel7-node2 - * Resource action: shooter1 monitor=60000 on rhel7-node2 - * Resource action: FAKE3 monitor=10000 on rhel7-node2 -+ * Resource action: remote-rsc migrate_from on rhel7-node3 -+ * Resource action: rhel7-node1 migrate_to on rhel7-node2 - * Resource action: rhel7-node1 migrate_from on rhel7-node3 - * Resource action: rhel7-node1 stop on rhel7-node2 - * Resource action: remote-rsc stop on rhel7-node2 -diff --git a/pengine/test10/whitebox-move.dot b/pengine/test10/whitebox-move.dot -index 418e63b..0442f43 100644 ---- a/pengine/test10/whitebox-move.dot -+++ b/pengine/test10/whitebox-move.dot -@@ -1,5 +1,6 @@ - digraph "g" { - "A_monitor_0 lxc2" -> "A_start_0 lxc1" [ style = bold] -+"A_monitor_0 lxc2" -> "lxc1_stop_0 18node1" [ style = bold] - "A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] - "A_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] - "A_start_0 lxc1" -> "A_monitor_10000 lxc1" [ style = bold] -diff --git a/pengine/test10/whitebox-move.exp b/pengine/test10/whitebox-move.exp -index 5d17dc6..9f8593c 100644 ---- a/pengine/test10/whitebox-move.exp -+++ b/pengine/test10/whitebox-move.exp -@@ -223,6 +223,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/whitebox-ms-ordering.dot b/pengine/test10/whitebox-ms-ordering.dot -index ba0dd30..59bdbaa 100644 ---- a/pengine/test10/whitebox-ms-ordering.dot -+++ b/pengine/test10/whitebox-ms-ordering.dot -@@ -46,10 +46,13 @@ - "lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] - "lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="orange"] - "lxc-ms_monitor_0 18node1" -> "lxc-ms-master_start_0" [ style = bold] -+"lxc-ms_monitor_0 18node1" -> "lxc-ms-master_stopped_0" [ style = bold] - "lxc-ms_monitor_0 18node1" [ style=bold color="green" fontcolor="black"] - "lxc-ms_monitor_0 18node2" -> "lxc-ms-master_start_0" [ style = bold] -+"lxc-ms_monitor_0 18node2" -> "lxc-ms-master_stopped_0" [ style = bold] - "lxc-ms_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] - "lxc-ms_monitor_0 18node3" -> "lxc-ms-master_start_0" [ style = bold] -+"lxc-ms_monitor_0 18node3" -> "lxc-ms-master_stopped_0" [ style = bold] - "lxc-ms_monitor_0 18node3" [ style=bold color="green" fontcolor="black"] - "lxc-ms_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"] - "lxc-ms_promote_0 lxc1" -> "lxc-ms-master_promoted_0" [ style = bold] -diff --git a/pengine/test10/whitebox-ms-ordering.exp b/pengine/test10/whitebox-ms-ordering.exp -index 80d8e5e..03761cf 100644 ---- a/pengine/test10/whitebox-ms-ordering.exp -+++ b/pengine/test10/whitebox-ms-ordering.exp -@@ -342,6 +342,15 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/pengine/test10/whitebox-orphaned.dot b/pengine/test10/whitebox-orphaned.dot -index 1511877..fdb1547 100644 ---- a/pengine/test10/whitebox-orphaned.dot -+++ b/pengine/test10/whitebox-orphaned.dot -@@ -1,6 +1,7 @@ - digraph "g" { - "A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] - "B_monitor_0 lxc2" -> "B_start_0 lxc2" [ style = bold] -+"B_monitor_0 lxc2" -> "lxc1_stop_0 18node2" [ style = bold] - "B_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] - "B_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"] - "B_start_0 lxc2" -> "B_monitor_10000 lxc2" [ style = bold] -diff --git a/pengine/test10/whitebox-orphaned.exp b/pengine/test10/whitebox-orphaned.exp -index 251aa49..ee7132a 100644 ---- a/pengine/test10/whitebox-orphaned.exp -+++ b/pengine/test10/whitebox-orphaned.exp -@@ -176,6 +176,9 @@ - - - -+ -+ -+ - - - -diff --git a/pengine/test10/whitebox-stop.dot b/pengine/test10/whitebox-stop.dot -index 304f134..8e03f9b 100644 ---- a/pengine/test10/whitebox-stop.dot -+++ b/pengine/test10/whitebox-stop.dot -@@ -1,6 +1,7 @@ - digraph "g" { - "A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] - "B_monitor_0 lxc2" -> "B_start_0 lxc2" [ style = bold] -+"B_monitor_0 lxc2" -> "lxc1_stop_0 18node2" [ style = bold] - "B_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] - "B_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"] - "B_start_0 lxc2" -> "B_monitor_10000 lxc2" [ style = bold] -diff --git a/pengine/test10/whitebox-stop.exp b/pengine/test10/whitebox-stop.exp -index 3640b03..81f55af 100644 ---- a/pengine/test10/whitebox-stop.exp -+++ b/pengine/test10/whitebox-stop.exp -@@ -125,6 +125,9 @@ - - - -+ -+ -+ - - - --- -1.8.3.1 - - -From e5b657fe11523c9b24a6f6098dbdb1bdba58003e Mon Sep 17 00:00:00 2001 -From: "Gao,Yan" -Date: Fri, 1 Mar 2019 17:32:50 +0100 -Subject: [PATCH 3/3] Test: scheduler: cl#5301 - respect order constraints when - relevant resources are being probed (new test) - ---- - pengine/regression.sh | 1 + - pengine/test10/order-first-probes.dot | 28 ++++++ - pengine/test10/order-first-probes.exp | 144 ++++++++++++++++++++++++++++++ - pengine/test10/order-first-probes.scores | 12 +++ - pengine/test10/order-first-probes.summary | 35 ++++++++ - pengine/test10/order-first-probes.xml | 96 ++++++++++++++++++++ - 6 files changed, 316 insertions(+) - create mode 100644 pengine/test10/order-first-probes.dot - create mode 100644 pengine/test10/order-first-probes.exp - create mode 100644 pengine/test10/order-first-probes.scores - create mode 100644 pengine/test10/order-first-probes.summary - create mode 100644 pengine/test10/order-first-probes.xml - -diff --git a/pengine/regression.sh b/pengine/regression.sh -index bb63a72..e25990d 100755 ---- a/pengine/regression.sh -+++ b/pengine/regression.sh -@@ -566,6 +566,7 @@ do_test honor_stonith_rsc_order2 "cl#5056- Honor order constraint, stonith clone - do_test honor_stonith_rsc_order3 "cl#5056- Honor order constraint, stonith clones with nested pure stonith group." - do_test honor_stonith_rsc_order4 "cl#5056- Honor order constraint, between two native stonith rscs." - do_test probe-timeout "cl#5099 - Default probe timeout" -+do_test order-first-probes "cl#5301 - respect order constraints when relevant resources are being probed" - - do_test concurrent-fencing "Allow performing fencing operations in parallel" - -diff --git a/pengine/test10/order-first-probes.dot b/pengine/test10/order-first-probes.dot -new file mode 100644 -index 0000000..1251318 ---- /dev/null -+++ b/pengine/test10/order-first-probes.dot -@@ -0,0 +1,28 @@ -+digraph "g" { -+"grpDummy_running_0" [ style=bold color="green" fontcolor="orange"] -+"grpDummy_start_0" -> "grpDummy_running_0" [ style = bold] -+"grpDummy_start_0" -> "prmDummy1_start_0 rh72-02" [ style = bold] -+"grpDummy_start_0" -> "prmDummy2_start_0 rh72-02" [ style = bold] -+"grpDummy_start_0" [ style=bold color="green" fontcolor="orange"] -+"grpDummy_stop_0" -> "grpDummy_stopped_0" [ style = bold] -+"grpDummy_stop_0" -> "prmDummy1_stop_0 rh72-01" [ style = bold] -+"grpDummy_stop_0" [ style=bold color="green" fontcolor="orange"] -+"grpDummy_stopped_0" -> "grpDummy_start_0" [ style = bold] -+"grpDummy_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"prmDummy1_monitor_10000 rh72-02" [ style=bold color="green" fontcolor="black"] -+"prmDummy1_start_0 rh72-02" -> "grpDummy_running_0" [ style = bold] -+"prmDummy1_start_0 rh72-02" -> "prmDummy1_monitor_10000 rh72-02" [ style = bold] -+"prmDummy1_start_0 rh72-02" -> "prmDummy2_start_0 rh72-02" [ style = bold] -+"prmDummy1_start_0 rh72-02" [ style=bold color="green" fontcolor="black"] -+"prmDummy1_stop_0 rh72-01" -> "grpDummy_stopped_0" [ style = bold] -+"prmDummy1_stop_0 rh72-01" -> "prmDummy1_start_0 rh72-02" [ style = bold] -+"prmDummy1_stop_0 rh72-01" [ style=bold color="green" fontcolor="black"] -+"prmDummy2_monitor_0 rh72-01" -> "grpDummy_stopped_0" [ style = bold] -+"prmDummy2_monitor_0 rh72-01" -> "prmDummy1_stop_0 rh72-01" [ style = bold] -+"prmDummy2_monitor_0 rh72-01" -> "prmDummy2_start_0 rh72-02" [ style = bold] -+"prmDummy2_monitor_0 rh72-01" [ style=bold color="green" fontcolor="black"] -+"prmDummy2_monitor_10000 rh72-02" [ style=bold color="green" fontcolor="black"] -+"prmDummy2_start_0 rh72-02" -> "grpDummy_running_0" [ style = bold] -+"prmDummy2_start_0 rh72-02" -> "prmDummy2_monitor_10000 rh72-02" [ style = bold] -+"prmDummy2_start_0 rh72-02" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/pengine/test10/order-first-probes.exp b/pengine/test10/order-first-probes.exp -new file mode 100644 -index 0000000..3ab8801 ---- /dev/null -+++ b/pengine/test10/order-first-probes.exp -@@ -0,0 +1,144 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/pengine/test10/order-first-probes.scores b/pengine/test10/order-first-probes.scores -new file mode 100644 -index 0000000..555caf4 ---- /dev/null -+++ b/pengine/test10/order-first-probes.scores -@@ -0,0 +1,12 @@ -+Allocation scores: -+Using the original execution date of: 2016-10-05 07:32:34Z -+group_color: grpDummy allocation score on rh72-01: 200 -+group_color: grpDummy allocation score on rh72-02: 100 -+group_color: prmDummy1 allocation score on rh72-01: INFINITY -+group_color: prmDummy1 allocation score on rh72-02: 100 -+group_color: prmDummy2 allocation score on rh72-01: 0 -+group_color: prmDummy2 allocation score on rh72-02: 0 -+native_color: prmDummy1 allocation score on rh72-01: -INFINITY -+native_color: prmDummy1 allocation score on rh72-02: 100 -+native_color: prmDummy2 allocation score on rh72-01: -INFINITY -+native_color: prmDummy2 allocation score on rh72-02: 0 -diff --git a/pengine/test10/order-first-probes.summary b/pengine/test10/order-first-probes.summary -new file mode 100644 -index 0000000..54b5bc1 ---- /dev/null -+++ b/pengine/test10/order-first-probes.summary -@@ -0,0 +1,35 @@ -+Using the original execution date of: 2016-10-05 07:32:34Z -+ -+Current cluster status: -+Node rh72-01 (3232238257): standby -+Online: [ rh72-02 ] -+ -+ Resource Group: grpDummy -+ prmDummy1 (ocf::pacemaker:Dummy1): Started rh72-01 -+ prmDummy2 (ocf::pacemaker:Dummy2): Stopped -+ -+Transition Summary: -+ * Move prmDummy1 ( rh72-01 -> rh72-02 ) -+ * Start prmDummy2 ( rh72-02 ) -+ -+Executing cluster transition: -+ * Pseudo action: grpDummy_stop_0 -+ * Resource action: prmDummy2 monitor on rh72-01 -+ * Resource action: prmDummy1 stop on rh72-01 -+ * Pseudo action: grpDummy_stopped_0 -+ * Pseudo action: grpDummy_start_0 -+ * Resource action: prmDummy1 start on rh72-02 -+ * Resource action: prmDummy2 start on rh72-02 -+ * Pseudo action: grpDummy_running_0 -+ * Resource action: prmDummy1 monitor=10000 on rh72-02 -+ * Resource action: prmDummy2 monitor=10000 on rh72-02 -+Using the original execution date of: 2016-10-05 07:32:34Z -+ -+Revised cluster status: -+Node rh72-01 (3232238257): standby -+Online: [ rh72-02 ] -+ -+ Resource Group: grpDummy -+ prmDummy1 (ocf::pacemaker:Dummy1): Started rh72-02 -+ prmDummy2 (ocf::pacemaker:Dummy2): Started rh72-02 -+ -diff --git a/pengine/test10/order-first-probes.xml b/pengine/test10/order-first-probes.xml -new file mode 100644 -index 0000000..6fe1be2 ---- /dev/null -+++ b/pengine/test10/order-first-probes.xml -@@ -0,0 +1,96 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ --- -1.8.3.1 - diff --git a/SOURCES/002-null-value.patch b/SOURCES/002-null-value.patch deleted file mode 100644 index 1377070..0000000 --- a/SOURCES/002-null-value.patch +++ /dev/null @@ -1,46 +0,0 @@ -From 0cfbb0797ba1788e131cf964ca53adcde8209e1f Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 21 Mar 2019 09:20:32 -0500 -Subject: [PATCH] Fix: libcrmcommon: pcmk_nvpair_t should handle NULL values - -Detected by static analysis; if pcmk_prepend_nvpair() were given a NULL value, -pcmk__new_nvpair() would try to strdup() it. Now, name is asserted to be -non-NULL, and NULL values are set directly. ---- - lib/common/nvpair.c | 10 +++++++--- - 1 file changed, 7 insertions(+), 3 deletions(-) - -diff --git a/lib/common/nvpair.c b/lib/common/nvpair.c -index 5d6853f..e9fec8a 100644 ---- a/lib/common/nvpair.c -+++ b/lib/common/nvpair.c -@@ -33,7 +33,7 @@ - * \internal - * \brief Allocate a new name/value pair - * -- * \param[in] name New name -+ * \param[in] name New name (required) - * \param[in] value New value - * - * \return Newly allocated name/value pair -@@ -43,11 +43,15 @@ - static pcmk_nvpair_t * - pcmk__new_nvpair(const char *name, const char *value) - { -- pcmk_nvpair_t *nvpair = calloc(1, sizeof(pcmk_nvpair_t)); -+ pcmk_nvpair_t *nvpair = NULL; - -+ CRM_ASSERT(name); -+ -+ nvpair = calloc(1, sizeof(pcmk_nvpair_t)); - CRM_ASSERT(nvpair); -+ - nvpair->name = strdup(name); -- nvpair->value = strdup(value); -+ nvpair->value = value? strdup(value) : NULL; - return nvpair; - } - --- -1.8.3.1 - diff --git a/SOURCES/003-fence-output.patch b/SOURCES/003-fence-output.patch deleted file mode 100644 index f174e6f..0000000 --- a/SOURCES/003-fence-output.patch +++ /dev/null @@ -1,860 +0,0 @@ -From 06ef8406cf224af6b94dc4672c9b6caa15133f89 Mon Sep 17 00:00:00 2001 -From: Klaus Wenninger -Date: Thu, 14 Feb 2019 13:27:46 +0100 -Subject: [PATCH] use common service interface for fence-agents and RAs - ---- - include/crm/services.h | 5 +- - lib/Makefile.am | 3 +- - lib/fencing/Makefile.am | 1 + - lib/fencing/st_client.c | 459 +++++++++------------------------------- - lib/services/services_linux.c | 93 ++++++++ - lib/services/services_private.h | 2 + - 6 files changed, 203 insertions(+), 360 deletions(-) - -diff --git a/include/crm/services.h b/include/crm/services.h -index 0186e66..eddafc3 100644 ---- a/include/crm/services.h -+++ b/include/crm/services.h -@@ -170,7 +170,10 @@ typedef struct svc_action_s { - char *agent; - - int timeout; -- GHashTable *params; /* used by OCF agents and alert agents */ -+ GHashTable *params; /* used for setting up environment for ocf-ra & -+ alert agents -+ and to be sent via stdin for fence-agents -+ */ - - int rc; - int pid; -diff --git a/lib/Makefile.am b/lib/Makefile.am -index 5563819..d73bf2e 100644 ---- a/lib/Makefile.am -+++ b/lib/Makefile.am -@@ -39,11 +39,10 @@ clean-local: - rm -f *.pc - - ## Subdirectories... --SUBDIRS = gnu common pengine transition cib fencing services lrmd cluster -+SUBDIRS = gnu common pengine transition cib services fencing lrmd cluster - DIST_SUBDIRS = $(SUBDIRS) ais - - if BUILD_CS_PLUGIN - SUBDIRS += ais - endif - -- -diff --git a/lib/fencing/Makefile.am b/lib/fencing/Makefile.am -index c3f4ea2..e447627 100644 ---- a/lib/fencing/Makefile.am -+++ b/lib/fencing/Makefile.am -@@ -15,6 +15,7 @@ libstonithd_la_CFLAGS = $(CFLAGS_HARDENED_LIB) - libstonithd_la_LDFLAGS += $(LDFLAGS_HARDENED_LIB) - - libstonithd_la_LIBADD = $(top_builddir)/lib/common/libcrmcommon.la -+libstonithd_la_LIBADD += $(top_builddir)/lib/services/libcrmservice.la - - libstonithd_la_SOURCES = st_client.c st_rhcs.c - if BUILD_LHA_SUPPORT -diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c -index 3898d45..1c56cf4 100644 ---- a/lib/fencing/st_client.c -+++ b/lib/fencing/st_client.c -@@ -22,6 +22,7 @@ - #include - #include - #include -+#include - - #include - #include -@@ -48,23 +49,18 @@ struct stonith_action_s { - char *agent; - char *action; - char *victim; -- char *args; -+ GHashTable *args; - int timeout; - int async; - void *userdata; - void (*done_cb) (GPid pid, gint status, const char *output, gpointer user_data); - -- /*! internal async track data */ -- int fd_stdout; -- int fd_stderr; -- int last_timeout_signo; -+ svc_action_t *svc_action; - - /*! internal timing information */ - time_t initial_start_time; - int tries; - int remaining_timeout; -- guint timer_sigterm; -- guint timer_sigkill; - int max_retries; - - /* device output data */ -@@ -448,13 +444,11 @@ stonith_api_register_level(stonith_t * st, int options, const char *node, int le - } - - static void --append_arg(const char *key, const char *value, char **args) -+append_arg(const char *key, const char *value, GHashTable **args) - { -- int len = 3; /* =, \n, \0 */ -- int last = 0; -- - CRM_CHECK(key != NULL, return); - CRM_CHECK(value != NULL, return); -+ CRM_CHECK(args != NULL, return); - - if (strstr(key, "pcmk_")) { - return; -@@ -464,15 +458,13 @@ append_arg(const char *key, const char *value, char **args) - return; - } - -- len += strlen(key); -- len += strlen(value); -- if (*args != NULL) { -- last = strlen(*args); -+ if (!*args) { -+ *args = crm_str_table_new(); - } - -- *args = realloc_safe(*args, last + len); -+ CRM_CHECK(*args != NULL, return); - crm_trace("Appending: %s=%s", key, value); -- sprintf((*args) + last, "%s=%s\n", key, value); -+ g_hash_table_replace(*args, strdup(key), strdup(value)); - } - - static void -@@ -489,7 +481,7 @@ append_config_arg(gpointer key, gpointer value, gpointer user_data) - } - - static void --append_host_specific_args(const char *victim, const char *map, GHashTable * params, char **arg_list) -+append_host_specific_args(const char *victim, const char *map, GHashTable * params, GHashTable **args) - { - char *name = NULL; - int last = 0, lpc = 0, max = 0; -@@ -497,7 +489,7 @@ append_host_specific_args(const char *victim, const char *map, GHashTable * para - if (map == NULL) { - /* The best default there is for now... */ - crm_debug("Using default arg map: port=uname"); -- append_arg("port", victim, arg_list); -+ append_arg("port", victim, args); - return; - } - -@@ -540,7 +532,7 @@ append_host_specific_args(const char *victim, const char *map, GHashTable * para - - if (value) { - crm_debug("Setting '%s'='%s' (%s) for %s", name, value, param, victim); -- append_arg(name, value, arg_list); -+ append_arg(name, value, args); - - } else { - crm_err("No node attribute '%s' for '%s'", name, victim); -@@ -560,12 +552,12 @@ append_host_specific_args(const char *victim, const char *map, GHashTable * para - free(name); - } - --static char * -+static GHashTable * - make_args(const char *agent, const char *action, const char *victim, uint32_t victim_nodeid, GHashTable * device_args, - GHashTable * port_map) - { - char buffer[512]; -- char *arg_list = NULL; -+ GHashTable *arg_list = NULL; - const char *value = NULL; - - CRM_CHECK(action != NULL, return NULL); -@@ -653,66 +645,6 @@ make_args(const char *agent, const char *action, const char *victim, uint32_t vi - return arg_list; - } - --static gboolean --st_child_term(gpointer data) --{ -- int rc = 0; -- stonith_action_t *track = data; -- -- crm_info("Child %d timed out, sending SIGTERM", track->pid); -- track->timer_sigterm = 0; -- track->last_timeout_signo = SIGTERM; -- rc = kill(-track->pid, SIGTERM); -- if (rc < 0) { -- crm_perror(LOG_ERR, "Couldn't send SIGTERM to %d", track->pid); -- } -- return FALSE; --} -- --static gboolean --st_child_kill(gpointer data) --{ -- int rc = 0; -- stonith_action_t *track = data; -- -- crm_info("Child %d timed out, sending SIGKILL", track->pid); -- track->timer_sigkill = 0; -- track->last_timeout_signo = SIGKILL; -- rc = kill(-track->pid, SIGKILL); -- if (rc < 0) { -- crm_perror(LOG_ERR, "Couldn't send SIGKILL to %d", track->pid); -- } -- return FALSE; --} -- --static void --stonith_action_clear_tracking_data(stonith_action_t * action) --{ -- if (action->timer_sigterm > 0) { -- g_source_remove(action->timer_sigterm); -- action->timer_sigterm = 0; -- } -- if (action->timer_sigkill > 0) { -- g_source_remove(action->timer_sigkill); -- action->timer_sigkill = 0; -- } -- if (action->fd_stdout) { -- close(action->fd_stdout); -- action->fd_stdout = 0; -- } -- if (action->fd_stderr) { -- close(action->fd_stderr); -- action->fd_stderr = 0; -- } -- free(action->output); -- action->output = NULL; -- free(action->error); -- action->error = NULL; -- action->rc = 0; -- action->pid = 0; -- action->last_timeout_signo = 0; --} -- - /*! - * \internal - * \brief Free all memory used by a stonith action -@@ -723,11 +655,17 @@ void - stonith__destroy_action(stonith_action_t *action) - { - if (action) { -- stonith_action_clear_tracking_data(action); - free(action->agent); -- free(action->args); -+ if (action->args) { -+ g_hash_table_destroy(action->args); -+ } - free(action->action); - free(action->victim); -+ if (action->svc_action) { -+ services_action_free(action->svc_action); -+ } -+ free(action->output); -+ free(action->error); - free(action); - } - } -@@ -809,38 +747,6 @@ stonith_action_create(const char *agent, - return action; - } - --#define READ_MAX 500 --static char * --read_output(int fd) --{ -- char buffer[READ_MAX]; -- char *output = NULL; -- int len = 0; -- int more = 0; -- -- if (!fd) { -- return NULL; -- } -- -- do { -- errno = 0; -- memset(&buffer, 0, READ_MAX); -- more = read(fd, buffer, READ_MAX - 1); -- -- if (more > 0) { -- buffer[more] = 0; /* Make sure it's nul-terminated for logging -- * 'more' is always less than our buffer size -- */ -- output = realloc_safe(output, len + more + 1); -- snprintf(output + len, more + 1, "%s", buffer); -- len += more; -- } -- -- } while (more == (READ_MAX - 1) || (more < 0 && errno == EINTR)); -- -- return output; --} -- - static gboolean - update_remaining_timeout(stonith_action_t * action) - { -@@ -860,58 +766,51 @@ update_remaining_timeout(stonith_action_t * action) - return action->remaining_timeout ? TRUE : FALSE; - } - --static void --stonith_action_async_done(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode) --{ -- stonith_action_t *action = mainloop_child_userdata(p); -- -- if (action->timer_sigterm > 0) { -- g_source_remove(action->timer_sigterm); -- action->timer_sigterm = 0; -- } -- if (action->timer_sigkill > 0) { -- g_source_remove(action->timer_sigkill); -- action->timer_sigkill = 0; -- } -+static int -+svc_action_to_errno(svc_action_t *svc_action) { -+ int rv = pcmk_ok; - -- action->output = read_output(action->fd_stdout); -- action->error = read_output(action->fd_stderr); -+ if (svc_action->rc > 0) { -+ /* Try to provide a useful error code based on the fence agent's -+ * error output. -+ */ -+ if (svc_action->rc == PCMK_OCF_TIMEOUT) { -+ rv = -ETIME; - -- if (action->last_timeout_signo) { -- action->rc = -ETIME; -- crm_notice("Child process %d performing action '%s' timed out with signal %d", -- pid, action->action, action->last_timeout_signo); -+ } else if (svc_action->stderr_data == NULL) { -+ rv = -ENODATA; - -- } else if (signo) { -- action->rc = -ECONNABORTED; -- crm_notice("Child process %d performing action '%s' timed out with signal %d", -- pid, action->action, signo); -+ } else if (strstr(svc_action->stderr_data, "imed out")) { -+ /* Some agents have their own internal timeouts */ -+ rv = -ETIME; - -- } else { -- crm_debug("Child process %d performing action '%s' exited with rc %d", -- pid, action->action, exitcode); -- if (exitcode > 0) { -- /* Try to provide a useful error code based on the fence agent's -- * error output. -- */ -- if (action->error == NULL) { -- exitcode = -ENODATA; -- -- } else if (strstr(action->error, "imed out")) { -- /* Some agents have their own internal timeouts */ -- exitcode = -ETIMEDOUT; -- -- } else if (strstr(action->error, "Unrecognised action")) { -- exitcode = -EOPNOTSUPP; -+ } else if (strstr(svc_action->stderr_data, "Unrecognised action")) { -+ rv = -EOPNOTSUPP; - -- } else { -- exitcode = -pcmk_err_generic; -- } -+ } else { -+ rv = -pcmk_err_generic; - } -- action->rc = exitcode; - } -+ return rv; -+} - -- log_action(action, pid); -+static void -+stonith_action_async_done(svc_action_t *svc_action) -+{ -+ stonith_action_t *action = (stonith_action_t *) svc_action->cb_data; -+ -+ action->rc = svc_action_to_errno(svc_action); -+ action->output = svc_action->stdout_data; -+ svc_action->stdout_data = NULL; -+ action->error = svc_action->stderr_data; -+ svc_action->stderr_data = NULL; -+ -+ svc_action->params = NULL; -+ -+ crm_debug("Child process %d performing action '%s' exited with rc %d", -+ action->pid, action->action, svc_action->rc); -+ -+ log_action(action, action->pid); - - if (action->rc != pcmk_ok && update_remaining_timeout(action)) { - int rc = internal_stonith_action_execute(action); -@@ -921,28 +820,21 @@ stonith_action_async_done(mainloop_child_t * p, pid_t pid, int core, int signo, - } - - if (action->done_cb) { -- action->done_cb(pid, action->rc, action->output, action->userdata); -+ action->done_cb(action->pid, action->rc, action->output, action->userdata); - } - -+ action->svc_action = NULL; // don't remove our caller - stonith__destroy_action(action); - } - - static int - internal_stonith_action_execute(stonith_action_t * action) - { -- int pid, status = 0, len, rc = -EPROTO; -- int ret; -- int total = 0; -- int p_read_fd, p_write_fd; /* parent read/write file descriptors */ -- int c_read_fd, c_write_fd; /* child read/write file descriptors */ -- int c_stderr_fd, p_stderr_fd; /* parent/child side file descriptors for stderr */ -- int fd1[2]; -- int fd2[2]; -- int fd3[2]; -+ int rc = -EPROTO; - int is_retry = 0; -- -- /* clear any previous tracking data */ -- stonith_action_clear_tracking_data(action); -+ svc_action_t *svc_action = NULL; -+ static int stonith_sequence = 0; -+ char *buffer = NULL; - - if (!action->tries) { - action->initial_start_time = time(NULL); -@@ -955,207 +847,60 @@ internal_stonith_action_execute(stonith_action_t * action) - is_retry = 1; - } - -- c_read_fd = c_write_fd = p_read_fd = p_write_fd = c_stderr_fd = p_stderr_fd = -1; -- - if (action->args == NULL || action->agent == NULL) - goto fail; -- len = strlen(action->args); -- -- if (pipe(fd1)) -- goto fail; -- p_read_fd = fd1[0]; -- c_write_fd = fd1[1]; -- -- if (pipe(fd2)) -- goto fail; -- c_read_fd = fd2[0]; -- p_write_fd = fd2[1]; -- -- if (pipe(fd3)) -- goto fail; -- p_stderr_fd = fd3[0]; -- c_stderr_fd = fd3[1]; -- -- crm_debug("forking"); -- pid = fork(); -- if (pid < 0) { -- rc = -ECHILD; -- goto fail; -- } -- -- if (!pid) { -- /* child */ -- setpgid(0, 0); -- -- close(1); -- /* coverity[leaked_handle] False positive */ -- if (dup(c_write_fd) < 0) -- goto fail; -- close(2); -- /* coverity[leaked_handle] False positive */ -- if (dup(c_stderr_fd) < 0) -- goto fail; -- close(0); -- /* coverity[leaked_handle] False positive */ -- if (dup(c_read_fd) < 0) -- goto fail; -- -- /* keep c_stderr_fd open so parent can report all errors. */ -- /* keep c_write_fd open so hostlist can be sent to parent. */ -- close(c_read_fd); -- close(p_read_fd); -- close(p_write_fd); -- close(p_stderr_fd); -- -- /* keep retries from executing out of control */ -- if (is_retry) { -- sleep(1); -- } -- execlp(action->agent, action->agent, NULL); -- exit(EXIT_FAILURE); -- } - -- /* parent */ -- action->pid = pid; -- ret = crm_set_nonblocking(p_read_fd); -- if (ret < 0) { -- crm_notice("Could not set output of %s to be non-blocking: %s " -- CRM_XS " rc=%d", -- action->agent, pcmk_strerror(rc), rc); -+ buffer = crm_strdup_printf(RH_STONITH_DIR "/%s", basename(action->agent)); -+ svc_action = services_action_create_generic(buffer, NULL); -+ free(buffer); -+ svc_action->timeout = 1000 * action->remaining_timeout; -+ svc_action->standard = strdup(PCMK_RESOURCE_CLASS_STONITH); -+ svc_action->id = crm_strdup_printf("%s_%s_%d", basename(action->agent), -+ action->action, action->tries); -+ svc_action->agent = strdup(action->agent); -+ svc_action->sequence = stonith_sequence++; -+ svc_action->params = action->args; -+ svc_action->cb_data = (void *) action; -+ -+ /* keep retries from executing out of control and free previous results */ -+ if (is_retry) { -+ free(action->output); -+ action->output = NULL; -+ free(action->error); -+ action->error = NULL; -+ sleep(1); - } -- ret = crm_set_nonblocking(p_stderr_fd); -- if (ret < 0) { -- crm_notice("Could not set error output of %s to be non-blocking: %s " -- CRM_XS " rc=%d", -- action->agent, pcmk_strerror(rc), rc); -- } -- -- errno = 0; -- do { -- crm_debug("sending args"); -- ret = write(p_write_fd, action->args + total, len - total); -- if (ret > 0) { -- total += ret; -- } -- -- } while (errno == EINTR && total < len); -- -- if (total != len) { -- crm_perror(LOG_ERR, "Sent %d not %d bytes", total, len); -- if (ret >= 0) { -- rc = -ECOMM; -- } -- goto fail; -- } -- -- close(p_write_fd); p_write_fd = -1; - -- /* async */ - if (action->async) { -- action->fd_stdout = p_read_fd; -- action->fd_stderr = p_stderr_fd; -- mainloop_child_add(pid, 0/* Move the timeout here? */, action->action, action, stonith_action_async_done); -- crm_trace("Op: %s on %s, pid: %d, timeout: %ds", action->action, action->agent, pid, -- action->remaining_timeout); -- action->last_timeout_signo = 0; -- if (action->remaining_timeout) { -- action->timer_sigterm = -- g_timeout_add(1000 * action->remaining_timeout, st_child_term, action); -- action->timer_sigkill = -- g_timeout_add(1000 * (action->remaining_timeout + 5), st_child_kill, action); -+ /* async */ -+ if(services_action_async(svc_action, &stonith_action_async_done) == FALSE) { -+ services_action_free(svc_action); -+ svc_action = NULL; - } else { -- crm_err("No timeout set for stonith operation %s with device %s", -- action->action, action->agent); -+ action->pid = svc_action->pid; -+ action->svc_action = svc_action; -+ rc = 0; - } - -- close(c_write_fd); -- close(c_read_fd); -- close(c_stderr_fd); -- return 0; -- - } else { - /* sync */ -- int timeout = action->remaining_timeout + 1; -- pid_t p = 0; -- -- while (action->remaining_timeout < 0 || timeout > 0) { -- p = waitpid(pid, &status, WNOHANG); -- if (p > 0) { -- break; -- } -- sleep(1); -- timeout--; -- } -- -- if (timeout == 0) { -- int killrc = kill(-pid, SIGKILL); -- -- if (killrc && errno != ESRCH) { -- crm_err("kill(%d, KILL) failed: %s (%d)", pid, pcmk_strerror(errno), errno); -- } -- /* -- * From sigprocmask(2): -- * It is not possible to block SIGKILL or SIGSTOP. Attempts to do so are silently ignored. -- * -- * This makes it safe to skip WNOHANG here -- */ -- p = waitpid(pid, &status, 0); -- } -- -- if (p <= 0) { -- crm_perror(LOG_ERR, "waitpid(%d)", pid); -- -- } else if (p != pid) { -- crm_err("Waited for %d, got %d", pid, p); -- } -- -- action->output = read_output(p_read_fd); -- action->error = read_output(p_stderr_fd); -- -- action->rc = -ECONNABORTED; -- -- log_action(action, pid); -- -- rc = action->rc; -- if (timeout == 0) { -- action->rc = -ETIME; -- } else if (WIFEXITED(status)) { -- crm_debug("result = %d", WEXITSTATUS(status)); -- action->rc = -WEXITSTATUS(status); -+ if (services_action_sync(svc_action)) { - rc = 0; -- -- } else if (WIFSIGNALED(status)) { -- crm_err("call %s for %s exited due to signal %d", action->action, action->agent, -- WTERMSIG(status)); -- -+ action->rc = svc_action_to_errno(svc_action); -+ action->output = svc_action->stdout_data; -+ svc_action->stdout_data = NULL; -+ action->error = svc_action->stderr_data; -+ svc_action->stderr_data = NULL; - } else { -- crm_err("call %s for %s returned unexpected status %#x", -- action->action, action->agent, status); -+ action->rc = -ECONNABORTED; -+ rc = action->rc; - } -- } - -- fail: -- -- if (p_read_fd >= 0) { -- close(p_read_fd); -- } -- if (p_write_fd >= 0) { -- close(p_write_fd); -- } -- if (p_stderr_fd >= 0) { -- close(p_stderr_fd); -- } -- -- if (c_read_fd >= 0) { -- close(c_read_fd); -- } -- if (c_write_fd >= 0) { -- close(c_write_fd); -- } -- if (c_stderr_fd >= 0) { -- close(c_stderr_fd); -+ svc_action->params = NULL; -+ services_action_free(svc_action); - } - -+ fail: - return rc; - } - -diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c -index a413484..d79c16d 100644 ---- a/lib/services/services_linux.c -+++ b/lib/services/services_linux.c -@@ -195,6 +195,39 @@ add_action_env_vars(const svc_action_t *op) - } - } - -+static void -+pipe_in_single_parameter(gpointer key, gpointer value, gpointer user_data) -+{ -+ svc_action_t *op = user_data; -+ char *buffer = crm_strdup_printf("%s=%s\n", (char *)key, (char *) value); -+ int ret, total = 0, len = strlen(buffer); -+ -+ do { -+ errno = 0; -+ ret = write(op->opaque->stdin_fd, buffer + total, len - total); -+ if (ret > 0) { -+ total += ret; -+ } -+ -+ } while ((errno == EINTR) && (total < len)); -+ free(buffer); -+} -+ -+/*! -+ * \internal -+ * \brief Pipe parameters in via stdin for action -+ * -+ * \param[in] op Action to use -+ */ -+static void -+pipe_in_action_stdin_parameters(const svc_action_t *op) -+{ -+ crm_debug("sending args"); -+ if (op->params) { -+ g_hash_table_foreach(op->params, pipe_in_single_parameter, (gpointer) op); -+ } -+} -+ - gboolean - recurring_action_timer(gpointer data) - { -@@ -284,6 +317,10 @@ operation_finished(mainloop_child_t * p, pid_t pid, int core, int signo, int exi - op->opaque->stdout_gsource = NULL; - } - -+ if (op->opaque->stdin_fd >= 0) { -+ close(op->opaque->stdin_fd); -+ } -+ - if (signo) { - if (mainloop_child_timeout(p)) { - crm_warn("%s - timed out after %dms", prefix, op->timeout); -@@ -605,6 +642,9 @@ action_synced_wait(svc_action_t * op, sigset_t *mask) - - close(op->opaque->stdout_fd); - close(op->opaque->stderr_fd); -+ if (op->opaque->stdin_fd >= 0) { -+ close(op->opaque->stdin_fd); -+ } - - #ifdef HAVE_SYS_SIGNALFD_H - close(sfd); -@@ -618,6 +658,7 @@ services_os_action_execute(svc_action_t * op) - { - int stdout_fd[2]; - int stderr_fd[2]; -+ int stdin_fd[2] = {-1, -1}; - int rc; - struct stat st; - sigset_t *pmask; -@@ -683,6 +724,25 @@ services_os_action_execute(svc_action_t * op) - return FALSE; - } - -+ if (safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_STONITH)) { -+ if (pipe(stdin_fd) < 0) { -+ rc = errno; -+ -+ close(stdout_fd[0]); -+ close(stdout_fd[1]); -+ close(stderr_fd[0]); -+ close(stderr_fd[1]); -+ -+ crm_err("pipe(stdin_fd) failed. '%s': %s (%d)", op->opaque->exec, pcmk_strerror(rc), rc); -+ -+ services_handle_exec_error(op, rc); -+ if (!op->synchronous) { -+ return operation_finalize(op); -+ } -+ return FALSE; -+ } -+ } -+ - if (op->synchronous) { - #ifdef HAVE_SYS_SIGNALFD_H - sigemptyset(&mask); -@@ -730,6 +790,10 @@ services_os_action_execute(svc_action_t * op) - close(stdout_fd[1]); - close(stderr_fd[0]); - close(stderr_fd[1]); -+ if (stdin_fd[0] >= 0) { -+ close(stdin_fd[0]); -+ close(stdin_fd[1]); -+ } - - crm_err("Could not execute '%s': %s (%d)", op->opaque->exec, pcmk_strerror(rc), rc); - services_handle_exec_error(op, rc); -@@ -743,6 +807,9 @@ services_os_action_execute(svc_action_t * op) - case 0: /* Child */ - close(stdout_fd[0]); - close(stderr_fd[0]); -+ if (stdin_fd[1] >= 0) { -+ close(stdin_fd[1]); -+ } - if (STDOUT_FILENO != stdout_fd[1]) { - if (dup2(stdout_fd[1], STDOUT_FILENO) != STDOUT_FILENO) { - crm_err("dup2() failed (stdout)"); -@@ -755,6 +822,13 @@ services_os_action_execute(svc_action_t * op) - } - close(stderr_fd[1]); - } -+ if ((stdin_fd[0] >= 0) && -+ (STDIN_FILENO != stdin_fd[0])) { -+ if (dup2(stdin_fd[0], STDIN_FILENO) != STDIN_FILENO) { -+ crm_err("dup2() failed (stdin)"); -+ } -+ close(stdin_fd[0]); -+ } - - if (op->synchronous) { - sigchld_cleanup(); -@@ -767,6 +841,9 @@ services_os_action_execute(svc_action_t * op) - /* Only the parent reaches here */ - close(stdout_fd[1]); - close(stderr_fd[1]); -+ if (stdin_fd[0] >= 0) { -+ close(stdin_fd[0]); -+ } - - op->opaque->stdout_fd = stdout_fd[0]; - rc = crm_set_nonblocking(op->opaque->stdout_fd); -@@ -784,6 +861,22 @@ services_os_action_execute(svc_action_t * op) - pcmk_strerror(rc), rc); - } - -+ op->opaque->stdin_fd = stdin_fd[1]; -+ if (op->opaque->stdin_fd >= 0) { -+ // using buffer behind non-blocking-fd here - that could be improved -+ // as long as no other standard uses stdin_fd assume stonith -+ rc = crm_set_nonblocking(op->opaque->stdin_fd); -+ if (rc < 0) { -+ crm_warn("Could not set child input non-blocking: %s " -+ CRM_XS " fd=%d,rc=%d", -+ pcmk_strerror(rc), op->opaque->stdin_fd, rc); -+ } -+ pipe_in_action_stdin_parameters(op); -+ // as long as we are handling parameters directly in here just close -+ close(op->opaque->stdin_fd); -+ op->opaque->stdin_fd = -1; -+ } -+ - if (op->synchronous) { - action_synced_wait(op, pmask); - sigchld_cleanup(); -diff --git a/lib/services/services_private.h b/lib/services/services_private.h -index 0676c6f..9735da7 100644 ---- a/lib/services/services_private.h -+++ b/lib/services/services_private.h -@@ -42,6 +42,8 @@ struct svc_action_private_s { - - int stdout_fd; - mainloop_io_t *stdout_gsource; -+ -+ int stdin_fd; - #if SUPPORT_DBUS - DBusPendingCall* pending; - unsigned timerid; --- -1.8.3.1 - diff --git a/SOURCES/004-group-ordering.patch b/SOURCES/004-group-ordering.patch deleted file mode 100644 index 7428744..0000000 --- a/SOURCES/004-group-ordering.patch +++ /dev/null @@ -1,602 +0,0 @@ -From d14fb0110208f270491c696bea0072300db2a947 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 29 Mar 2019 12:37:13 -0500 -Subject: [PATCH 1/6] Refactor: scheduler: functionize handling of restart - ordering - -for readability ---- - pengine/native.c | 97 +++++++++++++++++++++++++++++++++++--------------------- - 1 file changed, 60 insertions(+), 37 deletions(-) - -diff --git a/pengine/native.c b/pengine/native.c -index 2f8c011..653a93a 100644 ---- a/pengine/native.c -+++ b/pengine/native.c -@@ -1942,6 +1942,65 @@ native_action_flags(action_t * action, node_t * node) - return action->flags; - } - -+static inline bool -+is_primitive_action(pe_action_t *action) -+{ -+ return action && action->rsc && (action->rsc->variant == pe_native); -+} -+ -+/*! -+ * \internal -+ * \brief Set action bits appropriately when pe_restart_order is used -+ * -+ * \param[in] first 'First' action in an ordering with pe_restart_order -+ * \param[in] then 'Then' action in an ordering with pe_restart_order -+ * \param[in] filter What ordering flags to care about -+ * -+ * \note pe_restart_order is set for "stop resource before starting it" and -+ * "stop later group member before stopping earlier group member" -+ */ -+static void -+handle_restart_ordering(pe_action_t *first, pe_action_t *then, -+ enum pe_action_flags filter) -+{ -+ const char *reason = NULL; -+ -+ CRM_ASSERT(is_primitive_action(first)); -+ CRM_ASSERT(is_primitive_action(then)); -+ -+ if ((filter & pe_action_runnable) -+ && (then->flags & pe_action_runnable) == 0 -+ && (then->rsc->flags & pe_rsc_managed)) { -+ reason = "shutdown"; -+ } -+ -+ if ((filter & pe_action_optional) && (then->flags & pe_action_optional) == 0) { -+ reason = "recover"; -+ } -+ -+ if (reason && is_set(first->flags, pe_action_optional)) { -+ if (is_set(first->flags, pe_action_runnable) -+ || is_not_set(then->flags, pe_action_optional)) { -+ pe_rsc_trace(first->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid); -+ pe_action_implies(first, then, pe_action_optional); -+ } -+ } -+ -+ if (reason && is_not_set(first->flags, pe_action_optional) -+ && is_not_set(first->flags, pe_action_runnable)) { -+ pe_rsc_trace(then->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid); -+ pe_action_implies(then, first, pe_action_runnable); -+ } -+ -+ if (reason && -+ is_not_set(first->flags, pe_action_optional) && -+ is_set(first->flags, pe_action_migrate_runnable) && -+ is_not_set(then->flags, pe_action_migrate_runnable)) { -+ -+ pe_action_implies(first, then, pe_action_migrate_runnable); -+ } -+} -+ - enum pe_graph_flags - native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags, - enum pe_action_flags filter, enum pe_ordering type) -@@ -2069,43 +2128,7 @@ native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_ - } - - if (is_set(type, pe_order_restart)) { -- const char *reason = NULL; -- -- CRM_ASSERT(first->rsc && first->rsc->variant == pe_native); -- CRM_ASSERT(then->rsc && then->rsc->variant == pe_native); -- -- if ((filter & pe_action_runnable) -- && (then->flags & pe_action_runnable) == 0 -- && (then->rsc->flags & pe_rsc_managed)) { -- reason = "shutdown"; -- } -- -- if ((filter & pe_action_optional) && (then->flags & pe_action_optional) == 0) { -- reason = "recover"; -- } -- -- if (reason && is_set(first->flags, pe_action_optional)) { -- if (is_set(first->flags, pe_action_runnable) -- || is_not_set(then->flags, pe_action_optional)) { -- pe_rsc_trace(first->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid); -- pe_action_implies(first, then, pe_action_optional); -- } -- } -- -- if (reason && is_not_set(first->flags, pe_action_optional) -- && is_not_set(first->flags, pe_action_runnable)) { -- pe_rsc_trace(then->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid); -- pe_action_implies(then, first, pe_action_runnable); -- } -- -- if (reason && -- is_not_set(first->flags, pe_action_optional) && -- is_set(first->flags, pe_action_migrate_runnable) && -- is_not_set(then->flags, pe_action_migrate_runnable)) { -- -- pe_action_implies(first, then, pe_action_migrate_runnable); -- } -- -+ handle_restart_ordering(first, then, filter); - } - - if (then_flags != then->flags) { --- -1.8.3.1 - - -From dda6a0eb480c8a3079b5c15176e27ed62f98a6ac Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 29 Mar 2019 17:46:44 -0500 -Subject: [PATCH 2/6] Refactor: scheduler: use is_set()/is_not_set() in restart - ordering - -instead of direct bit comparisons, for readability and consistency ---- - pengine/native.c | 9 +++++---- - 1 file changed, 5 insertions(+), 4 deletions(-) - -diff --git a/pengine/native.c b/pengine/native.c -index 653a93a..f43d3cf 100644 ---- a/pengine/native.c -+++ b/pengine/native.c -@@ -1968,13 +1968,14 @@ handle_restart_ordering(pe_action_t *first, pe_action_t *then, - CRM_ASSERT(is_primitive_action(first)); - CRM_ASSERT(is_primitive_action(then)); - -- if ((filter & pe_action_runnable) -- && (then->flags & pe_action_runnable) == 0 -- && (then->rsc->flags & pe_rsc_managed)) { -+ if (is_set(filter, pe_action_runnable) -+ && is_not_set(then->flags, pe_action_runnable) -+ && is_set(then->rsc->flags, pe_rsc_managed)) { - reason = "shutdown"; - } - -- if ((filter & pe_action_optional) && (then->flags & pe_action_optional) == 0) { -+ if (is_set(filter, pe_action_optional) -+ && is_not_set(then->flags, pe_action_optional)) { - reason = "recover"; - } - --- -1.8.3.1 - - -From 69951375e4af9d1d1152980afddc94bb5881af5a Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 29 Mar 2019 17:49:10 -0500 -Subject: [PATCH 3/6] Log: scheduler: improve restart ordering trace logs - ---- - pengine/native.c | 28 ++++++++++++++++++---------- - 1 file changed, 18 insertions(+), 10 deletions(-) - -diff --git a/pengine/native.c b/pengine/native.c -index f43d3cf..dfcc910 100644 ---- a/pengine/native.c -+++ b/pengine/native.c -@@ -1968,33 +1968,41 @@ handle_restart_ordering(pe_action_t *first, pe_action_t *then, - CRM_ASSERT(is_primitive_action(first)); - CRM_ASSERT(is_primitive_action(then)); - -+ // We need to update the action in two cases: -+ -+ // ... if 'then' is required -+ if (is_set(filter, pe_action_optional) -+ && is_not_set(then->flags, pe_action_optional)) { -+ reason = "restart"; -+ } -+ -+ // ... if 'then' is managed but unrunnable - if (is_set(filter, pe_action_runnable) - && is_not_set(then->flags, pe_action_runnable) - && is_set(then->rsc->flags, pe_rsc_managed)) { -- reason = "shutdown"; -+ reason = "stop"; - } - -- if (is_set(filter, pe_action_optional) -- && is_not_set(then->flags, pe_action_optional)) { -- reason = "recover"; -+ if (reason == NULL) { -+ return; - } - -- if (reason && is_set(first->flags, pe_action_optional)) { -+ pe_rsc_trace(first->rsc, "Handling %s -> %s for %s", -+ first->uuid, then->uuid, reason); -+ -+ if (is_set(first->flags, pe_action_optional)) { - if (is_set(first->flags, pe_action_runnable) - || is_not_set(then->flags, pe_action_optional)) { -- pe_rsc_trace(first->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid); - pe_action_implies(first, then, pe_action_optional); - } - } - -- if (reason && is_not_set(first->flags, pe_action_optional) -+ if (is_not_set(first->flags, pe_action_optional) - && is_not_set(first->flags, pe_action_runnable)) { -- pe_rsc_trace(then->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid); - pe_action_implies(then, first, pe_action_runnable); - } - -- if (reason && -- is_not_set(first->flags, pe_action_optional) && -+ if (is_not_set(first->flags, pe_action_optional) && - is_set(first->flags, pe_action_migrate_runnable) && - is_not_set(then->flags, pe_action_migrate_runnable)) { - --- -1.8.3.1 - - -From 32da90e58a89d7f9f3cd6d1e3f961c24b646d734 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 29 Mar 2019 17:50:07 -0500 -Subject: [PATCH 4/6] Refactor: scheduler: simplify handling of restart - ordering - -Don't condition pe_action_implies() on the desired state not being already -present, because pe_action_implies() handles that. - -Don't condition clearing first's pe_action_migrate_runnable on first being -required, because if it's optional it doesn't matter, and if (now or in the -future) optional can be changed to required later, it will actually be -important. ---- - pengine/native.c | 27 +++++++++++++++------------ - 1 file changed, 15 insertions(+), 12 deletions(-) - -diff --git a/pengine/native.c b/pengine/native.c -index dfcc910..8912aa1 100644 ---- a/pengine/native.c -+++ b/pengine/native.c -@@ -1980,6 +1980,7 @@ handle_restart_ordering(pe_action_t *first, pe_action_t *then, - if (is_set(filter, pe_action_runnable) - && is_not_set(then->flags, pe_action_runnable) - && is_set(then->rsc->flags, pe_rsc_managed)) { -+ // If a resource should restart but can't start, we still want to stop - reason = "stop"; - } - -@@ -1990,24 +1991,26 @@ handle_restart_ordering(pe_action_t *first, pe_action_t *then, - pe_rsc_trace(first->rsc, "Handling %s -> %s for %s", - first->uuid, then->uuid, reason); - -- if (is_set(first->flags, pe_action_optional)) { -- if (is_set(first->flags, pe_action_runnable) -- || is_not_set(then->flags, pe_action_optional)) { -- pe_action_implies(first, then, pe_action_optional); -- } -+ // Make 'first' required if it is runnable -+ if (is_set(first->flags, pe_action_runnable)) { -+ pe_action_implies(first, then, pe_action_optional); - } - -- if (is_not_set(first->flags, pe_action_optional) -- && is_not_set(first->flags, pe_action_runnable)) { -- pe_action_implies(then, first, pe_action_runnable); -+ // Make 'first' required if 'then' is required -+ if (is_not_set(then->flags, pe_action_optional)) { -+ pe_action_implies(first, then, pe_action_optional); - } - -- if (is_not_set(first->flags, pe_action_optional) && -- is_set(first->flags, pe_action_migrate_runnable) && -- is_not_set(then->flags, pe_action_migrate_runnable)) { -- -+ // Make 'first' unmigratable if 'then' is unmigratable -+ if (is_not_set(then->flags, pe_action_migrate_runnable)) { - pe_action_implies(first, then, pe_action_migrate_runnable); - } -+ -+ // Make 'then' unrunnable if 'first' is required but unrunnable -+ if (is_not_set(first->flags, pe_action_optional) -+ && is_not_set(first->flags, pe_action_runnable)) { -+ pe_action_implies(then, first, pe_action_runnable); -+ } - } - - enum pe_graph_flags --- -1.8.3.1 - - -From 8cfe743d4373fad6b4e50ee64894a16f7f24afa1 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 29 Mar 2019 19:11:25 -0500 -Subject: [PATCH 5/6] Fix: scheduler: one group stop shouldn't make another - required - -1.1.7's 8d2f237d reused pe_order_restart ("stop resource before stopping it") -for "stop later group member before stopping earlier group member". - -pe_order_restart includes a check for an unrunnable 'then', because in a -restart, even if the start is unrunnable, we still want to perform the stop. -However this check does not make sense for group stop ordering, and as of -1.1.10, this caused a regression where a group member could be unnecessarily -stopped. - -Example scenario: if a resource is ordered after a group member, and the -resource failed with on-fail=block, that would make the group member's -(optional) stop blocked as well, and that blocked stop would unnecessarily make -stops of later group members required. - -This commit fixes the issue by only applying the check when the 'then' action -is a start. (RHBZ#1609453) ---- - pengine/native.c | 8 +++++--- - 1 file changed, 5 insertions(+), 3 deletions(-) - -diff --git a/pengine/native.c b/pengine/native.c -index 8912aa1..747cb10 100644 ---- a/pengine/native.c -+++ b/pengine/native.c -@@ -1976,11 +1976,13 @@ handle_restart_ordering(pe_action_t *first, pe_action_t *then, - reason = "restart"; - } - -- // ... if 'then' is managed but unrunnable -+ /* ... if 'then' is unrunnable start of managed resource (if a resource -+ * should restart but can't start, we still want to stop) -+ */ - if (is_set(filter, pe_action_runnable) - && is_not_set(then->flags, pe_action_runnable) -- && is_set(then->rsc->flags, pe_rsc_managed)) { -- // If a resource should restart but can't start, we still want to stop -+ && is_set(then->rsc->flags, pe_rsc_managed) -+ && safe_str_eq(then->task, RSC_START)) { - reason = "stop"; - } - --- -1.8.3.1 - - -From b8e388eff56143632ef848d52eddad9560aad2cf Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 29 Mar 2019 19:44:39 -0500 -Subject: [PATCH 6/6] Test: scheduler: one group stop shouldn't make another - required - ---- - pengine/regression.sh | 1 + - pengine/test10/group-stop-ordering.dot | 2 + - pengine/test10/group-stop-ordering.exp | 1 + - pengine/test10/group-stop-ordering.scores | 17 ++++ - pengine/test10/group-stop-ordering.summary | 25 ++++++ - pengine/test10/group-stop-ordering.xml | 132 +++++++++++++++++++++++++++++ - 6 files changed, 178 insertions(+) - create mode 100644 pengine/test10/group-stop-ordering.dot - create mode 100644 pengine/test10/group-stop-ordering.exp - create mode 100644 pengine/test10/group-stop-ordering.scores - create mode 100644 pengine/test10/group-stop-ordering.summary - create mode 100644 pengine/test10/group-stop-ordering.xml - -diff --git a/pengine/regression.sh b/pengine/regression.sh -index e25990d..504de46 100755 ---- a/pengine/regression.sh -+++ b/pengine/regression.sh -@@ -68,6 +68,7 @@ do_test group-fail "Ensure stop order is preserved for partially active groups" - do_test group-unmanaged "No need to restart r115 because r114 is unmanaged" - do_test group-unmanaged-stopped "Make sure r115 is stopped when r114 fails" - do_test group-dependents "Account for the location preferences of things colocated with a group" -+do_test group-stop-ordering "Ensure blocked group member stop does not force other member stops" - - echo "" - do_test rsc_dep1 "Must not " -diff --git a/pengine/test10/group-stop-ordering.dot b/pengine/test10/group-stop-ordering.dot -new file mode 100644 -index 0000000..4b30191 ---- /dev/null -+++ b/pengine/test10/group-stop-ordering.dot -@@ -0,0 +1,2 @@ -+digraph "g" { -+} -diff --git a/pengine/test10/group-stop-ordering.exp b/pengine/test10/group-stop-ordering.exp -new file mode 100644 -index 0000000..56e315f ---- /dev/null -+++ b/pengine/test10/group-stop-ordering.exp -@@ -0,0 +1 @@ -+ -diff --git a/pengine/test10/group-stop-ordering.scores b/pengine/test10/group-stop-ordering.scores -new file mode 100644 -index 0000000..5f144d2 ---- /dev/null -+++ b/pengine/test10/group-stop-ordering.scores -@@ -0,0 +1,17 @@ -+Allocation scores: -+group_color: grp allocation score on fastvm-rhel-7-5-73: 0 -+group_color: grp allocation score on fastvm-rhel-7-5-74: 0 -+group_color: inside_resource_2 allocation score on fastvm-rhel-7-5-73: 0 -+group_color: inside_resource_2 allocation score on fastvm-rhel-7-5-74: 0 -+group_color: inside_resource_3 allocation score on fastvm-rhel-7-5-73: 0 -+group_color: inside_resource_3 allocation score on fastvm-rhel-7-5-74: 0 -+native_color: fence-fastvm-rhel-7-5-73 allocation score on fastvm-rhel-7-5-73: -INFINITY -+native_color: fence-fastvm-rhel-7-5-73 allocation score on fastvm-rhel-7-5-74: 0 -+native_color: fence-fastvm-rhel-7-5-74 allocation score on fastvm-rhel-7-5-73: 0 -+native_color: fence-fastvm-rhel-7-5-74 allocation score on fastvm-rhel-7-5-74: -INFINITY -+native_color: inside_resource_2 allocation score on fastvm-rhel-7-5-73: 0 -+native_color: inside_resource_2 allocation score on fastvm-rhel-7-5-74: 0 -+native_color: inside_resource_3 allocation score on fastvm-rhel-7-5-73: -INFINITY -+native_color: inside_resource_3 allocation score on fastvm-rhel-7-5-74: 0 -+native_color: outside_resource allocation score on fastvm-rhel-7-5-73: INFINITY -+native_color: outside_resource allocation score on fastvm-rhel-7-5-74: 0 -diff --git a/pengine/test10/group-stop-ordering.summary b/pengine/test10/group-stop-ordering.summary -new file mode 100644 -index 0000000..0ec8eb6 ---- /dev/null -+++ b/pengine/test10/group-stop-ordering.summary -@@ -0,0 +1,25 @@ -+ -+Current cluster status: -+Online: [ fastvm-rhel-7-5-73 fastvm-rhel-7-5-74 ] -+ -+ fence-fastvm-rhel-7-5-73 (stonith:fence_xvm): Started fastvm-rhel-7-5-74 -+ fence-fastvm-rhel-7-5-74 (stonith:fence_xvm): Started fastvm-rhel-7-5-73 -+ outside_resource (ocf::pacemaker:Dummy): FAILED fastvm-rhel-7-5-73 (blocked) -+ Resource Group: grp -+ inside_resource_2 (ocf::pacemaker:Dummy): Started fastvm-rhel-7-5-74 -+ inside_resource_3 (ocf::pacemaker:Dummy): Started fastvm-rhel-7-5-74 -+ -+Transition Summary: -+ -+Executing cluster transition: -+ -+Revised cluster status: -+Online: [ fastvm-rhel-7-5-73 fastvm-rhel-7-5-74 ] -+ -+ fence-fastvm-rhel-7-5-73 (stonith:fence_xvm): Started fastvm-rhel-7-5-74 -+ fence-fastvm-rhel-7-5-74 (stonith:fence_xvm): Started fastvm-rhel-7-5-73 -+ outside_resource (ocf::pacemaker:Dummy): FAILED fastvm-rhel-7-5-73 (blocked) -+ Resource Group: grp -+ inside_resource_2 (ocf::pacemaker:Dummy): Started fastvm-rhel-7-5-74 -+ inside_resource_3 (ocf::pacemaker:Dummy): Started fastvm-rhel-7-5-74 -+ -diff --git a/pengine/test10/group-stop-ordering.xml b/pengine/test10/group-stop-ordering.xml -new file mode 100644 -index 0000000..8439c1f ---- /dev/null -+++ b/pengine/test10/group-stop-ordering.xml -@@ -0,0 +1,132 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ --- -1.8.3.1 - diff --git a/SOURCES/005-bug-url.patch b/SOURCES/005-bug-url.patch deleted file mode 100644 index 5f183b4..0000000 --- a/SOURCES/005-bug-url.patch +++ /dev/null @@ -1,109 +0,0 @@ -From 55c576d970ed5b3446204668e7439d626e6d1a13 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 29 Mar 2019 19:48:23 -0500 -Subject: [PATCH 1/3] Build: configure: make bug report URL configurable - -to allow distributions to direct users to their own bug reporters ---- - configure.ac | 10 ++++++++++ - 1 file changed, 10 insertions(+) - -diff --git a/configure.ac b/configure.ac -index 70e074e..6fd17fd 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -318,6 +318,12 @@ AC_ARG_WITH(brand, - [ test x"$withval" = x"no" || PUBLICAN_BRAND="$withval" ]) - AC_SUBST(PUBLICAN_BRAND) - -+BUG_URL="" -+AC_ARG_WITH(bug-url, -+ [ --with-bug-url=DIR Address where users should submit bug reports @<:@https://bugs.clusterlabs.org/enter_bug.cgi?product=Pacemaker@:>@], -+ [ BUG_URL="$withval" ] -+) -+ - ASCIIDOC_CLI_TYPE="pcs" - AC_ARG_WITH(doc-cli, - [ --with-doc-cli=cli_type CLI type to use for generated documentation. [$ASCIIDOC_CLI_TYPE]], -@@ -442,6 +448,10 @@ if test x"${CONFIGDIR}" = x""; then - fi - AC_SUBST(CONFIGDIR) - -+if test x"${BUG_URL}" = x""; then -+ BUG_URL="https://bugs.clusterlabs.org/enter_bug.cgi?product=Pacemaker" -+fi -+ - for j in prefix exec_prefix bindir sbindir libexecdir datadir sysconfdir \ - sharedstatedir localstatedir libdir includedir oldincludedir infodir \ - mandir INITDIR docdir CONFIGDIR --- -1.8.3.1 - - -From 6c5938df90f83dcd686f11453ae701099a98836b Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 29 Mar 2019 19:50:21 -0500 -Subject: [PATCH 2/3] Feature: tools: make crm_report bug report URL - configurable at build time - -should have been done with 52aaffd ---- - configure.ac | 1 + - tools/crm_report.in | 2 +- - 2 files changed, 2 insertions(+), 1 deletion(-) - -diff --git a/configure.ac b/configure.ac -index 6fd17fd..ed51f67 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -451,6 +451,7 @@ AC_SUBST(CONFIGDIR) - if test x"${BUG_URL}" = x""; then - BUG_URL="https://bugs.clusterlabs.org/enter_bug.cgi?product=Pacemaker" - fi -+AC_SUBST(BUG_URL) - - for j in prefix exec_prefix bindir sbindir libexecdir datadir sysconfdir \ - sharedstatedir localstatedir libdir includedir oldincludedir infodir \ -diff --git a/tools/crm_report.in b/tools/crm_report.in -index 8e0d4b6..63c137c 100755 ---- a/tools/crm_report.in -+++ b/tools/crm_report.in -@@ -230,7 +230,7 @@ EOF - log "Collected results are available in $fname" - log " " - log "Please create a bug entry at" -- log " http://bugs.clusterlabs.org/enter_bug.cgi?product=Pacemaker" -+ log " @BUG_URL@" - log "Include a description of your problem and attach this tarball" - log " " - log "Thank you for taking time to create this report." --- -1.8.3.1 - - -From d99c1fa3fd52be01e03ce27a964fb1b09d29c46c Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 2 Apr 2019 10:35:16 -0500 -Subject: [PATCH 3/3] Test: tools: define missing constant - -accidentally omitted in abc33eff ---- - tools/regression.sh | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/tools/regression.sh b/tools/regression.sh -index 0c38629..3680f13 100755 ---- a/tools/regression.sh -+++ b/tools/regression.sh -@@ -8,6 +8,8 @@ GREP_OPTIONS= - verbose=0 - tests="dates tools acls validity" - -+CRM_EX_OK=0 -+ - function test_assert() { - target=$1; shift - cib=$1; shift --- -1.8.3.1 - diff --git a/SOURCES/006-fence-output-fix.patch b/SOURCES/006-fence-output-fix.patch deleted file mode 100644 index d0dd09a..0000000 --- a/SOURCES/006-fence-output-fix.patch +++ /dev/null @@ -1,143 +0,0 @@ -From 70577be56d841d2f58545877d36f3c3eaeaaae63 Mon Sep 17 00:00:00 2001 -From: Klaus Wenninger -Date: Wed, 3 Apr 2019 16:11:20 +0200 -Subject: [PATCH] Fix: service-lib: avoid call-pattern leading to - use-after-free - ---- - include/crm/services.h | 8 +++++++- - lib/fencing/st_client.c | 18 +++++++++++++++--- - lib/services/services.c | 13 ++++++++++++- - lib/services/services_linux.c | 5 +++++ - lib/services/services_private.h | 1 + - 5 files changed, 40 insertions(+), 5 deletions(-) - -diff --git a/include/crm/services.h b/include/crm/services.h -index eddafc3..cbb2354 100644 ---- a/include/crm/services.h -+++ b/include/crm/services.h -@@ -307,11 +307,17 @@ typedef struct svc_action_s { - * - * \param[in] op services action data - * \param[in] action_callback callback for when the action completes -+ * \param[in] action_fork_callback callback for when action forked successfully - * - * \retval TRUE succesfully started execution - * \retval FALSE failed to start execution, no callback will be received - */ -- gboolean services_action_async(svc_action_t * op, void (*action_callback) (svc_action_t *)); -+ gboolean services_action_async_fork_notify(svc_action_t * op, -+ void (*action_callback) (svc_action_t *), -+ void (*action_fork_callback) (svc_action_t *)); -+ -+ gboolean services_action_async(svc_action_t * op, -+ void (*action_callback) (svc_action_t *)); - - gboolean services_action_cancel(const char *name, const char *action, int interval); - -diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c -index 1c56cf4..0c1eadc 100644 ---- a/lib/fencing/st_client.c -+++ b/lib/fencing/st_client.c -@@ -827,6 +827,18 @@ stonith_action_async_done(svc_action_t *svc_action) - stonith__destroy_action(action); - } - -+static void -+stonith_action_async_forked(svc_action_t *svc_action) -+{ -+ stonith_action_t *action = (stonith_action_t *) svc_action->cb_data; -+ -+ action->pid = svc_action->pid; -+ action->svc_action = svc_action; -+ -+ crm_trace("Child process %d performing action '%s' successfully forked", -+ action->pid, action->action); -+} -+ - static int - internal_stonith_action_execute(stonith_action_t * action) - { -@@ -873,12 +885,12 @@ internal_stonith_action_execute(stonith_action_t * action) - - if (action->async) { - /* async */ -- if(services_action_async(svc_action, &stonith_action_async_done) == FALSE) { -+ if(services_action_async_fork_notify(svc_action, -+ &stonith_action_async_done, -+ &stonith_action_async_forked) == FALSE) { - services_action_free(svc_action); - svc_action = NULL; - } else { -- action->pid = svc_action->pid; -- action->svc_action = svc_action; - rc = 0; - } - -diff --git a/lib/services/services.c b/lib/services/services.c -index 60402e7..ef2c5fc 100644 ---- a/lib/services/services.c -+++ b/lib/services/services.c -@@ -843,12 +843,17 @@ services_untrack_op(svc_action_t *op) - } - - gboolean --services_action_async(svc_action_t * op, void (*action_callback) (svc_action_t *)) -+services_action_async_fork_notify(svc_action_t * op, -+ void (*action_callback) (svc_action_t *), -+ void (*action_fork_callback) (svc_action_t *)) - { - op->synchronous = false; - if (action_callback) { - op->opaque->callback = action_callback; - } -+ if (action_fork_callback) { -+ op->opaque->fork_callback = action_fork_callback; -+ } - - if (op->interval > 0) { - init_recurring_actions(); -@@ -868,6 +873,12 @@ services_action_async(svc_action_t * op, void (*action_callback) (svc_action_t * - return action_exec_helper(op); - } - -+gboolean -+services_action_async(svc_action_t * op, -+ void (*action_callback) (svc_action_t *)) -+{ -+ return services_action_async_fork_notify(op, action_callback, NULL); -+} - - static gboolean processing_blocked_ops = FALSE; - -diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c -index d79c16d..705901e 100644 ---- a/lib/services/services_linux.c -+++ b/lib/services/services_linux.c -@@ -877,6 +877,11 @@ services_os_action_execute(svc_action_t * op) - op->opaque->stdin_fd = -1; - } - -+ // after fds are setup properly and before we plug anything into mainloop -+ if (op->opaque->fork_callback) { -+ op->opaque->fork_callback(op); -+ } -+ - if (op->synchronous) { - action_synced_wait(op, pmask); - sigchld_cleanup(); -diff --git a/lib/services/services_private.h b/lib/services/services_private.h -index 9735da7..227e17f 100644 ---- a/lib/services/services_private.h -+++ b/lib/services/services_private.h -@@ -36,6 +36,7 @@ struct svc_action_private_s { - - guint repeat_timer; - void (*callback) (svc_action_t * op); -+ void (*fork_callback) (svc_action_t * op); - - int stderr_fd; - mainloop_io_t *stderr_gsource; --- -1.8.3.1 - diff --git a/SOURCES/007-security.patch b/SOURCES/007-security.patch deleted file mode 100644 index 33fb725..0000000 --- a/SOURCES/007-security.patch +++ /dev/null @@ -1,2659 +0,0 @@ -From f91a961112ec9796181b42aa52f9c36dfa3c6a99 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Tue, 2 Apr 2019 10:13:21 +0200 -Subject: [PATCH 1/7] High: libservices: fix use-after-free wrt. alert handling - -This could possibly lead to unsolicited information disclosure by the -means of standard output of the immediately preceding agent/resource -execution leaking into the log stream under some circumstances. -It was hence assigned CVE-2019-3885. - -The provoked pathological state of pacemaker-execd daemon progresses -towards crashing it for hitting segmentation fault. ---- - lib/services/services.c | 40 +--------------------------------------- - lib/services/services_linux.c | 35 +++++++++++++++++++++++++++++++---- - 2 files changed, 32 insertions(+), 43 deletions(-) - -diff --git a/lib/services/services.c b/lib/services/services.c -index ef2c5fc..1d06c5d 100644 ---- a/lib/services/services.c -+++ b/lib/services/services.c -@@ -450,35 +450,6 @@ services_action_user(svc_action_t *op, const char *user) - return crm_user_lookup(user, &(op->opaque->uid), &(op->opaque->gid)); - } - --static void --set_alert_env(gpointer key, gpointer value, gpointer user_data) --{ -- int rc; -- -- if (value) { -- rc = setenv(key, value, 1); -- } else { -- rc = unsetenv(key); -- } -- -- if (rc < 0) { -- crm_perror(LOG_ERR, "setenv %s=%s", -- (char*)key, (value? (char*)value : "")); -- } else { -- crm_trace("setenv %s=%s", (char*)key, (value? (char*)value : "")); -- } --} -- --static void --unset_alert_env(gpointer key, gpointer value, gpointer user_data) --{ -- if (unsetenv(key) < 0) { -- crm_perror(LOG_ERR, "unset %s", (char*)key); -- } else { -- crm_trace("unset %s", (char*)key); -- } --} -- - /*! - * \brief Execute an alert agent action - * -@@ -493,18 +464,9 @@ unset_alert_env(gpointer key, gpointer value, gpointer user_data) - gboolean - services_alert_async(svc_action_t *action, void (*cb)(svc_action_t *op)) - { -- gboolean responsible; -- - action->synchronous = false; - action->opaque->callback = cb; -- if (action->params) { -- g_hash_table_foreach(action->params, set_alert_env, NULL); -- } -- responsible = services_os_action_execute(action); -- if (action->params) { -- g_hash_table_foreach(action->params, unset_alert_env, NULL); -- } -- return responsible; -+ return services_os_action_execute(action); - } - - #if SUPPORT_DBUS -diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c -index 705901e..2047b64 100644 ---- a/lib/services/services_linux.c -+++ b/lib/services/services_linux.c -@@ -159,6 +159,25 @@ set_ocf_env_with_prefix(gpointer key, gpointer value, gpointer user_data) - set_ocf_env(buffer, value, user_data); - } - -+static void -+set_alert_env(gpointer key, gpointer value, gpointer user_data) -+{ -+ int rc; -+ -+ if (value != NULL) { -+ rc = setenv(key, value, 1); -+ } else { -+ rc = unsetenv(key); -+ } -+ -+ if (rc < 0) { -+ crm_perror(LOG_ERR, "setenv %s=%s", -+ (char*)key, (value? (char*)value : "")); -+ } else { -+ crm_trace("setenv %s=%s", (char*)key, (value? (char*)value : "")); -+ } -+} -+ - /*! - * \internal - * \brief Add environment variables suitable for an action -@@ -168,12 +187,20 @@ set_ocf_env_with_prefix(gpointer key, gpointer value, gpointer user_data) - static void - add_action_env_vars(const svc_action_t *op) - { -- if (safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_OCF) == FALSE) { -- return; -+ void (*env_setter)(gpointer, gpointer, gpointer) = NULL; -+ if (op->agent == NULL) { -+ env_setter = set_alert_env; /* we deal with alert handler */ -+ -+ } else if (safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_OCF)) { -+ env_setter = set_ocf_env_with_prefix; - } - -- if (op->params) { -- g_hash_table_foreach(op->params, set_ocf_env_with_prefix, NULL); -+ if (env_setter != NULL && op->params != NULL) { -+ g_hash_table_foreach(op->params, env_setter, NULL); -+ } -+ -+ if (env_setter == NULL || env_setter == set_alert_env) { -+ return; - } - - set_ocf_env("OCF_RA_VERSION_MAJOR", "1", NULL); --- -1.8.3.1 - - -From ab44422fa955c2dff1ac1822521e7ad335d4aab7 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Mon, 15 Apr 2019 23:19:44 +0200 -Subject: [PATCH 2/7] High: pacemakerd vs. IPC/procfs confused deputy - authenticity issue (0/4) - -[0/4: make crm_pid_active more precise as to when detections fail] - -It would be bad if the function claimed the process is not active -when the only obstacle in the detection process was that none of the -detection methods worked for a plain lack of permissions to apply -them. Also, do some other minor cleanup of the function and add its -documentation. As an additional measure, log spamming is kept at -minimum for repeated queries about the same PID. ---- - include/crm_internal.h | 21 +++++++++++ - lib/common/utils.c | 96 +++++++++++++++++++++++++++----------------------- - 2 files changed, 73 insertions(+), 44 deletions(-) - -diff --git a/include/crm_internal.h b/include/crm_internal.h -index 5692929..0adeb7b 100644 ---- a/include/crm_internal.h -+++ b/include/crm_internal.h -@@ -140,6 +140,27 @@ extern int node_score_yellow; - extern int node_score_infinity; - - /* Assorted convenience functions */ -+ -+/*! -+ * \internal -+ * \brief Detect if process per PID and optionally exe path (component) exists -+ * -+ * \param[in] pid PID of process assumed alive, disproving of which to try -+ * \param[in] daemon exe path (component) to possibly match with procfs entry -+ * -+ * \return -1 on invalid PID specification, -2 when the calling process has no -+ * (is refused an) ability to (dis)prove the predicate, -+ * 0 if the negation of the predicate is confirmed (check-through-kill -+ * indicates so, or the subsequent check-through-procfs-match on -+ * \p daemon when provided and procfs available at the standard path), -+ * 1 if it cannot be disproved (reliably [modulo race conditions] -+ * when \p daemon provided, procfs available at the standard path -+ * and the calling process has permissions to access the respective -+ * procfs location, less so otherwise, since mere check-through-kill -+ * is exercised without powers to exclude PID recycled in the interim). -+ * -+ * \note This function cannot be used to verify \e authenticity of the process. -+ */ - int crm_pid_active(long pid, const char *daemon); - void crm_make_daemon(const char *name, gboolean daemonize, const char *pidfile); - -diff --git a/lib/common/utils.c b/lib/common/utils.c -index f3f60ed..2ac7901 100644 ---- a/lib/common/utils.c -+++ b/lib/common/utils.c -@@ -1,19 +1,10 @@ - /* -- * Copyright (C) 2004 Andrew Beekhof -+ * Copyright 2004-2019 the Pacemaker project contributors - * -- * This library is free software; you can redistribute it and/or -- * modify it under the terms of the GNU Lesser General Public -- * License as published by the Free Software Foundation; either -- * version 2.1 of the License, or (at your option) any later version. -+ * The version control history for this file may have further details. - * -- * This library is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- * Lesser General Public License for more details. -- * -- * You should have received a copy of the GNU Lesser General Public -- * License along with this library; if not, write to the Free Software -- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. - */ - - #include -@@ -717,16 +708,21 @@ crm_abort(const char *file, const char *function, int line, - int - crm_pid_active(long pid, const char *daemon) - { -+ static int last_asked_pid = 0; /* log spam prevention */ -+#if SUPPORT_PROCFS - static int have_proc_pid = 0; -+#else -+ static int have_proc_pid = -1; -+#endif -+ int rc = 0; - -- if(have_proc_pid == 0) { -+ if (have_proc_pid == 0) { -+ /* evaluation of /proc/PID/exe applicability via self-introspection */ - char proc_path[PATH_MAX], exe_path[PATH_MAX]; -- -- /* check to make sure pid hasn't been reused by another process */ -- snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", (long unsigned int)getpid()); -- -+ snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", -+ (long unsigned int) getpid()); - have_proc_pid = 1; -- if(readlink(proc_path, exe_path, PATH_MAX - 1) < 0) { -+ if (readlink(proc_path, exe_path, sizeof(exe_path) - 1) < 0) { - have_proc_pid = -1; - } - } -@@ -734,40 +730,52 @@ crm_pid_active(long pid, const char *daemon) - if (pid <= 0) { - return -1; - -- } else if (kill(pid, 0) < 0 && errno == ESRCH) { -- return 0; -+ } else if ((rc = kill(pid, 0)) < 0 && errno == ESRCH) { -+ return 0; /* no such PID detected */ - -- } else if(daemon == NULL || have_proc_pid == -1) { -- return 1; -+ } else if (rc < 0 && have_proc_pid == -1) { -+ if (last_asked_pid != pid) { -+ crm_info("Cannot examine PID %ld: %s", pid, strerror(errno)); -+ last_asked_pid = pid; -+ } -+ return -2; /* errno != ESRCH */ -+ -+ } else if (rc == 0 && (daemon == NULL || have_proc_pid == -1)) { -+ return 1; /* kill as the only indicator, cannot double check */ - - } else { -- int rc = 0; -+ /* make sure PID hasn't been reused by another process -+ XXX: might still be just a zombie, which could confuse decisions */ -+ bool checked_through_kill = (rc == 0); - char proc_path[PATH_MAX], exe_path[PATH_MAX], myexe_path[PATH_MAX]; -- -- /* check to make sure pid hasn't been reused by another process */ -- snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", pid); -- -- rc = readlink(proc_path, exe_path, PATH_MAX - 1); -- if (rc < 0 && errno == EACCES) { -- crm_perror(LOG_INFO, "Could not read from %s", proc_path); -- return 1; -+ snprintf(proc_path, sizeof(proc_path), "/proc/%ld/exe", pid); -+ -+ rc = readlink(proc_path, exe_path, sizeof(exe_path) - 1); -+ if ((rc < 0) && (errno == EACCES)) { -+ if (last_asked_pid != pid) { -+ crm_info("Could not read from %s: %s", proc_path, -+ strerror(errno)); -+ last_asked_pid = pid; -+ } -+ return checked_through_kill ? 1 : -2; - } else if (rc < 0) { -- crm_perror(LOG_ERR, "Could not read from %s", proc_path); -- return 0; -+ if (last_asked_pid != pid) { -+ crm_err("Could not read from %s: %s (%d)", proc_path, -+ strerror(errno), errno); -+ last_asked_pid = pid; -+ } -+ return 0; /* most likely errno == ENOENT */ - } -- -+ exe_path[rc] = '\0'; - -- exe_path[rc] = 0; -- -- if(daemon[0] != '/') { -- rc = snprintf(myexe_path, sizeof(proc_path), CRM_DAEMON_DIR"/%s", daemon); -- myexe_path[rc] = 0; -+ if (daemon[0] != '/') { -+ rc = snprintf(myexe_path, sizeof(myexe_path), CRM_DAEMON_DIR"/%s", -+ daemon); - } else { -- rc = snprintf(myexe_path, sizeof(proc_path), "%s", daemon); -- myexe_path[rc] = 0; -+ rc = snprintf(myexe_path, sizeof(myexe_path), "%s", daemon); - } -- -- if (strcmp(exe_path, myexe_path) == 0) { -+ -+ if (rc > 0 && rc < sizeof(myexe_path) && !strcmp(exe_path, myexe_path)) { - return 1; - } - } --- -1.8.3.1 - - -From 6888aaf3ad365ef772f8189c9958f58b85ec62d4 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Mon, 15 Apr 2019 23:20:42 +0200 -Subject: [PATCH 3/7] High: pacemakerd vs. IPC/procfs confused deputy - authenticity issue (1/4) - -[1/4: new helpers to allow IPC client side to authenticate the server] - -The title problem here could possibly lead to local privilege escalation -up to the root's level (and implicitly unguarded by some additional -protection layers like SELinux unless the defaults constrained further). - -Main problem is that the authenticity assumptions were built on two, -seemingly mutually supporting legs leading to two CVEs assigned: - -* procfs (mere process existence and right path to binary check) - used to verify (this part was assigned CVE-2018-16878), and - -* one-way only client-server authentication, putting the client - here at the mercy of the server not necessarily cross-verified - per the above point if at all (this part was assigned - CVE-2018-16877) - -whereas these two were in fact orthogonal, tearing security -assumptions about the "passive influencers" in the pacemaker's daemon -resilience-friendly constellation (orchestrated by the main of them, -pacemakerd) apart. Moreover, procfs-based approach is discouraged -for other reasons. - -The layout of the basic fix is as follows: -* 1/4: new helpers to allow IPC client side to authenticate the server - (this commit, along with unifying subsequent solution for - both CVEs) -* 2/4: pacemakerd to trust pre-existing processes via new checks instead - (along with unifying solution for both CVEs) -* 3/4: other daemons to authenticate IPC servers of fellow processes - (along with addressing CVE-2018-16877 alone, for parts of - pacemaker not covered earlier) -* 4/4: CPG users to be careful about now-more-probable rival processes - (this is merely to mitigate corner case fallout from the new - approaches taken to face CVE-2018-16878 in particular; - courtesy of Yan Gao of SUSE for reporting this) - -With "basic", it is meant that it constitutes a self-contained best -effort solution with some compromises that can only be overcome with the -assistance of IPC library, libqb, as is also elaborated in messages of -remaining "fix" commits. Beside that, also conventional encapsulation -of server-by-client authentication would be useful, but lack thereof -is not an obstacle (more so should there by any security related -neglectations on the IPC client side and its connection initiating -arrangement within libqb that has a potential to strike as early as -when the authenticity of the server side is yet to be examined). - -One extra kludge that's introduced for FreeBSD lacking Unix socket to -remote peer PID mapping is masquerading such an unspecified PID with -value of 1, since that shall always be around as "init" task and, -deferring to proof by contradiction, cannot be pacemakerd-spawned -child either even if PID 1 was pacemakerd (and running such a child -alone is rather nonsensical). The code making decisions based on that -value must acknowledge this craze and refrain from killing/signalling -the underlying process on this platform (but shall in general follow -the same elsewhere, keep in mind systemd socket-based activation for -instance, which would end up in such a situation easily!). ---- - configure.ac | 43 +++++++++++ - include/crm/common/Makefile.am | 7 +- - include/crm/common/ipc.h | 55 +++++++++---- - include/crm/common/ipc_internal.h | 69 +++++++++++++++++ - lib/common/ipc.c | 158 ++++++++++++++++++++++++++++++++++---- - 5 files changed, 303 insertions(+), 29 deletions(-) - create mode 100644 include/crm/common/ipc_internal.h - -diff --git a/configure.ac b/configure.ac -index ed51f67..ce02777 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -465,6 +465,48 @@ do - fi - done - -+us_auth= -+AC_CHECK_HEADER([sys/socket.h], [ -+ AC_CHECK_DECL([SO_PEERCRED], [ -+ # Linux -+ AC_CHECK_TYPE([struct ucred], [ -+ us_auth=peercred_ucred; -+ AC_DEFINE([US_AUTH_PEERCRED_UCRED], [1], -+ [Define if Unix socket auth method is -+ getsockopt(s, SO_PEERCRED, &ucred, ...)]) -+ ], [ -+ # OpenBSD -+ AC_CHECK_TYPE([struct sockpeercred], [ -+ us_auth=localpeercred_sockepeercred; -+ AC_DEFINE([US_AUTH_PEERCRED_SOCKPEERCRED], [1], -+ [Define if Unix socket auth method is -+ getsockopt(s, SO_PEERCRED, &sockpeercred, ...)]) -+ ], [], [[#include ]]) -+ ], [[#define _GNU_SOURCE -+ #include ]]) -+ ], [], [[#include ]]) -+]) -+ -+if test -z "${us_auth}"; then -+ # FreeBSD -+ AC_CHECK_DECL([getpeereid], [ -+ us_auth=getpeereid; -+ AC_DEFINE([US_AUTH_GETPEEREID], [1], -+ [Define if Unix socket auth method is -+ getpeereid(s, &uid, &gid)]) -+ ], [ -+ # Solaris/OpenIndiana -+ AC_CHECK_DECL([getpeerucred], [ -+ us_auth=getpeerucred; -+ AC_DEFINE([US_AUTH_GETPEERUCRED], [1], -+ [Define if Unix socket auth method is -+ getpeercred(s, &ucred)]) -+ ], [ -+ AC_MSG_ERROR([No way to authenticate a Unix socket peer]) -+ ], [[#include ]]) -+ ]) -+fi -+ - dnl This OS-based decision-making is poor autotools practice; - dnl feature-based mechanisms are strongly preferred. - dnl -@@ -2179,3 +2221,4 @@ AC_MSG_RESULT([ LDFLAGS_HARDENED_EXE = ${LDFLAGS_HARDENED_EXE}]) - AC_MSG_RESULT([ LDFLAGS_HARDENED_LIB = ${LDFLAGS_HARDENED_LIB}]) - AC_MSG_RESULT([ Libraries = ${LIBS}]) - AC_MSG_RESULT([ Stack Libraries = ${CLUSTERLIBS}]) -+AC_MSG_RESULT([ Unix socket auth method = ${us_auth}]) -diff --git a/include/crm/common/Makefile.am b/include/crm/common/Makefile.am -index b90ac79..aacb6ff 100644 ---- a/include/crm/common/Makefile.am -+++ b/include/crm/common/Makefile.am -@@ -1,5 +1,7 @@ - # --# Copyright 2004-2019 Andrew Beekhof -+# Copyright 2004-2019 the Pacemaker project contributors -+# -+# The version control history for this file may have further details. - # - # This source code is licensed under the GNU General Public License version 2 - # or later (GPLv2+) WITHOUT ANY WARRANTY. -@@ -11,7 +13,8 @@ headerdir=$(pkgincludedir)/crm/common - - header_HEADERS = xml.h ipc.h util.h iso8601.h mainloop.h logging.h \ - nvpair.h --noinst_HEADERS = ipcs.h internal.h remote_internal.h xml_internal.h -+noinst_HEADERS = ipcs.h internal.h remote_internal.h xml_internal.h \ -+ ipc_internal.h - if BUILD_CIBSECRETS - noinst_HEADERS += cib_secrets.h - endif -diff --git a/include/crm/common/ipc.h b/include/crm/common/ipc.h -index 8722252..df56bbe 100644 ---- a/include/crm/common/ipc.h -+++ b/include/crm/common/ipc.h -@@ -1,19 +1,10 @@ - /* -- * Copyright (C) 2013 Andrew Beekhof -+ * Copyright 2013-2019 the Pacemaker project contributors - * -- * This program is free software; you can redistribute it and/or -- * modify it under the terms of the GNU Lesser General Public -- * License as published by the Free Software Foundation; either -- * version 2 of the License, or (at your option) any later version. -+ * The version control history for this file may have further details. - * -- * This software is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- * General Public License for more details. -- * -- * You should have received a copy of the GNU Lesser General Public -- * License along with this library; if not, write to the Free Software -- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. - */ - #ifndef CRM_COMMON_IPC__H - # define CRM_COMMON_IPC__H -@@ -77,6 +68,44 @@ uint32_t crm_ipc_buffer_flags(crm_ipc_t * client); - const char *crm_ipc_name(crm_ipc_t * client); - unsigned int crm_ipc_default_buffer_size(void); - -+/*! -+ * \brief Check the authenticity of the IPC socket peer process -+ * -+ * If everything goes well, peer's authenticity is verified by the means -+ * of comparing against provided referential UID and GID (either satisfies), -+ * and the result of this check can be deduced from the return value. -+ * As an exception, detected UID of 0 ("root") satisfies arbitrary -+ * provided referential daemon's credentials. -+ * -+ * \param[in] sock IPC related, connected Unix socket to check peer of -+ * \param[in] refuid referential UID to check against -+ * \param[in] refgid referential GID to check against -+ * \param[out] gotpid to optionally store obtained PID of the peer -+ * (not available on FreeBSD, special value of 1 -+ * used instead, and the caller is required to -+ * special case this value respectively) -+ * \param[out] gotuid to optionally store obtained UID of the peer -+ * \param[out] gotgid to optionally store obtained GID of the peer -+ * -+ * \return 0 if IPC related socket's peer is not authentic given the -+ * referential credentials (see above), 1 if it is, -+ * negative value on error (generally expressing -errno unless -+ * it was zero even on nonhappy path, -pcmk_err_generic is -+ * returned then; no message is directly emitted) -+ * -+ * \note While this function is tolerant on what constitutes authorized -+ * IPC daemon process (its effective user matches UID=0 or \p refuid, -+ * or at least its group matches \p refroup), either or both (in case -+ * of UID=0) mismatches on the expected credentials of such peer -+ * process \e shall be investigated at the caller when value of 1 -+ * gets returned there, since higher-than-expected privileges in -+ * respect to the expected/intended credentials possibly violate -+ * the least privilege principle and may pose an additional risk -+ * (i.e. such accidental inconsistency shall be eventually fixed). -+ */ -+int crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid, -+ pid_t *gotpid, uid_t *gotuid, gid_t *gotgid); -+ - /* Utils */ - xmlNode *create_hello_message(const char *uuid, const char *client_name, - const char *major_version, const char *minor_version); -diff --git a/include/crm/common/ipc_internal.h b/include/crm/common/ipc_internal.h -new file mode 100644 -index 0000000..41a6653 ---- /dev/null -+++ b/include/crm/common/ipc_internal.h -@@ -0,0 +1,69 @@ -+/* -+ * Copyright 2019 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. -+ * -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -+ */ -+ -+#ifndef PCMK__IPC_INTERNAL_H -+#define PCMK__IPC_INTERNAL_H -+ -+#include -+ -+#include /* US_AUTH_GETPEEREID */ -+ -+ -+/* denotes "non yieldable PID" on FreeBSD, or actual PID1 in scenarios that -+ require a delicate handling anyway (socket-based activation with systemd); -+ we can be reasonably sure that this PID is never possessed by the actual -+ child daemon, as it gets taken either by the proper init, or by pacemakerd -+ itself (i.e. this precludes anything else); note that value of zero -+ is meant to carry "unset" meaning, and better not to bet on/conditionalize -+ over signedness of pid_t */ -+#define PCMK__SPECIAL_PID 1 -+ -+#if defined(US_AUTH_GETPEEREID) -+/* on FreeBSD, we don't want to expose "non-yieldable PID" (leading to -+ "IPC liveness check only") as its nominal representation, which could -+ cause confusion -- this is unambiguous as long as there's no -+ socket-based activation like with systemd (very improbable) */ -+#define PCMK__SPECIAL_PID_AS_0(p) (((p) == PCMK__SPECIAL_PID) ? 0 : (p)) -+#else -+#define PCMK__SPECIAL_PID_AS_0(p) (p) -+#endif -+ -+/*! -+ * \internal -+ * \brief Check the authenticity and liveness of the process via IPC end-point -+ * -+ * When IPC daemon under given IPC end-point (name) detected, its authenticity -+ * is verified by the means of comparing against provided referential UID and -+ * GID, and the result of this check can be deduced from the return value. -+ * As an exception, referential UID of 0 (~ root) satisfies arbitrary -+ * detected daemon's credentials. -+ * -+ * \param[in] name IPC name to base the search on -+ * \param[in] refuid referential UID to check against -+ * \param[in] refgid referential GID to check against -+ * \param[out] gotpid to optionally store obtained PID of the found process -+ * upon returning 1 or -2 -+ * (not available on FreeBSD, special value of 1, -+ * see PCMK__SPECIAL_PID, used instead, and the caller -+ * is required to special case this value respectively) -+ * -+ * \return 0 if no trace of IPC peer's liveness detected, 1 if it was, -+ * -1 on error, and -2 when the IPC blocked with unauthorized -+ * process (log message emitted in both latter cases) -+ * -+ * \note This function emits a log message also in case there isn't a perfect -+ * match in respect to \p reguid and/or \p refgid, for a possible -+ * least privilege principle violation. -+ * -+ * \see crm_ipc_is_authentic_process -+ */ -+int pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid, -+ gid_t refgid, pid_t *gotpid); -+ -+#endif -diff --git a/lib/common/ipc.c b/lib/common/ipc.c -index 3258bcb..5b47dd6 100644 ---- a/lib/common/ipc.c -+++ b/lib/common/ipc.c -@@ -1,23 +1,25 @@ - /* -- * Copyright (C) 2004 Andrew Beekhof -+ * Copyright 2004-2019 the Pacemaker project contributors - * -- * This library is free software; you can redistribute it and/or -- * modify it under the terms of the GNU Lesser General Public -- * License as published by the Free Software Foundation; either -- * version 2.1 of the License, or (at your option) any later version. -+ * The version control history for this file may have further details. - * -- * This library is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- * Lesser General Public License for more details. -- * -- * You should have received a copy of the GNU Lesser General Public -- * License along with this library; if not, write to the Free Software -- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. - */ - - #include - -+#if defined(US_AUTH_PEERCRED_UCRED) || defined(US_AUTH_PEERCRED_SOCKPEERCRED) -+# ifdef US_AUTH_PEERCRED_UCRED -+# ifndef _GNU_SOURCE -+# define _GNU_SOURCE -+# endif -+# endif -+# include -+#elif defined(US_AUTH_GETPEERUCRED) -+# include -+#endif -+ - #include - - #include -@@ -30,11 +32,13 @@ - #include - #include - --#include -+#include /* indirectly: pcmk_err_generic */ - #include - #include - #include - -+#include /* PCMK__SPECIAL_PID* */ -+ - #define PCMK_IPC_VERSION 1 - - /* Evict clients whose event queue grows this large (by default) */ -@@ -1375,6 +1379,132 @@ crm_ipc_send(crm_ipc_t * client, xmlNode * message, enum crm_ipc_flags flags, in - return rc; - } - -+int -+crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid, -+ pid_t *gotpid, uid_t *gotuid, gid_t *gotgid) { -+ int ret = 0; -+ pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; -+#if defined(US_AUTH_PEERCRED_UCRED) -+ struct ucred ucred; -+ socklen_t ucred_len = sizeof(ucred); -+ -+ if (!getsockopt(sock, SOL_SOCKET, SO_PEERCRED, -+ &ucred, &ucred_len) -+ && ucred_len == sizeof(ucred)) { -+ found_pid = ucred.pid; found_uid = ucred.uid; found_gid = ucred.gid; -+ -+#elif defined(US_AUTH_PEERCRED_SOCKPEERCRED) -+ struct sockpeercred sockpeercred; -+ socklen_t sockpeercred_len = sizeof(sockpeercred); -+ -+ if (!getsockopt(sock, SOL_SOCKET, SO_PEERCRED, -+ &sockpeercred, &sockpeercred_len) -+ && sockpeercred_len == sizeof(sockpeercred_len)) { -+ found_pid = sockpeercred.pid; -+ found_uid = sockpeercred.uid; found_gid = sockpeercred.gid; -+ -+#elif defined(US_AUTH_GETPEEREID) -+ if (!getpeereid(sock, &found_uid, &found_gid)) { -+ found_pid = PCMK__SPECIAL_PID; /* cannot obtain PID (FreeBSD) */ -+ -+#elif defined(US_AUTH_GETPEERUCRED) -+ ucred_t *ucred; -+ if (!getpeerucred(sock, &ucred)) { -+ errno = 0; -+ found_pid = ucred_getpid(ucred); -+ found_uid = ucred_geteuid(ucred); found_gid = ucred_getegid(ucred); -+ ret = -errno; -+ ucred_free(ucred); -+ if (ret) { -+ return (ret < 0) ? ret : -pcmk_err_generic; -+ } -+ -+#else -+# error "No way to authenticate a Unix socket peer" -+ errno = 0; -+ if (0) { -+#endif -+ if (gotpid != NULL) { -+ *gotpid = found_pid; -+ } -+ if (gotuid != NULL) { -+ *gotuid = found_uid; -+ } -+ if (gotgid != NULL) { -+ *gotgid = found_gid; -+ } -+ ret = (found_uid == 0 || found_uid == refuid || found_gid == refgid); -+ } else { -+ ret = (errno > 0) ? -errno : -pcmk_err_generic; -+ } -+ -+ return ret; -+} -+ -+int -+pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid, -+ gid_t refgid, pid_t *gotpid) { -+ static char last_asked_name[PATH_MAX / 2] = ""; /* log spam prevention */ -+ int fd, ret = 0; -+ pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; -+ qb_ipcc_connection_t *c; -+ -+ if ((c = qb_ipcc_connect(name, 0)) == NULL) { -+ crm_info("Could not connect to %s IPC: %s", name, strerror(errno)); -+ -+ } else if ((ret = qb_ipcc_fd_get(c, &fd))) { -+ crm_err("Could not get fd from %s IPC: %s (%d)", name, -+ strerror(-ret), -ret); -+ ret = -1; -+ -+ } else if ((ret = crm_ipc_is_authentic_process(fd, refuid, refgid, -+ &found_pid, &found_uid, -+ &found_gid)) < 0) { -+ if (ret == -pcmk_err_generic) { -+ crm_err("Could not get peer credentials from %s IPC", name); -+ } else { -+ crm_err("Could not get peer credentials from %s IPC: %s (%d)", -+ name, strerror(-ret), -ret); -+ } -+ ret = -1; -+ -+ } else { -+ if (gotpid != NULL) { -+ *gotpid = found_pid; -+ } -+ -+ if (!ret) { -+ crm_err("Daemon (IPC %s) effectively blocked with unauthorized" -+ " process %lld (uid: %lld, gid: %lld)", -+ name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ ret = -2; -+ } else if ((found_uid != refuid || found_gid != refgid) -+ && strncmp(last_asked_name, name, sizeof(last_asked_name))) { -+ if (!found_uid && refuid) { -+ crm_warn("Daemon (IPC %s) runs as root, whereas the expected" -+ " credentials are %lld:%lld, hazard of violating" -+ " the least privilege principle", -+ name, (long long) refuid, (long long) refgid); -+ } else { -+ crm_notice("Daemon (IPC %s) runs as %lld:%lld, whereas the" -+ " expected credentials are %lld:%lld, which may" -+ " mean a different set of privileges than expected", -+ name, (long long) found_uid, (long long) found_gid, -+ (long long) refuid, (long long) refgid); -+ } -+ memccpy(last_asked_name, name, '\0', sizeof(last_asked_name)); -+ } -+ } -+ -+ if (ret) { /* here, !ret only when we could not initially connect */ -+ qb_ipcc_disconnect(c); -+ } -+ -+ return ret; -+} -+ -+ - /* Utils */ - - xmlNode * --- -1.8.3.1 - - -From 904c53ea311fd6fae945a55202b0a7ccf3783465 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Tue, 16 Apr 2019 00:04:47 +0200 -Subject: [PATCH 4/7] High: pacemakerd vs. IPC/procfs confused deputy - authenticity issue (2/4) - -[2/4: pacemakerd to trust pre-existing processes via new checks instead] - -In pacemakerd in the context of entrusting pre-existing processes, -we now resort to procfs-based solution only in boundary, fouled cases, -and primarily examine the credentials of the processes already -occupying known IPC end-points before adopting them. - -The commit applies the new helpers from 1/1 so as to close the both -related sensitive problems, CVE-2018-16877 and CVE-2018-16878, in -a unified manner, this time limited to the main daemon of pacemaker -(pacemakerd). - -To be noted that it is clearly not 100% for this purpose for still -allowing for TOCTTOU, but that's what commit (3/3) is meant to solve -for the most part, plus there may be optimizations solving this concern -as a side effect, but that requires an active assistance on the libqb -side (https://github.com/ClusterLabs/libqb/issues/325) since any -improvement on pacemaker side in isolation would be very -cumbersome if generally possible at all, but either way -means a new, soft compatibility encumberment. - -As a follow-up to what was put in preceding 1/3 commit, PID of 1 tracked -as child's identification on FreeBSD (or when socket-based activation is -used with systemd) is treated specially, incl. special precaution with -child's PID discovered as 1 elsewhere. - -v2: courtesy of Yan Gao of SUSE for early discovery and report for - what's primarily solved with 4/4 commit, in extension, child - daemons in the initialization phase coinciding with IPC-feasibility - based process scan in pacemakerd in a way that those are missed - (although they are to come up fully just moments later only - to interfere with naturally spawned ones) are now considered so - that if any native children later fail for said clash, the - pre-existing counterpart may get adopted instead of ending up - with repeated spawn-bury loop ad nauseam without real progress - (note that PCMK_fail_fast=true could possibly help, but that's - rather a big hammer not suitable for all the use cases, not - the ones we try to deal with gracefully here) ---- - mcp/pacemaker.c | 431 +++++++++++++++++++++++++++++++++++++++++++++++--------- - 1 file changed, 362 insertions(+), 69 deletions(-) - -diff --git a/mcp/pacemaker.c b/mcp/pacemaker.c -index 2986be6..86df216 100644 ---- a/mcp/pacemaker.c -+++ b/mcp/pacemaker.c -@@ -1,5 +1,7 @@ - /* -- * Copyright 2010-2018 Andrew Beekhof -+ * Copyright 2010-2019 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. - * - * This source code is licensed under the GNU General Public License version 2 - * or later (GPLv2+) WITHOUT ANY WARRANTY. -@@ -10,17 +12,23 @@ - - #include - #include -+#include - #include - #include - #include - #include - #include - -+#include /* indirectly: CRM_EX_* */ -+#include /* cib_channel_ro */ - #include - #include - #include - #include - #include -+ -+#include /* PCMK__SPECIAL_PID*, ... */ -+ - #ifdef SUPPORT_COROSYNC - #include - #endif -@@ -31,6 +39,7 @@ - gboolean pcmk_quorate = FALSE; - gboolean fatal_error = FALSE; - GMainLoop *mainloop = NULL; -+static bool global_keep_tracking = false; - - #define PCMK_PROCESS_CHECK_INTERVAL 5 - -@@ -48,6 +57,7 @@ typedef struct pcmk_child_s { - const char *name; - const char *uid; - const char *command; -+ const char *endpoint; /* IPC server name */ - - gboolean active_before_startup; - } pcmk_child_t; -@@ -59,17 +69,35 @@ typedef struct pcmk_child_s { - static pcmk_child_t pcmk_children[] = { - { 0, crm_proc_none, 0, 0, FALSE, "none", NULL, NULL }, - { 0, crm_proc_plugin, 0, 0, FALSE, "ais", NULL, NULL }, -- { 0, crm_proc_lrmd, 3, 0, TRUE, "lrmd", NULL, CRM_DAEMON_DIR"/lrmd" }, -- { 0, crm_proc_cib, 1, 0, TRUE, "cib", CRM_DAEMON_USER, CRM_DAEMON_DIR"/cib" }, -- { 0, crm_proc_crmd, 6, 0, TRUE, "crmd", CRM_DAEMON_USER, CRM_DAEMON_DIR"/crmd" }, -- { 0, crm_proc_attrd, 4, 0, TRUE, "attrd", CRM_DAEMON_USER, CRM_DAEMON_DIR"/attrd" }, -- { 0, crm_proc_stonithd, 0, 0, TRUE, "stonithd", NULL, NULL }, -- { 0, crm_proc_pe, 5, 0, TRUE, "pengine", CRM_DAEMON_USER, CRM_DAEMON_DIR"/pengine" }, -- { 0, crm_proc_mgmtd, 0, 0, TRUE, "mgmtd", NULL, HB_DAEMON_DIR"/mgmtd" }, -- { 0, crm_proc_stonith_ng, 2, 0, TRUE, "stonith-ng", NULL, CRM_DAEMON_DIR"/stonithd" }, -+ { 0, crm_proc_lrmd, 3, 0, TRUE, "lrmd", NULL, CRM_DAEMON_DIR"/lrmd", -+ CRM_SYSTEM_LRMD -+ }, -+ { 0, crm_proc_cib, 1, 0, TRUE, "cib", CRM_DAEMON_USER, CRM_DAEMON_DIR"/cib", -+ cib_channel_ro -+ }, -+ { 0, crm_proc_crmd, 6, 0, TRUE, "crmd", CRM_DAEMON_USER, CRM_DAEMON_DIR"/crmd", -+ CRM_SYSTEM_CRMD -+ }, -+ { 0, crm_proc_attrd, 4, 0, TRUE, "attrd", CRM_DAEMON_USER, CRM_DAEMON_DIR"/attrd", -+ T_ATTRD -+ }, -+ { 0, crm_proc_stonithd, 0, 0, TRUE, "stonithd", NULL, NULL, -+ NULL -+ }, -+ { 0, crm_proc_pe, 5, 0, TRUE, "pengine", CRM_DAEMON_USER, CRM_DAEMON_DIR"/pengine", -+ CRM_SYSTEM_PENGINE -+ }, -+ { 0, crm_proc_mgmtd, 0, 0, TRUE, "mgmtd", NULL, HB_DAEMON_DIR"/mgmtd", -+ NULL -+ }, -+ { 0, crm_proc_stonith_ng, 2, 0, TRUE, "stonith-ng", NULL, CRM_DAEMON_DIR"/stonithd", -+ "stonith-ng" -+ }, - }; - /* *INDENT-ON* */ - -+static gboolean check_active_before_startup_processes(gpointer user_data); -+static int pcmk_child_active(pcmk_child_t *child); - static gboolean start_child(pcmk_child_t * child); - void update_process_clients(crm_client_t *client); - void update_process_peers(void); -@@ -131,14 +159,31 @@ pcmk_process_exit(pcmk_child_t * child) - } - - if (shutdown_trigger) { -+ /* resume step-wise shutdown (returned TRUE yields no parallelizing) */ - mainloop_set_trigger(shutdown_trigger); -+ /* intended to speed up propagating expected lay-off of the daemons? */ - update_node_processes(local_nodeid, NULL, get_process_list()); - -- } else if (child->respawn && crm_is_true(getenv("PCMK_fail_fast"))) { -+ } else if (!child->respawn) { -+ /* nothing to do */ -+ -+ } else if (crm_is_true(getenv("PCMK_fail_fast"))) { - crm_err("Rebooting system because of %s", child->name); - pcmk_panic(__FUNCTION__); - -- } else if (child->respawn) { -+ } else if (pcmk_child_active(child) == 1) { -+ crm_warn("One-off suppressing strict respawning of a child process %s," -+ " appears alright per %s IPC end-point", -+ child->name, child->endpoint); -+ /* need to monitor how it evolves, and start new process if badly */ -+ child->active_before_startup = TRUE; -+ if (!global_keep_tracking) { -+ global_keep_tracking = true; -+ g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL, -+ check_active_before_startup_processes, NULL); -+ } -+ -+ } else { - crm_notice("Respawning failed child process: %s", child->name); - start_child(child); - } -@@ -215,8 +260,13 @@ stop_child(pcmk_child_t * child, int signal) - signal = SIGTERM; - } - -- if (child->command == NULL) { -- crm_debug("Nothing to do for child \"%s\"", child->name); -+ /* why to skip PID of 1? -+ - FreeBSD ~ how untrackable process behind IPC is masqueraded as -+ - elsewhere: how "init" task is designated; in particular, in systemd -+ arrangement of socket-based activation, this is pretty real */ -+ if (child->command == NULL || child->pid == PCMK__SPECIAL_PID) { -+ crm_debug("Nothing to do for child \"%s\" (process %lld)", -+ child->name, (long long) PCMK__SPECIAL_PID_AS_0(child->pid)); - return TRUE; - } - -@@ -241,6 +291,11 @@ stop_child(pcmk_child_t * child, int signal) - static char *opts_default[] = { NULL, NULL }; - static char *opts_vgrind[] = { NULL, NULL, NULL, NULL, NULL }; - -+/* TODO once libqb is taught to juggle with IPC end-points carried over as -+ bare file descriptor (https://github.com/ClusterLabs/libqb/issues/325) -+ it shall hand over these descriptors here if/once they are successfully -+ pre-opened in (presumably) pcmk_child_active, to avoid any remaining -+ room for races */ - static gboolean - start_child(pcmk_child_t * child) - { -@@ -371,7 +426,10 @@ escalate_shutdown(gpointer data) - - pcmk_child_t *child = data; - -- if (child->pid) { -+ if (child->pid == PCMK__SPECIAL_PID) { -+ pcmk_process_exit(child); -+ -+ } else if (child->pid) { - /* Use SIGSEGV instead of SIGKILL to create a core so we can see what it was up to */ - crm_err("Child %s not terminating in a timely manner, forcing", child->name); - stop_child(child, SIGSEGV); -@@ -379,6 +437,8 @@ escalate_shutdown(gpointer data) - return FALSE; - } - -+#define SHUTDOWN_ESCALATION_PERIOD 180000 /* 3m */ -+ - static gboolean - pcmk_shutdown_worker(gpointer user_data) - { -@@ -407,11 +467,24 @@ pcmk_shutdown_worker(gpointer user_data) - time_t now = time(NULL); - - if (child->respawn) { -+ if (child->pid == PCMK__SPECIAL_PID) { -+ crm_warn("The process behind %s IPC cannot be" -+ " terminated, so either wait the graceful" -+ " period of %ld s for its native termination" -+ " if it vitally depends on some other daemons" -+ " going down in a controlled way already," -+ " or locate and kill the correct %s process" -+ " on your own; set PCMK_fail_fast=1 to avoid" -+ " this altogether next time around", -+ child->name, (long) SHUTDOWN_ESCALATION_PERIOD, -+ child->command); -+ } - next_log = now + 30; - child->respawn = FALSE; - stop_child(child, SIGTERM); - if (phase < pcmk_children[pcmk_child_crmd].start_seq) { -- g_timeout_add(180000 /* 3m */ , escalate_shutdown, child); -+ g_timeout_add(SHUTDOWN_ESCALATION_PERIOD, -+ escalate_shutdown, child); - } - - } else if (now >= next_log) { -@@ -696,7 +769,106 @@ mcp_chown(const char *path, uid_t uid, gid_t gid) - } - } - --#if SUPPORT_PROCFS -+/*! -+ * \internal -+ * \brief Check the liveness of the child based on IPC name and PID if tracked -+ * -+ * \param[inout] child Child tracked data -+ * -+ * \return 0 if no trace of child's liveness detected (while it is detectable -+ * to begin with, at least according to one of the two properties), -+ * 1 if everything is fine, 2 if it's up per PID, but not per IPC -+ * end-point (still starting?), -1 on error, and -2 when the child -+ * (its IPC) blocked with an unauthorized process (log message -+ * emitted in both latter cases) -+ * -+ * \note This function doesn't modify any of \p child members but \c pid, -+ * and is not actively toying with processes as such but invoking -+ * \c stop_child in one particular case (there's for some reason -+ * a different authentic holder of the IPC end-point). -+ */ -+static int -+pcmk_child_active(pcmk_child_t *child) { -+ static uid_t cl_uid = 0; -+ static gid_t cl_gid = 0; -+ const uid_t root_uid = 0; -+ const gid_t root_gid = 0; -+ const uid_t *ref_uid; -+ const gid_t *ref_gid; -+ int ret = 0; -+ pid_t ipc_pid = 0; -+ const char *use_name; -+ -+ if (child->endpoint == NULL -+ && (child->pid <= 0 || child->pid == PCMK__SPECIAL_PID)) { -+ crm_err("Cannot track child %s for missing both API end-point and PID", -+ child->name); -+ ret = -1; /* misuse of the function when child is not trackable */ -+ -+ } else if (child->endpoint != NULL) { -+ -+ ref_uid = (child->uid != NULL) ? &cl_uid : &root_uid; -+ ref_gid = (child->uid != NULL) ? &cl_gid : &root_gid; -+ -+ if (child->uid != NULL && !cl_uid && !cl_gid -+ && crm_user_lookup(CRM_DAEMON_USER, &cl_uid, &cl_gid) < 0) { -+ crm_err("Could not find user and group IDs for user %s", -+ CRM_DAEMON_USER); -+ ret = -1; -+ } else if ((ret = pcmk__ipc_is_authentic_process_active(child->endpoint, -+ *ref_uid, *ref_gid, -+ &ipc_pid)) < 0) { -+ /* game over */ -+ } else if (child->pid <= 0) { -+ /* hit new child to be initialized, or reset to zero -+ and investigate further for ret == 0 */ -+ child->pid = ipc_pid; -+ } else if (ipc_pid && child->pid != ipc_pid) { -+ /* ultimately strange for ret == 1; either way, investigate */ -+ ret = 0; -+ } -+ } -+ -+ if (!ret) { -+ use_name = (child->flag == crm_proc_stonith_ng) -+ ? "stonithd" /* compensate "simplification" 61fc951e9 */ -+ : child->name; -+ /* when no IPC based liveness detected (incl. if ever a child without -+ IPC is tracked), or detected for a different _authentic_ process; -+ safe on FreeBSD since the only change possible from a proper child's -+ PID into "special" PID of 1 behind more loosely related process */ -+ ret = crm_pid_active(child->pid, use_name); -+ if (ipc_pid && (ret != 1 -+ || ipc_pid == PCMK__SPECIAL_PID -+ || crm_pid_active(ipc_pid, use_name) == 1)) { -+ if (ret == 1) { -+ /* assume there's no forking-while-retaining-IPC-socket -+ involved in the "children's" lifecycle, hence that the -+ tracking got out of sync purely because of some external -+ (esotheric?) forces (user initiated process "refresh" by -+ force? or intentionally racing on start-up, even?), and -+ that switching over to this other detected, authentic -+ instance with an IPC already in possession is a better -+ trade-off than "neutralizing" it first so as to give -+ either the original or possibly a new to-be-spawned -+ daemon process a leeway for operation, which would -+ otherwise have to be carried out */ -+ /* not possessing IPC, afterall (what about corosync CPG?) */ -+ stop_child(child, SIGKILL); -+ } else { -+ ret = 1; -+ } -+ child->pid = ipc_pid; -+ } else if (ret == 1) { -+ ret = 2; /* up per PID, but not per IPC (still starting?) */ -+ } else if (!child->pid && ret == -1) { -+ ret = 0; /* correct -1 on FreeBSD from above back to 0 */ -+ } -+ } -+ -+ return ret; -+} -+ - static gboolean - check_active_before_startup_processes(gpointer user_data) - { -@@ -713,15 +885,41 @@ check_active_before_startup_processes(gpointer user_data) - continue; - } else { - const char *name = pcmk_children[lpc].name; -- if (pcmk_children[lpc].flag == crm_proc_stonith_ng) { -- name = "stonithd"; -- } -- -- if (crm_pid_active(pcmk_children[lpc].pid, name) != 1) { -- crm_notice("Process %s terminated (pid=%d)", -- name, pcmk_children[lpc].pid); -- pcmk_process_exit(&(pcmk_children[lpc])); -- continue; -+ int ret; -+ -+ switch ((ret = pcmk_child_active(&pcmk_children[lpc]))) { -+ case 1: -+ break; -+ case 0: -+ case 2: /* this very case: it was OK once already */ -+ if (pcmk_children[lpc].respawn == TRUE) { -+ /* presumably after crash, hence critical */ -+ crm_crit("Process %s terminated (pid=%lld)%s", \ -+ name, (long long) -+ PCMK__SPECIAL_PID_AS_0(pcmk_children[lpc].pid), -+ ret ? ", at least per IPC end-point that went AWOL" -+ : ""); -+ } else { -+ /* orderly shutdown */ -+ crm_notice("Process %s terminated (pid=%lld)%s", \ -+ name, (long long) -+ PCMK__SPECIAL_PID_AS_0(pcmk_children[lpc].pid), -+ ret ? ", at least per IPC end-point that went AWOL" -+ : ""); -+ } -+ pcmk_process_exit(&(pcmk_children[lpc])); -+ continue; -+ default: -+ crm_crit("Unexpected value from pcmk_child_active:" -+ " %d (pid=%lld)", ret, -+ (long long) PCMK__SPECIAL_PID_AS_0( -+ pcmk_children[lpc].pid)); -+ /* fall through */ -+ case -1: -+ case -2: -+ /* message(s) already emitted */ -+ crm_exit(DAEMON_RESPAWN_STOP); -+ break; /* static analysis/noreturn */ - } - } - /* at least one of the processes found at startup -@@ -730,61 +928,147 @@ check_active_before_startup_processes(gpointer user_data) - } - } - -+ global_keep_tracking = keep_tracking; - return keep_tracking; - } --#endif // SUPPORT_PROCFS - --static void -+/*! -+ * \internal -+ * \brief Initial one-off check of the pre-existing "child" processes -+ * -+ * With "child" process, we mean the subdaemon that defines an API end-point -+ * (all of them do as of the comment) -- the possible complement is skipped -+ * as it is deemed it has no such shared resources to cause conflicts about, -+ * hence it can presumably be started anew without hesitation. -+ * If that won't hold true in the future, the concept of a shared resource -+ * will have to be generalized beyond the API end-point. -+ * -+ * For boundary cases that the "child" is still starting (IPC end-point is yet -+ * to be witnessed), or more rarely (practically FreeBSD only), when there's -+ * a pre-existing "untrackable" authentic process, we give the situation some -+ * time to possibly unfold in the right direction, meaning that said socket -+ * will appear or the unattainable process will disappear per the observable -+ * IPC, respectively. -+ * -+ * \return 0 if no such "child" process found, positive number X when X -+ * "children" detected, -1 on an internal error, -2 when any -+ * would-be-used IPC is blocked with an unauthorized process -+ * -+ * \note Since this gets run at the very start, \c respawn_count fields -+ * for particular children get temporarily overloaded with "rounds -+ * of waiting" tracking, restored once we are about to finish with -+ * success (i.e. returning value >=0) and will remain unrestored -+ * otherwise. One way to suppress liveness detection logic for -+ * particular child is to set the said value to a negative number. -+ */ -+#define WAIT_TRIES 4 /* together with interleaved sleeps, worst case ~ 1s */ -+static int - find_and_track_existing_processes(void) - { --#if SUPPORT_PROCFS -- DIR *dp; -- struct dirent *entry; -- bool start_tracker = FALSE; -- char entry_name[64]; -- -- dp = opendir("/proc"); -- if (!dp) { -- /* no proc directory to search through */ -- crm_notice("Can not read /proc directory to track existing components"); -- return; -- } -- -- while ((entry = readdir(dp)) != NULL) { -- int pid; -- int max = SIZEOF(pcmk_children); -- int i; -- -- if (crm_procfs_process_info(entry, entry_name, &pid) < 0) { -- continue; -- } -- for (i = 0; i < max; i++) { -- const char *name = pcmk_children[i].name; -- -- if (pcmk_children[i].start_seq == 0) { -+ unsigned tracking = 0U; -+ bool wait_in_progress; -+ int cur; -+ size_t i, rounds; -+ -+ for (rounds = 1; rounds <= WAIT_TRIES; rounds++) { -+ wait_in_progress = false; -+ for (i = 0; i < SIZEOF(pcmk_children); i++) { -+ if (!pcmk_children[i].endpoint -+ || pcmk_children[i].respawn_count < 0 -+ || !(cur = pcmk_child_active(&pcmk_children[i]))) { -+ /* as a speculation, don't give up in the context of -+ pcmk_child_active check if there are more rounds to -+ come for other reasons, but don't artificially wait just -+ because of this, since we would preferably start ASAP */ - continue; - } -- if (pcmk_children[i].flag == crm_proc_stonith_ng) { -- name = "stonithd"; -- } -- if (safe_str_eq(entry_name, name) && (crm_pid_active(pid, NULL) == 1)) { -- crm_notice("Tracking existing %s process (pid=%d)", name, pid); -- pcmk_children[i].pid = pid; -- pcmk_children[i].active_before_startup = TRUE; -- start_tracker = TRUE; -- break; -+ pcmk_children[i].respawn_count = rounds; -+ switch (cur) { -+ case 1: -+ if (pcmk_children[i].pid == PCMK__SPECIAL_PID) { -+ if (crm_is_true(getenv("PCMK_fail_fast"))) { -+ crm_crit("Cannot reliably track pre-existing" -+ " authentic process behind %s IPC on this" -+ " platform and PCMK_fail_fast requested", -+ pcmk_children[i].endpoint); -+ return -1; -+ } else if (pcmk_children[i].respawn_count == WAIT_TRIES) { -+ crm_notice("Assuming pre-existing authentic, though" -+ " on this platform untrackable, process" -+ " behind %s IPC is stable (was in %d" -+ " previous samples) so rather than" -+ " bailing out (PCMK_fail_fast not" -+ " requested), we just switch to a less" -+ " optimal IPC liveness monitoring" -+ " (not very suitable for heavy load)", -+ pcmk_children[i].name, WAIT_TRIES - 1); -+ crm_warn("The process behind %s IPC cannot be" -+ " terminated, so the overall shutdown" -+ " will get delayed implicitly (%ld s)," -+ " which serves as a graceful period for" -+ " its native termination if it vitally" -+ " depends on some other daemons going" -+ " down in a controlled way already", -+ pcmk_children[i].name, -+ (long) SHUTDOWN_ESCALATION_PERIOD); -+ } else { -+ wait_in_progress = true; -+ crm_warn("Cannot reliably track pre-existing" -+ " authentic process behind %s IPC on this" -+ " platform, can still disappear in %d" -+ " attempt(s)", pcmk_children[i].endpoint, -+ WAIT_TRIES - pcmk_children[i].respawn_count); -+ continue; -+ } -+ } -+ crm_notice("Tracking existing %s process (pid=%lld)", -+ pcmk_children[i].name, -+ (long long) PCMK__SPECIAL_PID_AS_0( -+ pcmk_children[i].pid)); -+ pcmk_children[i].respawn_count = -1; /* 0~keep watching */ -+ pcmk_children[i].active_before_startup = TRUE; -+ tracking++; -+ break; -+ case 2: -+ if (pcmk_children[i].respawn_count == WAIT_TRIES) { -+ crm_crit("%s IPC end-point for existing authentic" -+ " process %lld did not (re)appear", -+ pcmk_children[i].endpoint, -+ (long long) PCMK__SPECIAL_PID_AS_0( -+ pcmk_children[i].pid)); -+ return -1; -+ } -+ wait_in_progress = true; -+ crm_warn("Cannot find %s IPC end-point for existing" -+ " authentic process %lld, can still (re)appear" -+ " in %d attempts (?)", -+ pcmk_children[i].endpoint, -+ (long long) PCMK__SPECIAL_PID_AS_0( -+ pcmk_children[i].pid), -+ WAIT_TRIES - pcmk_children[i].respawn_count); -+ continue; -+ case -1: -+ case -2: -+ return cur; /* messages already emitted */ -+ default: -+ crm_crit("Unexpected condition"CRM_XS"cur=%d", cur); -+ return -1; /* unexpected condition */ - } - } -+ if (!wait_in_progress) { -+ break; -+ } -+ (void) poll(NULL, 0, 250); /* a bit for changes to possibly happen */ -+ } -+ for (i = 0; i < SIZEOF(pcmk_children); i++) { -+ pcmk_children[i].respawn_count = 0; /* restore pristine state */ - } - -- if (start_tracker) { -- g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL, check_active_before_startup_processes, -- NULL); -+ if (tracking) { -+ g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL, -+ check_active_before_startup_processes, NULL); - } -- closedir(dp); --#else -- crm_notice("No procfs support, so skipping check for existing components"); --#endif // SUPPORT_PROCFS -+ return (tracking > INT_MAX) ? INT_MAX : tracking; - } - - static void -@@ -1106,7 +1390,16 @@ main(int argc, char **argv) - setenv("PCMK_watchdog", "false", 1); - } - -- find_and_track_existing_processes(); -+ switch (find_and_track_existing_processes()) { -+ case -1: -+ crm_crit("Internal fatality, see the log"); -+ crm_exit(DAEMON_RESPAWN_STOP); -+ case -2: -+ crm_crit("Blocked by foreign process, kill the offender"); -+ crm_exit(ENOLCK); -+ default: -+ break; -+ }; - - cluster.destroy = mcp_cpg_destroy; - cluster.cpg.cpg_deliver_fn = mcp_cpg_deliver; --- -1.8.3.1 - - -From 07a82c5c8f9d60989ea88c5a3cc316ee290ea784 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Tue, 16 Apr 2019 00:04:57 +0200 -Subject: [PATCH 5/7] High: pacemakerd vs. IPC/procfs confused deputy - authenticity issue (3/4) - -[3/4: other daemons to authenticate IPC servers of fellow processes] - -Now that CVE-2018-16877 issue alone is still only partially covered -based on the preceding commits in the set, put the server-by-client -authentication (enabled and 1/3 and partially sported in 2/3) into -practice widely amongst the communicating pacemaker child daemons and -towards CPG API provided by 3rd party but principally using the same -underlying IPC mechanism facilitated by libqb, and consequently close -the remaining "big gap". - -As a small justification to introducing yet another "return -value" int variable, type-correctness is restored for those -that shall be cs_error_t to begin with. ---- - lib/cluster/corosync.c | 178 +++++++++++++++++++++++++++++++++++++++++-------- - lib/cluster/cpg.c | 94 +++++++++++++++++++------- - lib/common/ipc.c | 43 +++++++++++- - mcp/corosync.c | 76 ++++++++++++++++----- - 4 files changed, 320 insertions(+), 71 deletions(-) - -diff --git a/lib/cluster/corosync.c b/lib/cluster/corosync.c -index 9719541..0acf9b2 100644 ---- a/lib/cluster/corosync.c -+++ b/lib/cluster/corosync.c -@@ -1,19 +1,10 @@ - /* -- * Copyright (C) 2004 Andrew Beekhof -+ * Copyright 2004-2019 the Pacemaker project contributors - * -- * This library is free software; you can redistribute it and/or -- * modify it under the terms of the GNU Lesser General Public -- * License as published by the Free Software Foundation; either -- * version 2.1 of the License, or (at your option) any later version. -+ * The version control history for this file may have further details. - * -- * This library is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- * Lesser General Public License for more details. -- * -- * You should have received a copy of the GNU Lesser General Public -- * License along with this library; if not, write to the Free Software -- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. - */ - - #include -@@ -40,6 +31,8 @@ - - #include - -+#include /* PCMK__SPECIAL_PID* */ -+ - quorum_handle_t pcmk_quorum_handle = 0; - - gboolean(*quorum_app_callback) (unsigned long long seq, gboolean quorate) = NULL; -@@ -52,10 +45,15 @@ char * - corosync_node_name(uint64_t /*cmap_handle_t */ cmap_handle, uint32_t nodeid) - { - int lpc = 0; -- int rc = CS_OK; -+ cs_error_t rc = CS_OK; - int retries = 0; - char *name = NULL; - cmap_handle_t local_handle = 0; -+ int fd = -1; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - /* nodeid == 0 == CMAN_NODEID_US */ - if (nodeid == 0) { -@@ -85,6 +83,27 @@ corosync_node_name(uint64_t /*cmap_handle_t */ cmap_handle, uint32_t nodeid) - - if (cmap_handle == 0) { - cmap_handle = local_handle; -+ -+ rc = cmap_fd_get(cmap_handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CMAP API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CMAP provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CMAP provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CMAP provider: %s (%d)", -+ strerror(-rv), -rv); -+ goto bail; -+ } - } - - while (name == NULL && cmap_handle != 0) { -@@ -126,6 +145,7 @@ corosync_node_name(uint64_t /*cmap_handle_t */ cmap_handle, uint32_t nodeid) - lpc++; - } - -+bail: - if(local_handle) { - cmap_finalize(local_handle); - } -@@ -249,11 +269,15 @@ gboolean - cluster_connect_quorum(gboolean(*dispatch) (unsigned long long, gboolean), - void (*destroy) (gpointer)) - { -- int rc = -1; -+ cs_error_t rc; - int fd = 0; - int quorate = 0; - uint32_t quorum_type = 0; - struct mainloop_fd_callbacks quorum_fd_callbacks; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - quorum_fd_callbacks.dispatch = pcmk_quorum_dispatch; - quorum_fd_callbacks.destroy = destroy; -@@ -262,7 +286,8 @@ cluster_connect_quorum(gboolean(*dispatch) (unsigned long long, gboolean), - - rc = quorum_initialize(&pcmk_quorum_handle, &quorum_callbacks, &quorum_type); - if (rc != CS_OK) { -- crm_err("Could not connect to the Quorum API: %d", rc); -+ crm_err("Could not connect to the Quorum API: %s (%d)", -+ cs_strerror(rc), rc); - goto bail; - - } else if (quorum_type != QUORUM_SET) { -@@ -270,6 +295,29 @@ cluster_connect_quorum(gboolean(*dispatch) (unsigned long long, gboolean), - goto bail; - } - -+ rc = quorum_fd_get(pcmk_quorum_handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the Quorum API connection: %s (%d)", -+ strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* Quorum provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("Quorum provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ rc = CS_ERR_ACCESS; -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of Quorum provider: %s (%d)", -+ strerror(-rv), -rv); -+ rc = CS_ERR_ACCESS; -+ goto bail; -+ } -+ - rc = quorum_getquorate(pcmk_quorum_handle, &quorate); - if (rc != CS_OK) { - crm_err("Could not obtain the current Quorum API state: %d", rc); -@@ -290,12 +338,6 @@ cluster_connect_quorum(gboolean(*dispatch) (unsigned long long, gboolean), - goto bail; - } - -- rc = quorum_fd_get(pcmk_quorum_handle, &fd); -- if (rc != CS_OK) { -- crm_err("Could not obtain the Quorum API connection: %d", rc); -- goto bail; -- } -- - mainloop_add_fd("quorum", G_PRIORITY_HIGH, fd, dispatch, &quorum_fd_callbacks); - - corosync_initialize_nodelist(NULL, FALSE, NULL); -@@ -486,10 +528,15 @@ gboolean - corosync_initialize_nodelist(void *cluster, gboolean force_member, xmlNode * xml_parent) - { - int lpc = 0; -- int rc = CS_OK; -+ cs_error_t rc = CS_OK; - int retries = 0; - gboolean any = FALSE; - cmap_handle_t cmap_handle; -+ int fd = -1; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - do { - rc = cmap_initialize(&cmap_handle); -@@ -507,6 +554,27 @@ corosync_initialize_nodelist(void *cluster, gboolean force_member, xmlNode * xml - return FALSE; - } - -+ rc = cmap_fd_get(cmap_handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CMAP API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CMAP provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CMAP provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CMAP provider: %s (%d)", -+ strerror(-rv), -rv); -+ goto bail; -+ } -+ - crm_peer_init(); - crm_trace("Initializing corosync nodelist"); - for (lpc = 0; TRUE; lpc++) { -@@ -560,6 +628,7 @@ corosync_initialize_nodelist(void *cluster, gboolean force_member, xmlNode * xml - - free(name); - } -+bail: - cmap_finalize(cmap_handle); - return any; - } -@@ -569,36 +638,68 @@ corosync_cluster_name(void) - { - cmap_handle_t handle; - char *cluster_name = NULL; -- int rc = CS_OK; -+ cs_error_t rc = CS_OK; -+ int fd = -1; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - rc = cmap_initialize(&handle); - if (rc != CS_OK) { -- crm_info("Failed to initialize the cmap API: %s (%d)", ais_error2text(rc), rc); -+ crm_info("Failed to initialize the cmap API: %s (%d)", -+ cs_strerror(rc), rc); - return NULL; - } - -+ rc = cmap_fd_get(handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CMAP API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CMAP provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CMAP provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CMAP provider: %s (%d)", -+ strerror(-rv), -rv); -+ goto bail; -+ } -+ - rc = cmap_get_string(handle, "totem.cluster_name", &cluster_name); - if (rc != CS_OK) { -- crm_info("Cannot get totem.cluster_name: %s (%d)", ais_error2text(rc), rc); -+ crm_info("Cannot get totem.cluster_name: %s (%d)", cs_strerror(rc), rc); - - } else { - crm_debug("cmap totem.cluster_name = '%s'", cluster_name); - } - -+bail: - cmap_finalize(handle); -- - return cluster_name; - } - - int - corosync_cmap_has_config(const char *prefix) - { -- int rc = CS_OK; -+ cs_error_t rc = CS_OK; - int retries = 0; - static int found = -1; - cmap_handle_t cmap_handle; - cmap_iter_handle_t iter_handle; - char key_name[CMAP_KEYNAME_MAXLEN + 1]; -+ int fd = -1; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - if(found != -1) { - return found; -@@ -621,6 +722,27 @@ corosync_cmap_has_config(const char *prefix) - return -1; - } - -+ rc = cmap_fd_get(cmap_handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CMAP API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CMAP provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CMAP provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CMAP provider: %s (%d)", -+ strerror(-rv), -rv); -+ goto bail; -+ } -+ - rc = cmap_iter_init(cmap_handle, prefix, &iter_handle); - if (rc != CS_OK) { - crm_warn("Failed to initialize iteration for corosync cmap '%s': %s (rc=%d)", -diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c -index 1e6cf79..a61d492 100644 ---- a/lib/cluster/cpg.c -+++ b/lib/cluster/cpg.c -@@ -1,19 +1,10 @@ - /* -- * Copyright (C) 2004 Andrew Beekhof -+ * Copyright 2004-2019 the Pacemaker project contributors - * -- * This library is free software; you can redistribute it and/or -- * modify it under the terms of the GNU Lesser General Public -- * License as published by the Free Software Foundation; either -- * version 2.1 of the License, or (at your option) any later version. -+ * The version control history for this file may have further details. - * -- * This library is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- * Lesser General Public License for more details. -- * -- * You should have received a copy of the GNU Lesser General Public -- * License along with this library; if not, write to the Free Software -- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. - */ - - #include -@@ -38,6 +29,8 @@ - - #include - -+#include /* PCMK__SPECIAL_PID* */ -+ - cpg_handle_t pcmk_cpg_handle = 0; /* TODO: Remove, use cluster.cpg_handle */ - - static bool cpg_evicted = FALSE; -@@ -71,11 +64,16 @@ cluster_disconnect_cpg(crm_cluster_t *cluster) - - uint32_t get_local_nodeid(cpg_handle_t handle) - { -- int rc = CS_OK; -+ cs_error_t rc = CS_OK; - int retries = 0; - static uint32_t local_nodeid = 0; - cpg_handle_t local_handle = handle; - cpg_callbacks_t cb = { }; -+ int fd = -1; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - if(local_nodeid != 0) { - return local_nodeid; -@@ -92,6 +90,32 @@ uint32_t get_local_nodeid(cpg_handle_t handle) - if(handle == 0) { - crm_trace("Creating connection"); - cs_repeat(retries, 5, rc = cpg_initialize(&local_handle, &cb)); -+ if (rc != CS_OK) { -+ crm_err("Could not connect to the CPG API: %s (%d)", -+ cs_strerror(rc), rc); -+ return 0; -+ } -+ -+ rc = cpg_fd_get(local_handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CPG API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CPG provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CPG provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CPG provider: %s (%d)", -+ strerror(-rv), -rv); -+ goto bail; -+ } - } - - if (rc == CS_OK) { -@@ -103,6 +127,8 @@ uint32_t get_local_nodeid(cpg_handle_t handle) - if (rc != CS_OK) { - crm_err("Could not get local node id from the CPG API: %s (%d)", ais_error2text(rc), rc); - } -+ -+bail: - if(handle == 0) { - crm_trace("Closing connection"); - cpg_finalize(local_handle); -@@ -435,12 +461,16 @@ pcmk_cpg_membership(cpg_handle_t handle, - gboolean - cluster_connect_cpg(crm_cluster_t *cluster) - { -- int rc = -1; -- int fd = 0; -+ cs_error_t rc; -+ int fd = -1; - int retries = 0; - uint32_t id = 0; - crm_node_t *peer = NULL; - cpg_handle_t handle = 0; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - struct mainloop_fd_callbacks cpg_fd_callbacks = { - .dispatch = pcmk_cpg_dispatch, -@@ -465,7 +495,31 @@ cluster_connect_cpg(crm_cluster_t *cluster) - - cs_repeat(retries, 30, rc = cpg_initialize(&handle, &cpg_callbacks)); - if (rc != CS_OK) { -- crm_err("Could not connect to the Cluster Process Group API: %d", rc); -+ crm_err("Could not connect to the CPG API: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ rc = cpg_fd_get(handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CPG API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CPG provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CPG provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ rc = CS_ERR_ACCESS; -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CPG provider: %s (%d)", -+ strerror(-rv), -rv); -+ rc = CS_ERR_ACCESS; - goto bail; - } - -@@ -484,12 +538,6 @@ cluster_connect_cpg(crm_cluster_t *cluster) - goto bail; - } - -- rc = cpg_fd_get(handle, &fd); -- if (rc != CS_OK) { -- crm_err("Could not obtain the CPG API connection: %d", rc); -- goto bail; -- } -- - pcmk_cpg_handle = handle; - cluster->cpg_handle = handle; - mainloop_add_fd("corosync-cpg", G_PRIORITY_MEDIUM, fd, cluster, &cpg_fd_callbacks); -diff --git a/lib/common/ipc.c b/lib/common/ipc.c -index 5b47dd6..3e547f3 100644 ---- a/lib/common/ipc.c -+++ b/lib/common/ipc.c -@@ -916,11 +916,18 @@ crm_ipc_new(const char *name, size_t max_size) - * - * \param[in] client Connection instance obtained from crm_ipc_new() - * -- * \return TRUE on success, FALSE otherwise (in which case errno will be set) -+ * \return TRUE on success, FALSE otherwise (in which case errno will be set; -+ * specifically, in case of discovering the remote side is not -+ * authentic, its value is set to ECONNABORTED). - */ - bool - crm_ipc_connect(crm_ipc_t * client) - { -+ static uid_t cl_uid = 0; -+ static gid_t cl_gid = 0; -+ pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; -+ int rv; -+ - client->need_reply = FALSE; - client->ipc = qb_ipcc_connect(client->name, client->buf_size); - -@@ -931,7 +938,39 @@ crm_ipc_connect(crm_ipc_t * client) - - client->pfd.fd = crm_ipc_get_fd(client); - if (client->pfd.fd < 0) { -- crm_debug("Could not obtain file descriptor for %s connection: %s (%d)", client->name, pcmk_strerror(errno), errno); -+ rv = errno; -+ /* message already omitted */ -+ crm_ipc_close(client); -+ errno = rv; -+ return FALSE; -+ } -+ -+ if (!cl_uid && !cl_gid -+ && (rv = crm_user_lookup(CRM_DAEMON_USER, &cl_uid, &cl_gid)) < 0) { -+ errno = -rv; -+ /* message already omitted */ -+ crm_ipc_close(client); -+ errno = -rv; -+ return FALSE; -+ } -+ -+ if (!(rv = crm_ipc_is_authentic_process(client->pfd.fd, cl_uid, cl_gid, -+ &found_pid, &found_uid, -+ &found_gid))) { -+ crm_err("Daemon (IPC %s) is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ client->name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ crm_ipc_close(client); -+ errno = ECONNABORTED; -+ return FALSE; -+ -+ } else if (rv < 0) { -+ errno = -rv; -+ crm_perror(LOG_ERR, "Could not verify authenticity of daemon (IPC %s)", -+ client->name); -+ crm_ipc_close(client); -+ errno = -rv; - return FALSE; - } - -diff --git a/mcp/corosync.c b/mcp/corosync.c -index 7502da7..407a63f 100644 ---- a/mcp/corosync.c -+++ b/mcp/corosync.c -@@ -1,19 +1,10 @@ - /* -- * Copyright (C) 2010 Andrew Beekhof -+ * Copyright 2010-2019 the Pacemaker project contributors - * -- * This program is free software; you can redistribute it and/or -- * modify it under the terms of the GNU General Public -- * License as published by the Free Software Foundation; either -- * version 2 of the License, or (at your option) any later version. -+ * The version control history for this file may have further details. - * -- * This software is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- * General Public License for more details. -- * -- * You should have received a copy of the GNU General Public -- * License along with this library; if not, write to the Free Software -- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -+ * This source code is licensed under the GNU General Public License version 2 -+ * or later (GPLv2+) WITHOUT ANY WARRANTY. - */ - #include - #include -@@ -33,8 +24,11 @@ - #endif - - #include -+#include /* for crm_ipc_is_authentic_process */ - #include - -+#include /* PCMK__SPECIAL_PID* */ -+ - #if SUPPORT_CMAN - # include - #endif -@@ -111,7 +105,10 @@ gboolean - cluster_connect_cfg(uint32_t * nodeid) - { - cs_error_t rc; -- int fd = 0, retries = 0; -+ int fd = -1, retries = 0, rv; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; - - static struct mainloop_fd_callbacks cfg_fd_callbacks = { - .dispatch = pcmk_cfg_dispatch, -@@ -121,13 +118,27 @@ cluster_connect_cfg(uint32_t * nodeid) - cs_repeat(retries, 30, rc = corosync_cfg_initialize(&cfg_handle, &cfg_callbacks)); - - if (rc != CS_OK) { -- crm_err("corosync cfg init error %d", rc); -+ crm_err("corosync cfg init: %s (%d)", cs_strerror(rc), rc); - return FALSE; - } - - rc = corosync_cfg_fd_get(cfg_handle, &fd); - if (rc != CS_OK) { -- crm_err("corosync cfg fd_get error %d", rc); -+ crm_err("corosync cfg fd_get: %s (%d)", cs_strerror(rc), rc); -+ goto bail; -+ } -+ -+ /* CFG provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CFG provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ goto bail; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CFG provider: %s (%d)", -+ strerror(-rv), -rv); - goto bail; - } - -@@ -264,7 +275,7 @@ get_config_opt(uint64_t unused, cmap_handle_t object_handle, const char *key, ch - gboolean - mcp_read_config(void) - { -- int rc = CS_OK; -+ cs_error_t rc = CS_OK; - int retries = 0; - - const char *const_value = NULL; -@@ -287,11 +298,16 @@ mcp_read_config(void) - } else { - break; - } -- - } while (retries < 5); -+ - #elif HAVE_CMAP - cmap_handle_t local_handle; - uint64_t config = 0; -+ int fd = -1; -+ uid_t found_uid = 0; -+ gid_t found_gid = 0; -+ pid_t found_pid = 0; -+ int rv; - - /* There can be only one (possibility if confdb isn't around) */ - do { -@@ -315,6 +331,30 @@ mcp_read_config(void) - return FALSE; - } - -+ rc = cmap_fd_get(local_handle, &fd); -+ if (rc != CS_OK) { -+ crm_err("Could not obtain the CMAP API connection: %s (%d)", -+ cs_strerror(rc), rc); -+ cmap_finalize(local_handle); -+ return FALSE; -+ } -+ -+ /* CMAP provider run as root (in given user namespace, anyway)? */ -+ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, -+ &found_uid, &found_gid))) { -+ crm_err("CMAP provider is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ cmap_finalize(local_handle); -+ return FALSE; -+ } else if (rv < 0) { -+ crm_err("Could not verify authenticity of CMAP provider: %s (%d)", -+ strerror(-rv), -rv); -+ cmap_finalize(local_handle); -+ return FALSE; -+ } -+ - stack = get_cluster_type(); - crm_info("Reading configure for stack: %s", name_for_cluster_type(stack)); - --- -1.8.3.1 - - -From 4d6f6e01b309cda7b3f8fe791247566d247d8028 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Tue, 16 Apr 2019 00:08:28 +0200 -Subject: [PATCH 6/7] High: pacemakerd vs. IPC/procfs confused deputy - authenticity issue (4/4) - -[4/4: CPG users to be careful about now-more-probable rival processes] - -In essence, this comes down to pacemaker confusing at-node CPG members -with effectively the only plausible to co-exist at particular node, -which doesn't hold and asks for a wider reconciliation of this -reality-check. - -However, in practical terms, since there are two factors lowering the -priority of doing so: - -1/ possibly the only non-self-inflicted scenario is either that - some of the cluster stack processes fail -- this the problem - that shall rather be deferred to arranged node disarming/fencing - to stay on the safe side with 100% certainty, at the cost of - possibly long-lasting failover process at other nodes - (for other possibility, someone running some of these by accident - so they effectively become rival processes, it's like getting - hands cut when playing with a lawnmower in an unintended way) - -2/ for state tracking of the peer nodes, it may possibly cause troubles - in case the process observed as left wasn't the last for the - particular node, even if presumably just temporary, since the - situation may eventually resolve with imposed serialization of - the rival processes via API end-point singleton restriction (this - is also the most likely cause of why such non-final leave gets - observed in the first place), except in one case -- the legitimate - API end-point carrier won't possibly acknowledged as returned - by its peers, at least not immediately, unless it tries to join - anew, which verges on undefined behaviour (at least per corosync - documentation) - -we make do just with a light code change so as to - -* limit 1/ some more with in-daemon self-check for pre-existing - end-point existence (this is to complement the checks already made in - the parent daemon prior to spawning new instances, only some moments - later; note that we don't have any lock file etc. mechanisms to - prevent parallel runs of the same daemons, and people could run these - on their own deliberation), and to - -* guard against the interferences from the rivals at the same node - per 2/ with ignoring their non-final leave messages altogether. - -Note that CPG at this point is already expected to be authenticity-safe. - -Regarding now-more-probable part, we actually traded the inherently racy -procfs scanning for something (exactly that singleton mentioned above) -rather firm (and unfakeable), but we admittedly got lost track of -processes that are after CPG membership (that is, another form of -a shared state) prior to (or in non-deterministic order allowing for -the same) carring about publishing the end-point. - -Big thanks is owed to Yan Gao of SUSE, for early discovery and reporting -this discrepancy arising from the earlier commits in the set. ---- - attrd/main.c | 19 ++++++++++- - cib/main.c | 35 ++++++++++++--------- - crmd/main.c | 35 ++++++++++++--------- - fencing/main.c | 32 +++++++++++-------- - lib/cluster/cpg.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++----- - 5 files changed, 163 insertions(+), 52 deletions(-) - -diff --git a/attrd/main.c b/attrd/main.c -index 4cc15cc..e0a1e7c 100644 ---- a/attrd/main.c -+++ b/attrd/main.c -@@ -1,5 +1,7 @@ - /* -- * Copyright 2013-2019 Andrew Beekhof -+ * Copyright 2013-2019 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public -@@ -336,6 +338,7 @@ main(int argc, char **argv) - int index = 0; - int argerr = 0; - qb_ipcs_service_t *ipcs = NULL; -+ crm_ipc_t *old_instance = NULL; - - attrd_init_mainloop(); - crm_log_preinit(NULL, argc, argv); -@@ -372,6 +375,20 @@ main(int argc, char **argv) - - crm_log_init(T_ATTRD, LOG_INFO, TRUE, FALSE, argc, argv, FALSE); - crm_info("Starting up"); -+ -+ old_instance = crm_ipc_new(T_ATTRD, 0); -+ if (crm_ipc_connect(old_instance)) { -+ /* IPC end-point already up */ -+ crm_ipc_close(old_instance); -+ crm_ipc_destroy(old_instance); -+ crm_err("attrd is already active, aborting startup"); -+ crm_exit(EX_OK); -+ } else { -+ /* not up or not authentic, we'll proceed either way */ -+ crm_ipc_destroy(old_instance); -+ old_instance = NULL; -+ } -+ - attributes = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_attribute); - - attrd_exit_status = attrd_cluster_connect(); -diff --git a/cib/main.c b/cib/main.c -index 5473d40..7c745da 100644 ---- a/cib/main.c -+++ b/cib/main.c -@@ -1,19 +1,10 @@ - /* -- * Copyright (C) 2004 Andrew Beekhof -+ * Copyright 2004-2019 the Pacemaker project contributors - * -- * This program is free software; you can redistribute it and/or -- * modify it under the terms of the GNU General Public -- * License as published by the Free Software Foundation; either -- * version 2 of the License, or (at your option) any later version. -+ * The version control history for this file may have further details. - * -- * This software is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- * General Public License for more details. -- * -- * You should have received a copy of the GNU General Public -- * License along with this library; if not, write to the Free Software -- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -+ * This source code is licensed under the GNU General Public License version 2 -+ * or later (GPLv2+) WITHOUT ANY WARRANTY. - */ - - #include -@@ -135,13 +126,12 @@ main(int argc, char **argv) - int index = 0; - int argerr = 0; - struct passwd *pwentry = NULL; -+ crm_ipc_t *old_instance = NULL; - - crm_log_preinit(NULL, argc, argv); - crm_set_options(NULL, "[options]", - long_options, "Daemon for storing and replicating the cluster configuration"); - -- crm_peer_init(); -- - mainloop_add_signal(SIGTERM, cib_shutdown); - mainloop_add_signal(SIGPIPE, cib_enable_writes); - -@@ -216,6 +206,19 @@ main(int argc, char **argv) - - crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE); - -+ old_instance = crm_ipc_new(cib_channel_ro, 0); -+ if (crm_ipc_connect(old_instance)) { -+ /* IPC end-point already up */ -+ crm_ipc_close(old_instance); -+ crm_ipc_destroy(old_instance); -+ crm_err("cib is already active, aborting startup"); -+ crm_exit(EX_OK); -+ } else { -+ /* not up or not authentic, we'll proceed either way */ -+ crm_ipc_destroy(old_instance); -+ old_instance = NULL; -+ } -+ - if (cib_root == NULL) { - if ((g_file_test(CRM_CONFIG_DIR "/cib.xml", G_FILE_TEST_EXISTS) == FALSE) - && (g_file_test(CRM_LEGACY_CONFIG_DIR "/cib.xml", G_FILE_TEST_EXISTS) == TRUE)) { -@@ -238,6 +241,8 @@ main(int argc, char **argv) - return 100; - } - -+ crm_peer_init(); -+ - /* read local config file */ - rc = cib_init(); - -diff --git a/crmd/main.c b/crmd/main.c -index e8baa12..6eb7c03 100644 ---- a/crmd/main.c -+++ b/crmd/main.c -@@ -1,19 +1,10 @@ - /* -- * Copyright (C) 2004 Andrew Beekhof -- * -- * This program is free software; you can redistribute it and/or -- * modify it under the terms of the GNU General Public -- * License as published by the Free Software Foundation; either -- * version 2 of the License, or (at your option) any later version. -- * -- * This software is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- * General Public License for more details. -- * -- * You should have received a copy of the GNU General Public -- * License along with this library; if not, write to the Free Software -- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -+ * Copyright 2004-2019 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. -+ * -+ * This source code is licensed under the GNU General Public License version 2 -+ * or later (GPLv2+) WITHOUT ANY WARRANTY. - */ - - #include -@@ -61,6 +52,7 @@ main(int argc, char **argv) - int flag; - int index = 0; - int argerr = 0; -+ crm_ipc_t *old_instance = NULL; - - crmd_mainloop = g_main_new(FALSE); - crm_log_preinit(NULL, argc, argv); -@@ -104,6 +96,19 @@ main(int argc, char **argv) - crm_help('?', EX_USAGE); - } - -+ old_instance = crm_ipc_new(CRM_SYSTEM_CRMD, 0); -+ if (crm_ipc_connect(old_instance)) { -+ /* IPC end-point already up */ -+ crm_ipc_close(old_instance); -+ crm_ipc_destroy(old_instance); -+ crm_err("crmd is already active, aborting startup"); -+ crm_exit(EX_OK); -+ } else { -+ /* not up or not authentic, we'll proceed either way */ -+ crm_ipc_destroy(old_instance); -+ old_instance = NULL; -+ } -+ - if (pcmk__daemon_can_write(PE_STATE_DIR, NULL) == FALSE) { - crm_err("Terminating due to bad permissions on " PE_STATE_DIR); - fprintf(stderr, -diff --git a/fencing/main.c b/fencing/main.c -index 16663f6..c46c9a5 100644 ---- a/fencing/main.c -+++ b/fencing/main.c -@@ -1,19 +1,10 @@ - /* -- * Copyright (C) 2009 Andrew Beekhof -+ * Copyright 2009-2019 the Pacemaker project contributors - * -- * This program is free software; you can redistribute it and/or -- * modify it under the terms of the GNU General Public -- * License as published by the Free Software Foundation; either -- * version 2 of the License, or (at your option) any later version. -+ * The version control history for this file may have further details. - * -- * This software is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- * General Public License for more details. -- * -- * You should have received a copy of the GNU General Public -- * License along with this library; if not, write to the Free Software -- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -+ * This source code is licensed under the GNU General Public License version 2 -+ * or later (GPLv2+) WITHOUT ANY WARRANTY. - */ - - #include -@@ -1289,6 +1280,7 @@ main(int argc, char **argv) - int option_index = 0; - crm_cluster_t cluster; - const char *actions[] = { "reboot", "off", "on", "list", "monitor", "status" }; -+ crm_ipc_t *old_instance = NULL; - - crm_log_preinit("stonith-ng", argc, argv); - crm_set_options(NULL, "mode [options]", long_options, -@@ -1459,6 +1451,20 @@ main(int argc, char **argv) - } - - crm_log_init("stonith-ng", LOG_INFO, TRUE, FALSE, argc, argv, FALSE); -+ -+ old_instance = crm_ipc_new("stonith-ng", 0); -+ if (crm_ipc_connect(old_instance)) { -+ /* IPC end-point already up */ -+ crm_ipc_close(old_instance); -+ crm_ipc_destroy(old_instance); -+ crm_err("stonithd is already active, aborting startup"); -+ crm_exit(EX_OK); -+ } else { -+ /* not up or not authentic, we'll proceed either way */ -+ crm_ipc_destroy(old_instance); -+ old_instance = NULL; -+ } -+ - mainloop_add_signal(SIGTERM, stonith_shutdown); - - crm_peer_init(); -diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c -index a61d492..c5ecc67 100644 ---- a/lib/cluster/cpg.c -+++ b/lib/cluster/cpg.c -@@ -385,6 +385,20 @@ pcmk_message_common_cs(cpg_handle_t handle, uint32_t nodeid, uint32_t pid, void - return NULL; - } - -+static int cmp_member_list_nodeid(const void *first, -+ const void *second) -+{ -+ const struct cpg_address *const a = *((const struct cpg_address **) first), -+ *const b = *((const struct cpg_address **) second); -+ if (a->nodeid < b->nodeid) { -+ return -1; -+ } else if (a->nodeid > b->nodeid) { -+ return 1; -+ } -+ /* don't bother with "reason" nor "pid" */ -+ return 0; -+} -+ - void - pcmk_cpg_membership(cpg_handle_t handle, - const struct cpg_name *groupName, -@@ -396,29 +410,91 @@ pcmk_cpg_membership(cpg_handle_t handle, - gboolean found = FALSE; - static int counter = 0; - uint32_t local_nodeid = get_local_nodeid(handle); -+ const struct cpg_address *key, **rival, **sorted; -+ -+ sorted = malloc(member_list_entries * sizeof(const struct cpg_address *)); -+ CRM_ASSERT(sorted != NULL); -+ -+ for (size_t iter = 0; iter < member_list_entries; iter++) { -+ sorted[iter] = member_list + iter; -+ } -+ /* so that the cross-matching multiply-subscribed nodes is then cheap */ -+ qsort(sorted, member_list_entries, sizeof(const struct cpg_address *), -+ cmp_member_list_nodeid); - - for (i = 0; i < left_list_entries; i++) { - crm_node_t *peer = crm_find_peer(left_list[i].nodeid, NULL); - -- crm_info("Node %u left group %s (peer=%s, counter=%d.%d)", -+ crm_info("Node %u left group %s (peer=%s:%llu, counter=%d.%d)", - left_list[i].nodeid, groupName->value, -- (peer? peer->uname : ""), counter, i); -+ (peer? peer->uname : ""), -+ (unsigned long long) left_list[i].pid, counter, i); -+ -+ /* in CPG world, NODE:PROCESS-IN-MEMBERSHIP-OF-G is an 1:N relation -+ and not playing by this rule may go wild in case of multiple -+ residual instances of the same pacemaker daemon at the same node -+ -- we must ensure that the possible local rival(s) won't make us -+ cry out and bail (e.g. when they quit themselves), since all the -+ surrounding logic denies this simple fact that the full membership -+ is discriminated also per the PID of the process beside mere node -+ ID (and implicitly, group ID); practically, this will be sound in -+ terms of not preventing progress, since all the CPG joiners are -+ also API end-point carriers, and that's what matters locally -+ (who's the winner); -+ remotely, we will just compare leave_list and member_list and if -+ the left process has it's node retained in member_list (under some -+ other PID, anyway) we will just ignore it as well -+ XXX: long-term fix is to establish in-out PID-aware tracking? */ - if (peer) { -- crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, OFFLINESTATUS); -+ key = &left_list[i]; -+ rival = bsearch(&key, sorted, member_list_entries, -+ sizeof(const struct cpg_address *), -+ cmp_member_list_nodeid); -+ if (rival == NULL) { -+ crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, -+ OFFLINESTATUS); -+ } else if (left_list[i].nodeid == local_nodeid) { -+ crm_info("Ignoring the above event %s.%d, comes from a local" -+ " rival process (presumably not us): %llu", -+ groupName->value, counter, -+ (unsigned long long) left_list[i].pid); -+ } else { -+ crm_info("Ignoring the above event %s.%d, comes from" -+ " a rival-rich node: %llu (e.g. %llu process" -+ " carries on)", -+ groupName->value, counter, -+ (unsigned long long) left_list[i].pid, -+ (unsigned long long) (*rival)->pid); -+ } - } - } -+ free(sorted); -+ sorted = NULL; - - for (i = 0; i < joined_list_entries; i++) { -- crm_info("Node %u joined group %s (counter=%d.%d)", -- joined_list[i].nodeid, groupName->value, counter, i); -+ crm_info("Node %u joined group %s (counter=%d.%d, pid=%llu," -+ " unchecked for rivals)", -+ joined_list[i].nodeid, groupName->value, counter, i, -+ (unsigned long long) left_list[i].pid); - } - - for (i = 0; i < member_list_entries; i++) { - crm_node_t *peer = crm_get_peer(member_list[i].nodeid, NULL); - -- crm_info("Node %u still member of group %s (peer=%s, counter=%d.%d)", -+ crm_info("Node %u still member of group %s (peer=%s:%llu," -+ " counter=%d.%d, at least once)", - member_list[i].nodeid, groupName->value, -- (peer? peer->uname : ""), counter, i); -+ (peer? peer->uname : ""), member_list[i].pid, -+ counter, i); -+ -+ if (member_list[i].nodeid == local_nodeid -+ && member_list[i].pid != getpid()) { -+ /* see the note above */ -+ crm_info("Ignoring the above event %s.%d, comes from a local rival" -+ " process: %llu", groupName->value, counter, -+ (unsigned long long) member_list[i].pid); -+ continue; -+ } - - /* Anyone that is sending us CPG messages must also be a _CPG_ member. - * But it's _not_ safe to assume it's in the quorum membership. -@@ -438,7 +514,9 @@ pcmk_cpg_membership(cpg_handle_t handle, - * - * Set the threshold to 1 minute - */ -- crm_err("Node %s[%u] appears to be online even though we think it is dead", peer->uname, peer->id); -+ crm_err("Node %s[%u] appears to be online even though we think" -+ " it is dead (unchecked for rivals)", -+ peer->uname, peer->id); - if (crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_MEMBER, 0)) { - peer->votes = 0; - } --- -1.8.3.1 - - -From 9dc38d81cb6e1967c368faed78de1927cabf06b3 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= -Date: Wed, 17 Apr 2019 15:17:50 +0200 -Subject: [PATCH 7/7] Med: controld: fix possible NULL pointer dereference - -This is now more likely triggerable once the problems related to -CVE-2018-16878 are avoided. ---- - crmd/control.c | 32 +++++++++++++------------------- - 1 file changed, 13 insertions(+), 19 deletions(-) - -diff --git a/crmd/control.c b/crmd/control.c -index e01066a..488ea88 100644 ---- a/crmd/control.c -+++ b/crmd/control.c -@@ -1,19 +1,10 @@ - /* -- * Copyright (C) 2004 Andrew Beekhof -+ * Copyright 2004-2019 the Pacemaker project contributors - * -- * This program is free software; you can redistribute it and/or -- * modify it under the terms of the GNU General Public -- * License as published by the Free Software Foundation; either -- * version 2 of the License, or (at your option) any later version. -+ * The version control history for this file may have further details. - * -- * This software is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- * General Public License for more details. -- * -- * You should have received a copy of the GNU General Public -- * License along with this library; if not, write to the Free Software -- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -+ * This source code is licensed under the GNU General Public License version 2 -+ * or later (GPLv2+) WITHOUT ANY WARRANTY. - */ - - #include -@@ -127,12 +118,15 @@ do_ha_control(long long action, - } - #endif - } -- controld_election_init(cluster->uname); -- fsa_our_uname = cluster->uname; -- fsa_our_uuid = cluster->uuid; -- if(cluster->uuid == NULL) { -- crm_err("Could not obtain local uuid"); -- registered = FALSE; -+ -+ if (registered == TRUE) { -+ controld_election_init(cluster->uname); -+ fsa_our_uname = cluster->uname; -+ fsa_our_uuid = cluster->uuid; -+ if(cluster->uuid == NULL) { -+ crm_err("Could not obtain local uuid"); -+ registered = FALSE; -+ } - } - - if (registered == FALSE) { --- -1.8.3.1 - diff --git a/SOURCES/008-security-log.patch b/SOURCES/008-security-log.patch deleted file mode 100644 index 0fde849..0000000 --- a/SOURCES/008-security-log.patch +++ /dev/null @@ -1,297 +0,0 @@ -From 83811e2115f5516a7faec2e653b1be3d58b35a79 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 12 Apr 2019 09:46:51 -0500 -Subject: [PATCH 1/2] Log: libcrmcluster: improve CPG membership messages - -Show CPG event reason when provided by corosync, make messages more readable, -upgrade duplicate pid messages to warnings (and log only one message in those -cases). - -This also fixes a typo in 4d6f6e01 that led to using an index with the wrong -array, potentially leading to use of an uninitialized value or invalid memory -access. ---- - lib/cluster/cpg.c | 95 +++++++++++++++++++++++++++++++++---------------------- - 1 file changed, 58 insertions(+), 37 deletions(-) - -diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c -index c5ecc67..85476be 100644 ---- a/lib/cluster/cpg.c -+++ b/lib/cluster/cpg.c -@@ -399,6 +399,32 @@ static int cmp_member_list_nodeid(const void *first, - return 0; - } - -+static const char * -+cpgreason2str(cpg_reason_t reason) -+{ -+ switch (reason) { -+ case CPG_REASON_JOIN: return " via cpg_join"; -+ case CPG_REASON_LEAVE: return " via cpg_leave"; -+ case CPG_REASON_NODEDOWN: return " via cluster exit"; -+ case CPG_REASON_NODEUP: return " via cluster join"; -+ case CPG_REASON_PROCDOWN: return " for unknown reason"; -+ default: break; -+ } -+ return ""; -+} -+ -+static inline const char * -+peer_name(crm_node_t *peer) -+{ -+ if (peer == NULL) { -+ return "unknown node"; -+ } else if (peer->uname == NULL) { -+ return "peer node"; -+ } else { -+ return peer->uname; -+ } -+} -+ - void - pcmk_cpg_membership(cpg_handle_t handle, - const struct cpg_name *groupName, -@@ -410,7 +436,7 @@ pcmk_cpg_membership(cpg_handle_t handle, - gboolean found = FALSE; - static int counter = 0; - uint32_t local_nodeid = get_local_nodeid(handle); -- const struct cpg_address *key, **rival, **sorted; -+ const struct cpg_address *key, **sorted; - - sorted = malloc(member_list_entries * sizeof(const struct cpg_address *)); - CRM_ASSERT(sorted != NULL); -@@ -424,11 +450,7 @@ pcmk_cpg_membership(cpg_handle_t handle, - - for (i = 0; i < left_list_entries; i++) { - crm_node_t *peer = crm_find_peer(left_list[i].nodeid, NULL); -- -- crm_info("Node %u left group %s (peer=%s:%llu, counter=%d.%d)", -- left_list[i].nodeid, groupName->value, -- (peer? peer->uname : ""), -- (unsigned long long) left_list[i].pid, counter, i); -+ const struct cpg_address **rival = NULL; - - /* in CPG world, NODE:PROCESS-IN-MEMBERSHIP-OF-G is an 1:N relation - and not playing by this rule may go wild in case of multiple -@@ -442,7 +464,7 @@ pcmk_cpg_membership(cpg_handle_t handle, - also API end-point carriers, and that's what matters locally - (who's the winner); - remotely, we will just compare leave_list and member_list and if -- the left process has it's node retained in member_list (under some -+ the left process has its node retained in member_list (under some - other PID, anyway) we will just ignore it as well - XXX: long-term fix is to establish in-out PID-aware tracking? */ - if (peer) { -@@ -450,51 +472,51 @@ pcmk_cpg_membership(cpg_handle_t handle, - rival = bsearch(&key, sorted, member_list_entries, - sizeof(const struct cpg_address *), - cmp_member_list_nodeid); -- if (rival == NULL) { -+ } -+ -+ if (rival == NULL) { -+ crm_info("Group %s event %d: %s (node %u pid %u) left%s", -+ groupName->value, counter, peer_name(peer), -+ left_list[i].nodeid, left_list[i].pid, -+ cpgreason2str(left_list[i].reason)); -+ if (peer) { - crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, - OFFLINESTATUS); -- } else if (left_list[i].nodeid == local_nodeid) { -- crm_info("Ignoring the above event %s.%d, comes from a local" -- " rival process (presumably not us): %llu", -- groupName->value, counter, -- (unsigned long long) left_list[i].pid); -- } else { -- crm_info("Ignoring the above event %s.%d, comes from" -- " a rival-rich node: %llu (e.g. %llu process" -- " carries on)", -- groupName->value, counter, -- (unsigned long long) left_list[i].pid, -- (unsigned long long) (*rival)->pid); - } -+ } else if (left_list[i].nodeid == local_nodeid) { -+ crm_warn("Group %s event %d: duplicate local pid %u left%s", -+ groupName->value, counter, -+ left_list[i].pid, cpgreason2str(left_list[i].reason)); -+ } else { -+ crm_warn("Group %s event %d: " -+ "%s (node %u) duplicate pid %u left%s (%u remains)", -+ groupName->value, counter, peer_name(peer), -+ left_list[i].nodeid, left_list[i].pid, -+ cpgreason2str(left_list[i].reason), (*rival)->pid); - } - } - free(sorted); - sorted = NULL; - - for (i = 0; i < joined_list_entries; i++) { -- crm_info("Node %u joined group %s (counter=%d.%d, pid=%llu," -- " unchecked for rivals)", -- joined_list[i].nodeid, groupName->value, counter, i, -- (unsigned long long) left_list[i].pid); -+ crm_info("Group %s event %d: node %u pid %u joined%s", -+ groupName->value, counter, joined_list[i].nodeid, -+ joined_list[i].pid, cpgreason2str(joined_list[i].reason)); - } - - for (i = 0; i < member_list_entries; i++) { - crm_node_t *peer = crm_get_peer(member_list[i].nodeid, NULL); - -- crm_info("Node %u still member of group %s (peer=%s:%llu," -- " counter=%d.%d, at least once)", -- member_list[i].nodeid, groupName->value, -- (peer? peer->uname : ""), member_list[i].pid, -- counter, i); -- - if (member_list[i].nodeid == local_nodeid - && member_list[i].pid != getpid()) { - /* see the note above */ -- crm_info("Ignoring the above event %s.%d, comes from a local rival" -- " process: %llu", groupName->value, counter, -- (unsigned long long) member_list[i].pid); -+ crm_warn("Group %s event %d: detected duplicate local pid %u", -+ groupName->value, counter, member_list[i].pid); - continue; - } -+ crm_info("Group %s event %d: %s (node %u pid %u) is member", -+ groupName->value, counter, peer_name(peer), -+ member_list[i].nodeid, member_list[i].pid); - - /* Anyone that is sending us CPG messages must also be a _CPG_ member. - * But it's _not_ safe to assume it's in the quorum membership. -@@ -514,9 +536,8 @@ pcmk_cpg_membership(cpg_handle_t handle, - * - * Set the threshold to 1 minute - */ -- crm_err("Node %s[%u] appears to be online even though we think" -- " it is dead (unchecked for rivals)", -- peer->uname, peer->id); -+ crm_warn("Node %u is member of group %s but was believed offline", -+ member_list[i].nodeid, groupName->value); - if (crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_MEMBER, 0)) { - peer->votes = 0; - } -@@ -529,7 +550,7 @@ pcmk_cpg_membership(cpg_handle_t handle, - } - - if (!found) { -- crm_err("We're not part of CPG group '%s' anymore!", groupName->value); -+ crm_err("Local node was evicted from group %s", groupName->value); - cpg_evicted = TRUE; - } - --- -1.8.3.1 - - -From 87769895ebccc1033a876ef98a21577d6f4d1c0e Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 18 Apr 2019 22:18:27 -0500 -Subject: [PATCH 2/2] Fix: libcrmcluster,pacemakerd: restore compatibility with - corosync 1 - -Pacemaker 1.1 supports older versions of corosync that don't supply -cs_strerror() or CMAP. This simply drops usage cs_strerror() (in favor of just -the raw error code, as before 07a82c5c) and properly conditionalizes CMAP -usage. ---- - lib/cluster/cpg.c | 12 ++++-------- - mcp/corosync.c | 13 +++++++------ - 2 files changed, 11 insertions(+), 14 deletions(-) - -diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c -index 85476be..e4783e5 100644 ---- a/lib/cluster/cpg.c -+++ b/lib/cluster/cpg.c -@@ -91,15 +91,13 @@ uint32_t get_local_nodeid(cpg_handle_t handle) - crm_trace("Creating connection"); - cs_repeat(retries, 5, rc = cpg_initialize(&local_handle, &cb)); - if (rc != CS_OK) { -- crm_err("Could not connect to the CPG API: %s (%d)", -- cs_strerror(rc), rc); -+ crm_err("Could not connect to the CPG API (rc=%d)", rc); - return 0; - } - - rc = cpg_fd_get(local_handle, &fd); - if (rc != CS_OK) { -- crm_err("Could not obtain the CPG API connection: %s (%d)", -- cs_strerror(rc), rc); -+ crm_err("Could not obtain the CPG API connection (rc=%d)", rc); - goto bail; - } - -@@ -594,15 +592,13 @@ cluster_connect_cpg(crm_cluster_t *cluster) - - cs_repeat(retries, 30, rc = cpg_initialize(&handle, &cpg_callbacks)); - if (rc != CS_OK) { -- crm_err("Could not connect to the CPG API: %s (%d)", -- cs_strerror(rc), rc); -+ crm_err("Could not connect to the CPG API (rc=%d)", rc); - goto bail; - } - - rc = cpg_fd_get(handle, &fd); - if (rc != CS_OK) { -- crm_err("Could not obtain the CPG API connection: %s (%d)", -- cs_strerror(rc), rc); -+ crm_err("Could not obtain the CPG API connection (rc=%d)", rc); - goto bail; - } - -diff --git a/mcp/corosync.c b/mcp/corosync.c -index 407a63f..40be727 100644 ---- a/mcp/corosync.c -+++ b/mcp/corosync.c -@@ -118,13 +118,13 @@ cluster_connect_cfg(uint32_t * nodeid) - cs_repeat(retries, 30, rc = corosync_cfg_initialize(&cfg_handle, &cfg_callbacks)); - - if (rc != CS_OK) { -- crm_err("corosync cfg init: %s (%d)", cs_strerror(rc), rc); -+ crm_err("corosync cfg init error %d", rc); - return FALSE; - } - - rc = corosync_cfg_fd_get(cfg_handle, &fd); - if (rc != CS_OK) { -- crm_err("corosync cfg fd_get: %s (%d)", cs_strerror(rc), rc); -+ crm_err("corosync cfg fd_get error %d", rc); - goto bail; - } - -@@ -314,8 +314,8 @@ mcp_read_config(void) - rc = cmap_initialize(&local_handle); - if (rc != CS_OK) { - retries++; -- printf("cmap connection setup failed: %s. Retrying in %ds\n", cs_strerror(rc), retries); -- crm_info("cmap connection setup failed: %s. Retrying in %ds", cs_strerror(rc), retries); -+ printf("cmap connection setup failed: error %d. Retrying in %ds\n", rc, retries); -+ crm_info("cmap connection setup failed: error %d. Retrying in %ds", rc, retries); - sleep(retries); - - } else { -@@ -331,10 +331,10 @@ mcp_read_config(void) - return FALSE; - } - -+#if HAVE_CMAP - rc = cmap_fd_get(local_handle, &fd); - if (rc != CS_OK) { -- crm_err("Could not obtain the CMAP API connection: %s (%d)", -- cs_strerror(rc), rc); -+ crm_err("Could not obtain the CMAP API connection: error %d", rc); - cmap_finalize(local_handle); - return FALSE; - } -@@ -354,6 +354,7 @@ mcp_read_config(void) - cmap_finalize(local_handle); - return FALSE; - } -+#endif - - stack = get_cluster_type(); - crm_info("Reading configure for stack: %s", name_for_cluster_type(stack)); --- -1.8.3.1 - diff --git a/SOURCES/009-use-after-free.patch b/SOURCES/009-use-after-free.patch deleted file mode 100644 index d419a0f..0000000 --- a/SOURCES/009-use-after-free.patch +++ /dev/null @@ -1,158 +0,0 @@ -From dd521e724c4c2d4f074ffcf30a9e7a844fcfb7d2 Mon Sep 17 00:00:00 2001 -From: Klaus Wenninger -Date: Sat, 18 May 2019 06:17:36 +0200 -Subject: [PATCH] Fix: fence-lib: avoid use-after-free on early failure return - -Bailing out in case of non-existant fence-agent is such a case. ---- - fencing/commands.c | 30 +++++++++++++++++++++--------- - include/crm/fencing/internal.h | 5 +++-- - lib/fencing/st_client.c | 28 +++++++++++++++++++++------- - 3 files changed, 45 insertions(+), 18 deletions(-) - -diff --git a/fencing/commands.c b/fencing/commands.c -index af5324a..d47a5ea 100644 ---- a/fencing/commands.c -+++ b/fencing/commands.c -@@ -107,6 +107,7 @@ typedef struct async_command_s { - int last_timeout_signo; - - stonith_device_t *active_on; -+ stonith_device_t *activating_on; - } async_command_t; - - static xmlNode *stonith_construct_async_reply(async_command_t * cmd, const char *output, -@@ -301,6 +302,19 @@ get_active_cmds(stonith_device_t * device) - return counter; - } - -+static void -+fork_cb(GPid pid, gpointer user_data) -+{ -+ async_command_t *cmd = (async_command_t *) user_data; -+ stonith_device_t * device = cmd->activating_on; -+ -+ crm_debug("Operation %s%s%s on %s now running with pid=%d, timeout=%ds", -+ cmd->action, cmd->victim ? " for node " : "", cmd->victim ? cmd->victim : "", -+ device->id, pid, cmd->timeout); -+ cmd->active_on = device; -+ cmd->activating_on = NULL; -+} -+ - static gboolean - stonith_device_execute(stonith_device_t * device) - { -@@ -387,19 +401,17 @@ stonith_device_execute(stonith_device_t * device) - cmd->victim_nodeid, - cmd->timeout, device->params, device->aliases); - -- /* for async exec, exec_rc is pid if positive and error code if negative/zero */ -- exec_rc = stonith_action_execute_async(action, (void *)cmd, cmd->done_cb); -- -- if (exec_rc > 0) { -- crm_debug("Operation %s%s%s on %s now running with pid=%d, timeout=%ds", -- cmd->action, cmd->victim ? " for node " : "", cmd->victim ? cmd->victim : "", -- device->id, exec_rc, cmd->timeout); -- cmd->active_on = device; -+ /* for async exec, exec_rc is negative for early error exit -+ otherwise handling of success/errors is done via callbacks */ -+ cmd->activating_on = device; -+ exec_rc = stonith_action_execute_async(action, (void *)cmd, -+ cmd->done_cb, fork_cb); - -- } else { -+ if (exec_rc < 0) { - crm_warn("Operation %s%s%s on %s failed: %s (%d)", - cmd->action, cmd->victim ? " for node " : "", cmd->victim ? cmd->victim : "", - device->id, pcmk_strerror(exec_rc), exec_rc); -+ cmd->activating_on = NULL; - cmd->done_cb(0, exec_rc, NULL, cmd); - } - return TRUE; -diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h -index 90673bb..24df230 100644 ---- a/include/crm/fencing/internal.h -+++ b/include/crm/fencing/internal.h -@@ -24,11 +24,12 @@ void stonith__destroy_action(stonith_action_t *action); - void stonith__action_result(stonith_action_t *action, int *rc, char **output, - char **error_output); - --GPid -+int - stonith_action_execute_async(stonith_action_t * action, - void *userdata, - void (*done) (GPid pid, int rc, const char *output, -- gpointer user_data)); -+ gpointer user_data), -+ void (*fork_cb) (GPid pid, gpointer user_data)); - - int stonith__execute(stonith_action_t *action); - -diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c -index 0c1eadc..c38f356 100644 ---- a/lib/fencing/st_client.c -+++ b/lib/fencing/st_client.c -@@ -54,6 +54,7 @@ struct stonith_action_s { - int async; - void *userdata; - void (*done_cb) (GPid pid, gint status, const char *output, gpointer user_data); -+ void (*fork_cb) (GPid pid, gpointer user_data); - - svc_action_t *svc_action; - -@@ -835,6 +836,10 @@ stonith_action_async_forked(svc_action_t *svc_action) - action->pid = svc_action->pid; - action->svc_action = svc_action; - -+ if (action->fork_cb) { -+ (action->fork_cb) (svc_action->pid, action->userdata); -+ } -+ - crm_trace("Child process %d performing action '%s' successfully forked", - action->pid, action->action); - } -@@ -916,25 +921,34 @@ internal_stonith_action_execute(stonith_action_t * action) - return rc; - } - --GPid -+/*! -+ * \internal -+ * \brief Kick off execution of an async stonith action -+ * -+ * \param[in,out] action Action to be executed -+ * \param[in,out] userdata Datapointer to be passed to callbacks -+ * \param[in] done Callback to notify action has failed/succeeded -+ * \param[in] fork_callback Callback to notify successful fork of child -+ * -+ * \return pcmk_ok if ownership of action has been taken, -errno otherwise -+ */ -+int - stonith_action_execute_async(stonith_action_t * action, - void *userdata, - void (*done) (GPid pid, int rc, const char *output, -- gpointer user_data)) -+ gpointer user_data), -+ void (*fork_cb) (GPid pid, gpointer user_data)) - { -- int rc = 0; -- - if (!action) { - return -1; - } - - action->userdata = userdata; - action->done_cb = done; -+ action->fork_cb = fork_cb; - action->async = 1; - -- rc = internal_stonith_action_execute(action); -- -- return rc < 0 ? rc : action->pid; -+ return internal_stonith_action_execute(action); - } - - /*! --- -1.8.3.1 - diff --git a/SOURCES/01-rollup.patch b/SOURCES/01-rollup.patch new file mode 100644 index 0000000..b06cebe --- /dev/null +++ b/SOURCES/01-rollup.patch @@ -0,0 +1,11661 @@ +From c46477fe38bdede01a070183052e5fa76f3631ef Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= +Date: Thu, 6 Jun 2019 14:47:40 +0200 +Subject: [PATCH 01/96] Maint: introduce mock/experimental skeletons, now for + cib (based) daemon + +[this is a rebase from 2.0, occurrences of "based" mean "cib", but they +are not changed everywhere, especially not in the name of the mocked +daemon, for easy comparability amongst the branches] + +This is meant for multiple purposes (non-exhaustive): + +- simulating scenarios that are rather hard to achieve in practice + (overwhelming number of nodes or artificially capped resources) + +- robustness testing (will other daemons survive intermittently, yet + frequently missing responses to the based queries? to what extent + do they rely on some causality relations like notify-after-write? + what if a chance of continuously changed CIB contents causing + troubles?) + +- anything not fitting the former two categories + +Another option would be to start from a full-fledged daemon, cutting it +to pieces self-contained, patchable pieces, but since in case of based, +it has grown rather organically as a monolith, it would be substantially +more hassle than start grab the most vital pieces and start with a clean +room. The aim is to keep additional functionality on top of said +skeleton, and let the additional functionality stay apart in opt-in +loosly coupled modules. We shall see how this works out. +--- + maint/mocked/Makefile | 42 +++++++ + maint/mocked/based.c | 328 ++++++++++++++++++++++++++++++++++++++++++++++++++ + maint/mocked/based.h | 47 ++++++++ + 4 files changed, 418 insertions(+) + create mode 100644 maint/mocked/Makefile + create mode 100644 maint/mocked/based.c + create mode 100644 maint/mocked/based.h + +diff --git a/maint/mocked/Makefile b/maint/mocked/Makefile +new file mode 100644 +index 0000000..05b3cb4 +--- /dev/null ++++ b/maint/mocked/Makefile +@@ -0,0 +1,42 @@ ++# ++# Copyright 2019 the Pacemaker project contributors ++# ++# The version control history for this file may have further details. ++# ++# Copying and distribution of this file, with or without modification, ++# are permitted in any medium without royalty provided the copyright ++# notice and this notice are preserved. This file is offered as-is, ++# without any warranty. ++# ++ ++#BASED_LDFLAGS = $$(pkgconf -libs glib-2.0) \ ++# $$(pkgconf -libs libxml-2.0) \ ++# $$(pkgconf -libs libqb) \ ++# $$(pkgconf -libs pacemaker) ++BASED_LDFLAGS = $$(pkgconf -libs glib-2.0) \ ++ $$(pkgconf -libs libxml-2.0) \ ++ $$(pkgconf -libs libqb) \ ++ -Wl,-rpath=$(CURDIR)/../../lib/common/.libs \ ++ -L../../lib/common/.libs -lcrmcommon \ ++ -L../../lib/pacemaker/.libs -lpacemaker ++ ++BASED_CPPFLAGS = $$(pkgconf -cflags glib-2.0) \ ++ $$(pkgconf -cflags libxml-2.0) \ ++ $$(pkgconf -cflags libqb) \ ++ -DCS_USES_LIBQB \ ++ -I ../.. -I ../../include -g ++ ++PROGRAMS = based ++ ++BASED_OBJECTS = based.o ++ ++all: ${PROGRAMS} ++ ++based: $(BASED_OBJECTS) ++ $(CC) $(BASED_LDFLAGS) $^ -o $@ ++ ++$(BASED_OBJECTS): %.o: %.c ++ $(CC) $(BASED_CPPFLAGS) $(BASED_LDFLAGS) -c $< -o $@ ++ ++clean: ++ rm ${PROGRAMS} $(BASED_OBJECTS) +diff --git a/maint/mocked/based.c b/maint/mocked/based.c +new file mode 100644 +index 0000000..451a384 +--- /dev/null ++++ b/maint/mocked/based.c +@@ -0,0 +1,328 @@ ++/* ++ * Copyright 2019 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. ++ * ++ * Licensed under the GNU General Public License version 2 or later (GPLv2+). ++ */ ++ ++/* ++ * Clean room attempt (admittedly with lot of code borrowed or inspired from ++ * the full-blown daemon), minimalistic implementation of based daemon, with ++ * only important aspects being implemented at the moment. ++ * ++ * Hopefully easy to adapt for variety of purposes. ++ * ++ * NOTE: currently, only cib_rw API end-point is opened, future refinements ++ * as new modules are added should conditionalize per what the module ++ * indicates in the context (which is intentionally very loose data glue ++ * between the skeleton and modules themselves (like CGI variables so ++ * to say, but more structurally predestined so as to avoid complexities ++ * of hash table lookups etc.) ++ */ ++ ++#include ++#if 0 ++#include "crm/common/ipcs.h" /* crm_client_t */ ++#include "crm/common/xml.h" /* crm_xml_add */ ++#endif ++#include "crm/cib/internal.h" /* T_CIB */ ++#include "crm/msg_xml.h" /* F_SUBTYPE */ ++#include "cib/callbacks.h" /* cib_notify_diff */ ++ ++#include /* qb_ipcs_connection_t */ ++ ++#include "based.h" ++ ++ ++/* direct global access violated in one case only ++ - mock_based_ipc_accept adds a reference to it to crm_cient_t->userdata */ ++mock_based_context_t mock_based_context; ++ ++ ++/* see based/based_callbacks.c:cib_ipc_accept */ ++static int32_t ++mock_based_ipc_accept(qb_ipcs_connection_t *c, uid_t uid, gid_t gid) ++{ ++ int32_t ret = 0; ++ crm_client_t *cib_client; ++ ++ crm_trace("Connection %p", c); ++ if ((cib_client = crm_client_new(c, uid, gid)) == NULL) { ++ ret = -EIO; ++ } ++ ++ cib_client->userdata = &mock_based_context; ++ ++ return ret; ++} ++ ++/* see based/based_callbacks.c:cib_ipc_created */ ++static void ++mock_based_ipc_created(qb_ipcs_connection_t *c) ++{ ++ crm_trace("Connection %p", c); ++} ++ ++/* see based/based_callbacks.c:cib_ipc_closed */ ++static int32_t ++mock_based_ipc_closed(qb_ipcs_connection_t *c) ++{ ++ crm_client_t *client = crm_client_get(c); ++ ++ if (client != NULL) { ++ crm_trace("Connection %p", c); ++ crm_client_destroy(client); ++ } ++ ++ return 0; ++} ++ ++/* see based/based_callbacks.c:cib_ipc_destroy */ ++static void ++mock_based_ipc_destroy(qb_ipcs_connection_t *c) ++{ ++ crm_trace("Connection %p", c); ++ mock_based_ipc_closed(c); ++} ++ ++/* see based/based_callbacks.c:cib_process_command (and more) */ ++static void ++mock_based_handle_query(crm_client_t *cib_client, uint32_t flags, ++ const xmlNode *op_request) ++{ ++ xmlNode *reply, *cib; ++ const char cib_str[] = ++#if 0 ++""; ++#else ++""\ ++" "\ ++" "\ ++" "\ ++" "\ ++" "\ ++" "\ ++" "\ ++""; ++#endif ++ cib = xmlReadMemory(cib_str, sizeof(cib_str), "file:///tmp/foo", NULL, 0)->children; ++ ++ reply = create_xml_node(NULL, "cib-reply"); ++ crm_xml_add(reply, F_TYPE, T_CIB); ++ crm_xml_add(reply, F_CIB_OPERATION, ++ crm_element_value(op_request, F_CIB_OPERATION)); ++ crm_xml_add(reply, F_CIB_CALLID, ++ crm_element_value(op_request, F_CIB_CALLID)); ++ crm_xml_add(reply, F_CIB_CLIENTID, ++ crm_element_value(op_request, F_CIB_CLIENTID)); ++ crm_xml_add_int(reply, F_CIB_CALLOPTS, flags); ++ crm_xml_add_int(reply, F_CIB_RC, pcmk_ok); ++ ++ if (cib != NULL) { ++ crm_trace("Attaching reply output"); ++ add_message_xml(reply, F_CIB_CALLDATA, cib); ++ } ++ ++ crm_ipcs_send(cib_client, cib_client->request_id, reply, ++ (flags & cib_sync_call) ? crm_ipc_flags_none ++ : crm_ipc_server_event); ++ ++ free_xml(reply); ++ free_xml(cib); ++} ++ ++/* see based/based_callbacks.c:cib_common_callback_worker */ ++static void ++mock_based_common_callback_worker(uint32_t id, uint32_t flags, ++ xmlNode *op_request, crm_client_t *cib_client) ++{ ++ const char *op = crm_element_value(op_request, F_CIB_OPERATION); ++ ++ if (!strcmp(op, CRM_OP_REGISTER)) { ++ if (flags & crm_ipc_client_response) { ++ xmlNode *ack = create_xml_node(NULL, __FUNCTION__); ++ crm_xml_add(ack, F_CIB_OPERATION, CRM_OP_REGISTER); ++ crm_xml_add(ack, F_CIB_CLIENTID, cib_client->id); ++ crm_ipcs_send(cib_client, id, ack, flags); ++ cib_client->request_id = 0; ++ free_xml(ack); ++ } ++ ++ } else if (!strcmp(op, T_CIB_NOTIFY)) { ++ int on_off = 0; ++ const char *type = crm_element_value(op_request, F_CIB_NOTIFY_TYPE); ++ crm_element_value_int(op_request, F_CIB_NOTIFY_ACTIVATE, &on_off); ++ ++ crm_debug("Setting %s callbacks for %s (%s): %s", ++ type, cib_client->name, cib_client->id, on_off ? "on" : "off"); ++ ++ if (!strcmp(type, T_CIB_DIFF_NOTIFY) && on_off) { ++ cib_client->options |= cib_notify_diff; ++ } ++ ++ if (flags & crm_ipc_client_response) { ++ crm_ipcs_send_ack(cib_client, id, flags, "ack", __FUNCTION__, __LINE__); ++ } ++ ++ } else if (!strcmp(op, CIB_OP_QUERY)) { ++ mock_based_handle_query(cib_client, flags, op_request); ++ ++ } else { ++ crm_notice("Discarded request %s", op); ++ } ++} ++ ++/* see based/based_callbacks.c:cib_ipc_dispatch_rw */ ++static int32_t ++mock_based_dispatch_command(qb_ipcs_connection_t *c, void *data, size_t size) ++{ ++ uint32_t id = 0, flags = 0; ++ int call_options = 0; ++ crm_client_t *cib_client = crm_client_get(c); ++ xmlNode *op_request = crm_ipcs_recv(cib_client, data, size, &id, &flags); ++ ++ crm_notice("Got connection %p", c); ++ assert(op_request != NULL); ++ ++ if (cib_client == NULL || op_request == NULL) { ++ if (op_request == NULL) { ++ crm_trace("Invalid message from %p", c); ++ crm_ipcs_send_ack(cib_client, id, flags, "nack", __FUNCTION__, __LINE__); ++ } ++ return 0; ++ } ++ ++ crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options); ++ if (call_options & cib_sync_call) { ++ assert(flags & crm_ipc_client_response); ++ cib_client->request_id = id; /* reply only to last in-flight request */ ++ } ++ ++ assert(cib_client->name == NULL); ++ crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options); ++ crm_xml_add(op_request, F_CIB_CLIENTID, cib_client->id); ++ crm_xml_add(op_request, F_CIB_CLIENTNAME, cib_client->name); ++ ++ mock_based_common_callback_worker(id, flags, op_request, cib_client); ++ free_xml(op_request); ++ ++ return 0; ++} ++ ++/* * */ ++ ++size_t mock_based_register_module(module_t mod) { ++ module_t *module; ++ size_t ret = mock_based_context.modules_cnt++; ++ ++ mock_based_context.modules = realloc(mock_based_context.modules, ++ sizeof(*mock_based_context.modules) ++ * mock_based_context.modules_cnt); ++ if (mock_based_context.modules == NULL ++ || (module = malloc(sizeof(module_t))) == NULL) { ++ abort(); ++ } ++ ++ memcpy(module, &mod, sizeof(mod)); ++ mock_based_context.modules[mock_based_context.modules_cnt - 1] = module; ++ ++ return ret; ++} ++ ++static int ++mock_based_options(mock_based_context_t *ctxt, ++ bool usage, int argc, const char *argv[]) ++{ ++ const char **args2argv; ++ char *s; ++ int ret = 0; ++ ++ if (argc <= 1) { ++ const char *help_argv[] = {argv[0], "-h"}; ++ return mock_based_options(ctxt, false, 2, (const char **) &help_argv); ++ } ++ ++ for (size_t i = 1; i < argc; i++) { ++ if (argv[i][0] == '-' && argv[i][1] != '-' && argv[i][1] != '\0') { ++ if (usage) { ++ printf("\t-%c\t", argv[i][1]); ++ } ++ switch(argv[i][1]) { ++ case 'h': ++ if (usage) { ++ printf("show this help message\n"); ++ ret = 1; ++ ++ } else { ++ if ((args2argv ++ = malloc((ctxt->modules_cnt + 2) * sizeof(*args2argv))) == NULL ++ || (s ++ = malloc((ctxt->modules_cnt * 2 + 2) * sizeof(*s))) == NULL) { ++ return -1; ++ } ++ s[0] = 'h'; ++ args2argv[ctxt->modules_cnt + 1] = (char[]){'-', 'h', '\0'}; ++ for (size_t c = ctxt->modules_cnt; c > 0; c--) { ++ args2argv[c] = (char[]){'-', ctxt->modules[c - 1]->shortopt, '\0'}; ++ s[(ctxt->modules_cnt - i) + 1] = '|'; ++ s[(ctxt->modules_cnt - i) + 2] = ctxt->modules[c - 1]->shortopt; ++ } ++ s[ctxt->modules_cnt * 2 + 1] = '\0'; ++ printf("Usage: %s [-{%s}]\n", argv[0], s); ++ (void) mock_based_options(ctxt, true, 2 + ctxt->modules_cnt, args2argv); ++ free(args2argv); ++ free(s); ++ } ++ return ret; ++ default: ++ for (size_t c = ctxt->modules_cnt; c > 0; c--) { ++ if (ctxt->modules[c - 1]->shortopt == argv[i][1]) { ++ ret = ctxt->modules[c - 1]->hooks.argparse(ctxt, usage, argc - i, &argv[i]); ++ if (ret < 0) { ++ break; ++ } else if (ret > 1) { ++ i += (ret - 1); ++ } ++ } ++ } ++ if (ret == 0) { ++ printf("uknown option \"%s\"\n", argv[i]); ++ } ++ break; ++ } ++ } ++ } ++ return ret; ++} ++ ++int main(int argc, char *argv[]) ++{ ++ mock_based_context_t *ctxt = &mock_based_context; ++ ++ if (mock_based_options(ctxt, false, argc, (const char **) argv) > 0) { ++ struct qb_ipcs_service_handlers cib_ipc_callbacks = { ++ .connection_accept = mock_based_ipc_accept, ++ .connection_created = mock_based_ipc_created, ++ .msg_process = mock_based_dispatch_command, ++ .connection_closed = mock_based_ipc_closed, ++ .connection_destroyed = mock_based_ipc_destroy, ++ }; ++ crm_log_preinit(NULL, argc, argv); ++ crm_log_init(NULL, LOG_DEBUG, false, true, argc, argv, false); ++ qb_ipcs_service_t *ipcs_command = ++ mainloop_add_ipc_server(cib_channel_rw, QB_IPC_NATIVE, ++ &cib_ipc_callbacks); ++ g_main_loop_run(g_main_loop_new(NULL, false)); ++ qb_ipcs_destroy(ipcs_command); ++ } ++ ++ for (size_t c = ctxt->modules_cnt; c > 0; c--) { ++ if (ctxt->modules[c - 1]->hooks.destroy != NULL) { ++ ctxt->modules[c - 1]->hooks.destroy(ctxt->modules[c - 1]); ++ } ++ free(mock_based_context.modules[c - 1]); ++ } ++ ++ free(mock_based_context.modules); ++} +diff --git a/maint/mocked/based.h b/maint/mocked/based.h +new file mode 100644 +index 0000000..04d8eed +--- /dev/null ++++ b/maint/mocked/based.h +@@ -0,0 +1,47 @@ ++/* ++ * Copyright 2019 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. ++ * ++ * Licensed under the GNU General Public License version 2 or later (GPLv2+). ++ */ ++ ++#pragma once ++ ++#include /* size_t */ ++#include /* bool */ ++ ++#include /* crm_client_t */ ++ ++ ++struct module_s; ++ ++typedef struct mock_based_context_s { ++ size_t modules_cnt; ++ struct module_s** modules; ++} mock_based_context_t; ++ ++ ++typedef int (*mock_based_argparse_hook)(mock_based_context_t *, ++ bool, int, ++ const char *[]); ++ ++typedef void (*mock_based_destroy_hook)(struct module_s *); ++ ++/* specialized callbacks... */ ++ ++typedef struct mock_based_hooks_s { ++ /* generic ones */ ++ mock_based_argparse_hook argparse; ++ mock_based_destroy_hook destroy; ++ ++ /* specialized callbacks... */ ++} mock_based_hooks_t; ++ ++typedef struct module_s { ++ char shortopt; ++ mock_based_hooks_t hooks; ++ void *priv; ++} module_t; ++ ++size_t mock_based_register_module(module_t mod); +-- +1.8.3.1 + + +From 1498a811283a00623c4410c7e9abbd8b9ff3fe53 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= +Date: Wed, 15 May 2019 18:23:56 +0200 +Subject: [PATCH 02/96] Maint: mocked/based: add based-notifyfenced module + +[this is a rebase from 2.0, occurrences of "based" mean "cib", just as +"fenced" means "stonithd" in this context , but these are not changed +everywhere, especially not in the name of the mocked daemon + modules, +for easy comparability amongst the branches] + +This is to demonstrate how the current arrangement at _other_ +daemons (fenced in particular, as the name may suggest) is sometimes +insufficient (see the following fix for fenced daemon) regarding +liveness requirements, since high-rate stream of notifications from +pacemaker-based can effectively block any progress regarding their +own native services they provide (starving their own clients out). + +It would be rather difficult to achieve the same triggering +circumstances artificially in vanilla settings, especially when +constrained at number-of-nodes/resources dimension (bothersome +artificial-messaging-load-through-configuration) -- leveraging the +skeleton from the previous commit, we can now emulate the same just +with a single node under test and with next to zero configuration +-- just configure a single node cluster through corosync.conf, start +corosync, run "./based -N" (preferably as hacluster:haclient), only +then pacemaker-fenced and try to communicate with it (e.g. via +stonith_admin) -- see the in-line comment wrt. how to use this module. + +Note that this first module has some parts ifdef'd out since it's +intented also as a template for writing additional modules -- you'll: + +- copy based-notifyspam.c as based-mymodule.c, and edit it, so that... + +- OPTCHAR is a new, unique short option character + (preferably uppercase, leaving lower-cased letters reserved as + action modifiers, cf. xml/regression.sh) + +- drop everything unneeded except for mock_based_MOD_argparse_hook + and mock_based_MOD_init, configure the callbacks there respectively + +- should the new hook mounting place be needed, declare new hook + prototype in based.h, append such respective member + to the struct mock_based_hooks_s there, locate the corresponding + location for its application in based.c and apply it here + (follow the example of hooks.cib_notify) + +- add the respective "BASED_OBJECTS += based-MOD.o" to Makefile + +- test... +--- + maint/mocked/Makefile | 3 + + maint/mocked/based-notifyfenced.c | 247 ++++++++++++++++++++++++++++++++++++++ + maint/mocked/based.c | 8 ++ + maint/mocked/based.h | 2 + + 4 files changed, 260 insertions(+) + create mode 100644 maint/mocked/based-notifyfenced.c + +diff --git a/maint/mocked/Makefile b/maint/mocked/Makefile +index 05b3cb4..5e3a1a8 100644 +--- a/maint/mocked/Makefile ++++ b/maint/mocked/Makefile +@@ -30,6 +30,9 @@ PROGRAMS = based + + BASED_OBJECTS = based.o + ++# include or not the modules as you wish ++BASED_OBJECTS += based-notifyfenced.o ++ + all: ${PROGRAMS} + + based: $(BASED_OBJECTS) +diff --git a/maint/mocked/based-notifyfenced.c b/maint/mocked/based-notifyfenced.c +new file mode 100644 +index 0000000..90cad48 +--- /dev/null ++++ b/maint/mocked/based-notifyfenced.c +@@ -0,0 +1,247 @@ ++/* ++ * Copyright 2019 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. ++ * ++ * Licensed under the GNU General Public License version 2 or later (GPLv2+). ++ */ ++ ++/* ++ * Intended demo use case: ++ * ++ * - as root, start corosync ++ * - start "./based -N"; hint: ++ * su -s /bin/sh -c './based -N' hacluster ++ * - start stonithd; hint: ++ * su -c 'env PCMK_logpriority=crit ../../fencing/stonithd' ++ * - wait a bit (5 < seconds < 20) ++ * - as haclient group (or root), run "stonith_admin --list-registered" ++ * - observe whether such invocation is blocked or not ++ */ ++ ++ ++#include /* printf, perror */ ++ ++#include "crm/cib.h" /* cib_zero_copy */ ++#include "crm/cib/internal.h" /* CIB_OP_CREATE */ ++#include "crm/msg_xml.h" /* F_SUBTYPE */ ++#include "cib/callbacks.h" /* cib_notify_diff */ ++ ++#include "based.h" ++ ++ ++#define OPTCHAR 'N' ++static size_t module_handle; ++ ++ ++struct cib_notification_s { ++ xmlNode *msg; ++ struct iovec *iov; ++ int32_t iov_size; ++}; ++ ++/* see based/based_notify.c:cib_notify_send_one */ ++static bool ++mock_based_cib_notify_send_one(crm_client_t *client, xmlNode *xml) ++{ ++ const char *type = NULL; ++ bool do_send = false; ++ ++ struct iovec *iov; ++ ssize_t rc = crm_ipc_prepare(0, xml, &iov, 0); ++ struct cib_notification_s update = { ++ .msg = xml, ++ .iov = iov, ++ .iov_size = rc, ++ }; ++ ++ CRM_CHECK(client != NULL, return true); ++ if (client->ipcs == NULL && client->remote == NULL) { ++ crm_warn("Skipping client with NULL channel"); ++ return FALSE; ++ } ++ ++ type = crm_element_value(update.msg, F_SUBTYPE); ++ CRM_LOG_ASSERT(type != NULL); ++ if (is_set(client->options, cib_notify_diff) ++ && safe_str_eq(type, T_CIB_DIFF_NOTIFY)) { ++ ++ if (crm_ipcs_sendv(client, update.iov, crm_ipc_server_event) < 0) ++ crm_warn("Notification of client %s/%s failed", client->name, client->id); ++ ++ } ++ if (iov) { ++ free(iov[0].iov_base); ++ free(iov[1].iov_base); ++ free(iov); ++ } ++ ++ return FALSE; ++} ++ ++/* see based/based_notify.c:do_cib_notify + cib_notify_send */ ++void ++do_cib_notify(crm_client_t *cib_client, int options, const char *op, ++ xmlNode *update, int result, xmlNode *result_data, ++ const char *msg_type) ++{ ++ xmlNode *update_msg = NULL; ++ const char *id = NULL; ++ ++ update_msg = create_xml_node(NULL, "notify"); ++ ++ ++ crm_xml_add(update_msg, F_TYPE, T_CIB_NOTIFY); ++ crm_xml_add(update_msg, F_SUBTYPE, msg_type); ++ crm_xml_add(update_msg, F_CIB_OPERATION, op); ++ crm_xml_add_int(update_msg, F_CIB_RC, result); ++ ++ if (result_data != NULL) { ++ id = crm_element_value(result_data, XML_ATTR_ID); ++ if (id != NULL) ++ crm_xml_add(update_msg, F_CIB_OBJID, id); ++ } ++ ++ if (update != NULL) { ++ crm_trace("Setting type to update->name: %s", crm_element_name(update)); ++ crm_xml_add(update_msg, F_CIB_OBJTYPE, crm_element_name(update)); ++ ++ } else if (result_data != NULL) { ++ crm_trace("Setting type to new_obj->name: %s", crm_element_name(result_data)); ++ crm_xml_add(update_msg, F_CIB_OBJTYPE, crm_element_name(result_data)); ++ ++ } else { ++ crm_trace("Not Setting type"); ++ } ++ ++#if 0 ++ attach_cib_generation(update_msg, "cib_generation", the_cib); ++#endif ++ ++ if (update != NULL) { ++ add_message_xml(update_msg, F_CIB_UPDATE, update); ++ } ++ if (result_data != NULL) { ++ add_message_xml(update_msg, F_CIB_UPDATE_RESULT, result_data); ++ } ++ ++ mock_based_cib_notify_send_one(cib_client, update_msg); ++ free_xml(update_msg); ++} ++ ++static gboolean ++mock_based_notifyfencedmer_callback_worker(gpointer data) ++{ ++ crm_client_t *cib_client = (crm_client_t *) data; ++ ++ xmlNode *result_data; ++ xmlNode *input, *update; ++ int options; ++ char update_str[4096]; ++ ++ options |= cib_zero_copy; ++ ++ ++ input = create_xml_node(NULL, "cib"); ++ ++ /* spam it */ ++#if 0 ++ for (size_t i = 0; i < SIZE_MAX - 1; i++) { ++#else ++ for (size_t i = 0; i < 10000; i++) { ++#endif ++ /* NOTE: we need to trigger fenced attention, add new fence device */ ++ snprintf(update_str, sizeof(update_str), ++"\n" ++" \n" ++" \n" ++" \n" ++" \n" ++" \n" ++" \n" ++" \n" ++" \n" ++" \n" ++" \n" ++" \n" ++" \n" ++"\n", i, i+1); ++ update = xmlReadMemory(update_str, sizeof(update_str), ++ "file:///tmp/update", NULL, 0)->children; ++ do_cib_notify(cib_client, options, CIB_OP_CREATE, input, pcmk_ok, ++ update, T_CIB_DIFF_NOTIFY); ++ free_xml(update); ++ }; ++ ++ free_xml(input); ++} ++ ++static void ++mock_based_notifyfenced_cib_notify_hook(crm_client_t *cib_client) ++{ ++ ++ /* MOCK: client asked for upcoming diff's, let's ++ spam it a bit after a while... */ ++ crm_info("Going to spam %s (%s) in 5 seconds...", ++ cib_client->name, cib_client->id); ++ mainloop_timer_start(mainloop_timer_add("spammer", 5000, FALSE, ++ mock_based_notifyfencedmer_callback_worker, ++ cib_client)); ++} ++ ++/* * */ ++ ++static int ++mock_based_notifyfenced_argparse_hook(struct mock_based_context_s *ctxt, ++ bool usage, int argc_to_go, ++ const char *argv_to_go[]) ++{ ++ const char *opt = *argv_to_go; ++restart: ++ switch(*opt) { ++ case '-': ++ if (opt == *argv_to_go) { ++ opt++; ++ goto restart; ++ } ++ break; ++ case OPTCHAR: ++ if (usage) { ++ printf("spam the \"cib diff\" notification client" ++ " (targeting pacemaker-fenced in particular)\n"); ++ ++ } else { ++#if 0 ++ ctxt->modules[module_handle]->priv = ++ malloc(sizeof(mock_based_notifyfenced_priv_t)); ++ if (ctxt->modules[module_handle]->priv == NULL) { ++ perror("malloc"); ++ return -1; ++ } ++#endif ++ } ++ return 1; ++ } ++ return 0; ++} ++ ++#if 0 ++static void ++mock_based_notifyfenced_destroy_hook(module_t *mod) { ++ free(mod->priv); ++} ++#endif ++ ++__attribute__((__constructor__)) ++void ++mock_based_notifyfenced_init(void) { ++ module_handle = mock_based_register_module((module_t){ ++ .shortopt = OPTCHAR, ++ .hooks = { ++ .argparse = mock_based_notifyfenced_argparse_hook, ++ //.destroy = mock_based_notifyfenced_destroy_hook, ++ /* specialized hooks */ ++ .cib_notify = mock_based_notifyfenced_cib_notify_hook, ++ } ++ }); ++} +diff --git a/maint/mocked/based.c b/maint/mocked/based.c +index 451a384..d340274 100644 +--- a/maint/mocked/based.c ++++ b/maint/mocked/based.c +@@ -138,6 +138,7 @@ mock_based_common_callback_worker(uint32_t id, uint32_t flags, + xmlNode *op_request, crm_client_t *cib_client) + { + const char *op = crm_element_value(op_request, F_CIB_OPERATION); ++ mock_based_context_t *ctxt; + + if (!strcmp(op, CRM_OP_REGISTER)) { + if (flags & crm_ipc_client_response) { +@@ -161,6 +162,13 @@ mock_based_common_callback_worker(uint32_t id, uint32_t flags, + cib_client->options |= cib_notify_diff; + } + ++ ctxt = (mock_based_context_t *) cib_client->userdata; ++ for (size_t c = ctxt->modules_cnt; c > 0; c--) { ++ if (ctxt->modules[c - 1]->hooks.cib_notify != NULL) { ++ ctxt->modules[c - 1]->hooks.cib_notify(cib_client); ++ } ++ } ++ + if (flags & crm_ipc_client_response) { + crm_ipcs_send_ack(cib_client, id, flags, "ack", __FUNCTION__, __LINE__); + } +diff --git a/maint/mocked/based.h b/maint/mocked/based.h +index 04d8eed..dcebf0e 100644 +--- a/maint/mocked/based.h ++++ b/maint/mocked/based.h +@@ -29,6 +29,7 @@ typedef int (*mock_based_argparse_hook)(mock_based_context_t *, + typedef void (*mock_based_destroy_hook)(struct module_s *); + + /* specialized callbacks... */ ++typedef void (*mock_based_cib_notify_hook)(crm_client_t *); + + typedef struct mock_based_hooks_s { + /* generic ones */ +@@ -36,6 +37,7 @@ typedef struct mock_based_hooks_s { + mock_based_destroy_hook destroy; + + /* specialized callbacks... */ ++ mock_based_cib_notify_hook cib_notify; + } mock_based_hooks_t; + + typedef struct module_s { +-- +1.8.3.1 + + +From eee76118f2a557a79bda0214ea5c0974a7cd40c2 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= +Date: Wed, 29 Aug 2018 15:49:58 +0200 +Subject: [PATCH 03/96] Low: mainloop: make it possible to specify server's + priority in mainloop + +--- + include/crm/common/mainloop.h | 24 +++++++++++++ + lib/common/mainloop.c | 82 +++++++++++++++++++++++++++++++++++++++++-- + 2 files changed, 103 insertions(+), 3 deletions(-) + +diff --git a/include/crm/common/mainloop.h b/include/crm/common/mainloop.h +index eab31ac..e00da48 100644 +--- a/include/crm/common/mainloop.h ++++ b/include/crm/common/mainloop.h +@@ -67,6 +67,30 @@ struct ipc_client_callbacks { + qb_ipcs_service_t *mainloop_add_ipc_server(const char *name, enum qb_ipc_type type, + struct qb_ipcs_service_handlers *callbacks); + ++/*! ++ * \brief Start server-side API end-point, hooked into the internal event loop ++ * ++ * \param[in] name name of the IPC end-point ("address" for the client) ++ * \param[in] type selects libqb's IPC back-end (or use #QB_IPC_NATIVE) ++ * \param[in] callbacks defines libqb's IPC service-level handlers ++ * \param[in] priority priority relative to other events handled in the ++ * abstract handling loop, use #QB_LOOP_MED when unsure ++ * ++ * \return libqb's opaque handle to the created service abstraction ++ * ++ * \note For portability concerns, do not use this function if you keep ++ * \p priority as #QB_LOOP_MED, stick with #mainloop_add_ipc_server ++ * (with exactly such semantics) instead (once you link with this new ++ * symbol employed, you can't downgrade the library freely anymore). ++ * ++ * \note The intended effect will only get fully reflected when run-time ++ * linked to patched libqb: https://github.com/ClusterLabs/libqb/pull/352 ++ */ ++qb_ipcs_service_t *mainloop_add_ipc_server_with_prio(const char *name, ++ enum qb_ipc_type type, ++ struct qb_ipcs_service_handlers *callbacks, ++ enum qb_loop_priority prio); ++ + void mainloop_del_ipc_server(qb_ipcs_service_t * server); + + mainloop_io_t *mainloop_add_ipc_client(const char *name, int priority, size_t max_size, +diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c +index 60726cb..9bdd026 100644 +--- a/lib/common/mainloop.c ++++ b/lib/common/mainloop.c +@@ -456,6 +456,65 @@ gio_poll_destroy(gpointer data) + } + } + ++/*! ++ * \internal ++ * \brief Convert libqb's poll priority into GLib's one ++ * ++ * \param[in] prio libqb's poll priority (#QB_LOOP_MED assumed as fallback) ++ * ++ * \return best matching GLib's priority ++ */ ++static gint ++conv_prio_libqb2glib(enum qb_loop_priority prio) ++{ ++ gint ret = G_PRIORITY_DEFAULT; ++ switch (prio) { ++ case QB_LOOP_LOW: ++ ret = G_PRIORITY_LOW; ++ break; ++ case QB_LOOP_HIGH: ++ ret = G_PRIORITY_HIGH; ++ break; ++ default: ++ crm_trace("Invalid libqb's loop priority %d, assuming QB_LOOP_MED", ++ prio); ++ /* fall-through */ ++ case QB_LOOP_MED: ++ break; ++ } ++ return ret; ++} ++ ++/*! ++ * \internal ++ * \brief Convert libqb's poll priority to rate limiting spec ++ * ++ * \param[in] prio libqb's poll priority (#QB_LOOP_MED assumed as fallback) ++ * ++ * \return best matching rate limiting spec ++ */ ++static enum qb_ipcs_rate_limit ++conv_libqb_prio2ratelimit(enum qb_loop_priority prio) ++{ ++ /* this is an inversion of what libqb's qb_ipcs_request_rate_limit does */ ++ enum qb_ipcs_rate_limit ret = QB_IPCS_RATE_NORMAL; ++ switch (prio) { ++ case QB_LOOP_LOW: ++ ret = QB_IPCS_RATE_SLOW; ++ break; ++ case QB_LOOP_HIGH: ++ ret = QB_IPCS_RATE_FAST; ++ break; ++ default: ++ crm_trace("Invalid libqb's loop priority %d, assuming QB_LOOP_MED", ++ prio); ++ /* fall-through */ ++ case QB_LOOP_MED: ++ break; ++ } ++ return ret; ++} ++ + static int32_t + gio_poll_dispatch_update(enum qb_loop_priority p, int32_t fd, int32_t evts, + void *data, qb_ipcs_dispatch_fn_t fn, int32_t add) +@@ -502,8 +561,8 @@ gio_poll_dispatch_update(enum qb_loop_priority p, int32_t fd, int32_t evts, + adaptor->p = p; + adaptor->is_used++; + adaptor->source = +- g_io_add_watch_full(channel, G_PRIORITY_DEFAULT, evts, gio_read_socket, adaptor, +- gio_poll_destroy); ++ g_io_add_watch_full(channel, conv_prio_libqb2glib(p), evts, ++ gio_read_socket, adaptor, gio_poll_destroy); + + /* Now that mainloop now holds a reference to channel, + * thanks to g_io_add_watch_full(), drop ours from g_io_channel_unix_new(). +@@ -587,7 +646,15 @@ pick_ipc_type(enum qb_ipc_type requested) + + qb_ipcs_service_t * + mainloop_add_ipc_server(const char *name, enum qb_ipc_type type, +- struct qb_ipcs_service_handlers * callbacks) ++ struct qb_ipcs_service_handlers *callbacks) ++{ ++ return mainloop_add_ipc_server_with_prio(name, type, callbacks, QB_LOOP_MED); ++} ++ ++qb_ipcs_service_t * ++mainloop_add_ipc_server_with_prio(const char *name, enum qb_ipc_type type, ++ struct qb_ipcs_service_handlers *callbacks, ++ enum qb_loop_priority prio) + { + int rc = 0; + qb_ipcs_service_t *server = NULL; +@@ -599,6 +666,15 @@ mainloop_add_ipc_server(const char *name, enum qb_ipc_type type, + crm_client_init(); + server = qb_ipcs_create(name, 0, pick_ipc_type(type), callbacks); + ++ if (server == NULL) { ++ crm_err("Could not create %s IPC server: %s (%d)", name, pcmk_strerror(rc), rc); ++ return NULL; ++ } ++ ++ if (prio != QB_LOOP_MED) { ++ qb_ipcs_request_rate_limit(server, conv_libqb_prio2ratelimit(prio)); ++ } ++ + #ifdef HAVE_IPCS_GET_BUFFER_SIZE + /* All clients should use at least ipc_buffer_max as their buffer size */ + qb_ipcs_enforce_buffer_size(server, crm_ipc_default_buffer_size()); +-- +1.8.3.1 + + +From 428a9c873b661947af1e142ec8fa9fcf85328dcd Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= +Date: Wed, 29 Aug 2018 15:50:57 +0200 +Subject: [PATCH 04/96] High: stonith-ng's function cannot be blocked with CIB + updates forever + +In the high-load (or high-rate-config-change) scenarios, +pacemaker-fenced would be unable to provide service when basically DoS'd +with CIB update notifications. Try to reconcile that with elevated +priority of the server's proper listening interface in the mainloop, at +worst, it will try to fence with slightly outdated config, but appears +to be less bad than not carrying the execution at all, for instance. +Other daemons might be considered as well. + +Prerequisites: +- https://github.com/ClusterLabs/libqb/pull/352 + (libqb used to contain a bug due to which one particular step in the + initial-client-connection-accepting-at-the-server procedure that would + be carried out with hard-coded (and hence possibly lower than competing + events') priority, which backfires exactly in this case (once the + pacemaker part is fixed -- by the means of elevating priority for + the API end-point of fenced so that it won't get consistently + overridden with a non-socket-based event source/trigger) + +How to verify: +- mocked/based -N (see commit adding that module to mocked based daemon) +--- + lib/common/utils.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/lib/common/utils.c b/lib/common/utils.c +index f3f60ed..b87454e 100644 +--- a/lib/common/utils.c ++++ b/lib/common/utils.c +@@ -1223,7 +1223,8 @@ attrd_ipc_server_init(qb_ipcs_service_t **ipcs, struct qb_ipcs_service_handlers + void + stonith_ipc_server_init(qb_ipcs_service_t **ipcs, struct qb_ipcs_service_handlers *cb) + { +- *ipcs = mainloop_add_ipc_server("stonith-ng", QB_IPC_NATIVE, cb); ++ *ipcs = mainloop_add_ipc_server_with_prio("stonith-ng", QB_IPC_NATIVE, cb, ++ QB_LOOP_HIGH); + + if (*ipcs == NULL) { + crm_err("Failed to create stonith-ng servers: exiting and inhibiting respawn."); +-- +1.8.3.1 + + +From 75f507e6432e414c78938fc83d18493a998c98b6 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= +Date: Wed, 5 Jun 2019 15:12:23 +0200 +Subject: [PATCH 05/96] Doc: Pacemaker Development: intro of mocked daemons + +Partly as a documentation for ourselves. +--- + doc/Pacemaker_Development/en-US/Ch-Hacking.txt | 52 ++++++++++++++++++++++ + .../en-US/Pacemaker_Development.xml | 11 ++--- + .../en-US/Revision_History.xml | 13 ++++++ + 3 files changed, 71 insertions(+), 5 deletions(-) + create mode 100644 doc/Pacemaker_Development/en-US/Ch-Hacking.txt + +diff --git a/doc/Pacemaker_Development/en-US/Ch-Hacking.txt b/doc/Pacemaker_Development/en-US/Ch-Hacking.txt +new file mode 100644 +index 0000000..d8d8173 +--- /dev/null ++++ b/doc/Pacemaker_Development/en-US/Ch-Hacking.txt +@@ -0,0 +1,52 @@ ++:compat-mode: legacy ++= Advanced Hacking on the Project = ++ ++anchor:ch-hacking[Chapter 4. Hacking on Pacemaker] ++ ++[id="hacking-foreword"] ++== Foreword == ++ ++This chapter aims to be a gentle introduction (or perhaps, rather ++a summarization of advanced techniques we developed for backreferences) ++to how deal with the Pacemaker internals effectively. ++for instance, how to: ++ ++* verify various interesting interaction-based properties ++ ++or simply put, all that is in the interest of the core contributors ++on the project to know, master, and (preferably) also evolve ++-- way beyond what is in the presumed repertoire of a generic ++contributor role, which is detailed in other chapters of this guide. ++ ++Therefore, if you think you will not benefit from any such details ++in the scope of this chapter, feel free to skip it. ++ ++== Working with mocked daemons == ++ ++Since the Pacemaker run-time consists of multiple co-operating daemons ++as detailed elsewhere, tracking down the interaction details amongst ++them can be rather cumbersome. Since rebuilding existing daemons in ++a more modular way as opposed to clusters of mutually dependent ++functions, we elected to grow separate bare-bones counterparts built ++evolutionary as skeletons just to get the basic (long-term stabilized) ++communication with typical daemon clients going, and to add new modules ++in their outer circles (plus minimalistic hook support at those cores) ++on a demand-driven basis. ++ ++The code for these is located at `maint/mocked`; for instance, ++`based-notifyfenced.c` module of `based.c` skeleton mocking ++`pacemaker-based` daemon was exactly to fulfill investigation helper ++role (the case at hand was also an impulse to kick off this very ++sort of maintenance support material, to begin with). ++ ++Non-trivial knowledge of Pacemaker internals and other skills are ++needed to use such devised helpers, but given the other way around, ++some sorts of investigation may be even heftier, it may be the least ++effort choice. And when that's the case, advanced contributors are ++expected to contribute their own extensions they used to validate ++the reproducibility/actual correctness of the fix along the actual ++code modifications. This way, the rest of the development teams is ++not required to deal with elaborate preconditions, be at guess, or ++even forced to use a blind faith regarding the causes, consequences ++and validity regarding the raised issues/fixes, for the greater ++benefit of all. +diff --git a/doc/Pacemaker_Development/en-US/Pacemaker_Development.xml b/doc/Pacemaker_Development/en-US/Pacemaker_Development.xml +index 854d77a..6641d3b 100644 +--- a/doc/Pacemaker_Development/en-US/Pacemaker_Development.xml ++++ b/doc/Pacemaker_Development/en-US/Pacemaker_Development.xml +@@ -4,10 +4,11 @@ + %BOOK_ENTITIES; + ]> + +- +- +- +- +- ++ ++ ++ ++ ++ ++ + + +diff --git a/doc/Pacemaker_Development/en-US/Revision_History.xml b/doc/Pacemaker_Development/en-US/Revision_History.xml +index fd29d52..10ae102 100644 +--- a/doc/Pacemaker_Development/en-US/Revision_History.xml ++++ b/doc/Pacemaker_Development/en-US/Revision_History.xml +@@ -35,6 +35,19 @@ + + + ++ ++ 1-2 ++ Fri 17 May 2019 ++ ++ JanPokorný ++ poki@redhat.com ++ ++ ++ Start capturing hacking howto ++ for advanced contributors ++ ++ ++ + + + +-- +1.8.3.1 + + +From 96244e2c9bb4f32dc38bd4fa1c091c9f0d5bbf57 Mon Sep 17 00:00:00 2001 +From: "Gao,Yan" +Date: Fri, 26 Apr 2019 11:52:59 +0200 +Subject: [PATCH 06/96] Fix: libcrmcommon: correctly apply XML diffs with + multiple move/create changes + +Given a resource group: +``` + + + + + + + +``` + +, if we'd like to change it to: +``` + + + + + + + +``` + +, the generated XML diff would be like: +``` + + + + + + +``` + +Previously after applying the XML diff, the resulting XML would be a mess: +``` + + + + + + + +``` +It's because the positions of the already moved XML objects could be +affected by the later moved objects. + +This commit fixes it by temporarily putting "move" objects after the +last sibling and also delaying the adding of any "create" objects, then +placing them to the target positions in the right order. +--- + lib/common/xml.c | 126 ++++++++++++++++++++++++++++++++++++++++++------------- + 1 file changed, 97 insertions(+), 29 deletions(-) + +diff --git a/lib/common/xml.c b/lib/common/xml.c +index 6728247..f7d6c70 100644 +--- a/lib/common/xml.c ++++ b/lib/common/xml.c +@@ -1467,11 +1467,40 @@ __xml_find_path(xmlNode *top, const char *key, int target_position) + return target; + } + ++typedef struct xml_change_obj_s { ++ xmlNode *change; ++ xmlNode *match; ++} xml_change_obj_t; ++ ++static gint ++sort_change_obj_by_position(gconstpointer a, gconstpointer b) ++{ ++ const xml_change_obj_t *change_obj_a = a; ++ const xml_change_obj_t *change_obj_b = b; ++ int position_a = -1; ++ int position_b = -1; ++ ++ crm_element_value_int(change_obj_a->change, XML_DIFF_POSITION, &position_a); ++ crm_element_value_int(change_obj_b->change, XML_DIFF_POSITION, &position_b); ++ ++ if (position_a < position_b) { ++ return -1; ++ ++ } else if (position_a > position_b) { ++ return 1; ++ } ++ ++ return 0; ++} ++ + static int + xml_apply_patchset_v2(xmlNode *xml, xmlNode *patchset) + { + int rc = pcmk_ok; + xmlNode *change = NULL; ++ GListPtr change_objs = NULL; ++ GListPtr gIter = NULL; ++ + for (change = __xml_first_child(patchset); change != NULL; change = __xml_next(change)) { + xmlNode *match = NULL; + const char *op = crm_element_value(change, XML_DIFF_OP); +@@ -1483,6 +1512,7 @@ xml_apply_patchset_v2(xmlNode *xml, xmlNode *patchset) + continue; + } + ++ // "delete" changes for XML comments are generated with "position" + if(strcmp(op, "delete") == 0) { + crm_element_value_int(change, XML_DIFF_POSITION, &position); + } +@@ -1502,7 +1532,71 @@ xml_apply_patchset_v2(xmlNode *xml, xmlNode *patchset) + rc = -pcmk_err_diff_failed; + continue; + +- } else if(strcmp(op, "create") == 0) { ++ } else if (strcmp(op, "create") == 0 || strcmp(op, "move") == 0) { ++ // Delay the adding of a "create" object ++ xml_change_obj_t *change_obj = calloc(1, sizeof(xml_change_obj_t)); ++ ++ CRM_ASSERT(change_obj != NULL); ++ ++ change_obj->change = change; ++ change_obj->match = match; ++ ++ change_objs = g_list_append(change_objs, change_obj); ++ ++ if (strcmp(op, "move") == 0) { ++ // Temporarily put the "move" object after the last sibling ++ if (match->parent != NULL && match->parent->last != NULL) { ++ xmlAddNextSibling(match->parent->last, match); ++ } ++ } ++ ++ } else if(strcmp(op, "delete") == 0) { ++ free_xml(match); ++ ++ } else if(strcmp(op, "modify") == 0) { ++ xmlAttr *pIter = pcmk__first_xml_attr(match); ++ xmlNode *attrs = __xml_first_child(first_named_child(change, XML_DIFF_RESULT)); ++ ++ if(attrs == NULL) { ++ rc = -ENOMSG; ++ continue; ++ } ++ while(pIter != NULL) { ++ const char *name = (const char *)pIter->name; ++ ++ pIter = pIter->next; ++ xml_remove_prop(match, name); ++ } ++ ++ for (pIter = pcmk__first_xml_attr(attrs); pIter != NULL; pIter = pIter->next) { ++ const char *name = (const char *)pIter->name; ++ const char *value = crm_element_value(attrs, name); ++ ++ crm_xml_add(match, name, value); ++ } ++ ++ } else { ++ crm_err("Unknown operation: %s", op); ++ } ++ } ++ ++ // Changes should be generated in the right order. Double checking. ++ change_objs = g_list_sort(change_objs, sort_change_obj_by_position); ++ ++ for (gIter = change_objs; gIter; gIter = gIter->next) { ++ xml_change_obj_t *change_obj = gIter->data; ++ xmlNode *match = change_obj->match; ++ const char *op = NULL; ++ const char *xpath = NULL; ++ ++ change = change_obj->change; ++ ++ op = crm_element_value(change, XML_DIFF_OP); ++ xpath = crm_element_value(change, XML_DIFF_PATH); ++ ++ crm_trace("Continue performing %s on %s with %p", op, xpath, match); ++ ++ if(strcmp(op, "create") == 0) { + int position = 0; + xmlNode *child = NULL; + xmlNode *match_child = NULL; +@@ -1570,36 +1664,10 @@ xml_apply_patchset_v2(xmlNode *xml, xmlNode *patchset) + match->name, ID(match), __xml_offset(match), position, match->prev); + rc = -pcmk_err_diff_failed; + } +- +- } else if(strcmp(op, "delete") == 0) { +- free_xml(match); +- +- } else if(strcmp(op, "modify") == 0) { +- xmlAttr *pIter = pcmk__first_xml_attr(match); +- xmlNode *attrs = __xml_first_child(first_named_child(change, XML_DIFF_RESULT)); +- +- if(attrs == NULL) { +- rc = -ENOMSG; +- continue; +- } +- while(pIter != NULL) { +- const char *name = (const char *)pIter->name; +- +- pIter = pIter->next; +- xml_remove_prop(match, name); +- } +- +- for (pIter = pcmk__first_xml_attr(attrs); pIter != NULL; pIter = pIter->next) { +- const char *name = (const char *)pIter->name; +- const char *value = crm_element_value(attrs, name); +- +- crm_xml_add(match, name, value); +- } +- +- } else { +- crm_err("Unknown operation: %s", op); + } + } ++ ++ g_list_free_full(change_objs, free); + return rc; + } + +-- +1.8.3.1 + + +From a3de5c611febf265880c17a8b49267eaa968c741 Mon Sep 17 00:00:00 2001 +From: "Gao,Yan" +Date: Tue, 30 Apr 2019 00:15:03 +0200 +Subject: [PATCH 07/96] Fix: libcrmcommon: avoid possible use-of-NULL when + applying XML diffs + +--- + lib/common/xml.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/lib/common/xml.c b/lib/common/xml.c +index f7d6c70..5f52600 100644 +--- a/lib/common/xml.c ++++ b/lib/common/xml.c +@@ -1507,11 +1507,12 @@ xml_apply_patchset_v2(xmlNode *xml, xmlNode *patchset) + const char *xpath = crm_element_value(change, XML_DIFF_PATH); + int position = -1; + +- crm_trace("Processing %s %s", change->name, op); + if(op == NULL) { + continue; + } + ++ crm_trace("Processing %s %s", change->name, op); ++ + // "delete" changes for XML comments are generated with "position" + if(strcmp(op, "delete") == 0) { + crm_element_value_int(change, XML_DIFF_POSITION, &position); +-- +1.8.3.1 + + +From f40dad3645bfee99ad6b2bafbe69242171c44cf3 Mon Sep 17 00:00:00 2001 +From: "Gao,Yan" +Date: Tue, 30 Apr 2019 00:19:46 +0200 +Subject: [PATCH 08/96] Fix: libcrmcommon: return error when applying XML diffs + containing unknown operations + +--- + lib/common/xml.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/lib/common/xml.c b/lib/common/xml.c +index 5f52600..91c0edb 100644 +--- a/lib/common/xml.c ++++ b/lib/common/xml.c +@@ -1578,6 +1578,7 @@ xml_apply_patchset_v2(xmlNode *xml, xmlNode *patchset) + + } else { + crm_err("Unknown operation: %s", op); ++ rc = -pcmk_err_diff_failed; + } + } + +-- +1.8.3.1 + + +From 3a8c61bc744a3f59d097403231a4f3f3c39a990c Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 22 Mar 2019 17:49:30 -0500 +Subject: [PATCH 09/96] Log: controller: improve failed recurring action + messages + +Recurring action status changes can be reported in any later transition, not +just the one they were initially scheduled in. Previously, they would be logged +as an "Old event". Now, distinguish this situation. +--- + crmd/te_events.c | 34 +++++++++++++++++++--------------- + 1 file changed, 19 insertions(+), 15 deletions(-) + +diff --git a/crmd/te_events.c b/crmd/te_events.c +index 1f7a34c..eb1a8ca 100644 +--- a/crmd/te_events.c ++++ b/crmd/te_events.c +@@ -1,19 +1,10 @@ + /* +- * Copyright (C) 2004 Andrew Beekhof ++ * Copyright 2004-2019 the Pacemaker project contributors + * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of the GNU General Public +- * License as published by the Free Software Foundation; either +- * version 2 of the License, or (at your option) any later version. ++ * The version control history for this file may have further details. + * +- * This software is distributed in the hope that it will be useful, +- * but WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * General Public License for more details. +- * +- * You should have received a copy of the GNU General Public +- * License along with this library; if not, write to the Free Software +- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA ++ * This source code is licensed under the GNU General Public License version 2 ++ * or later (GPLv2+) WITHOUT ANY WARRANTY. + */ + + #include +@@ -495,8 +486,21 @@ process_graph_event(xmlNode *event, const char *event_node) + abort_transition(INFINITY, tg_restart, "Foreign event", event); + + } else if (transition_graph->id != transition_num) { +- desc = "arrived really late"; +- abort_transition(INFINITY, tg_restart, "Old event", event); ++ int interval_ms = 0; ++ ++ if (parse_op_key(id, NULL, NULL, &interval_ms) ++ && (interval_ms != 0)) { ++ /* Recurring actions have the transition number they were first ++ * scheduled in. ++ */ ++ desc = "arrived after initial scheduling"; ++ abort_transition(INFINITY, tg_restart, "Change in recurring result", ++ event); ++ ++ } else { ++ desc = "arrived really late"; ++ abort_transition(INFINITY, tg_restart, "Old event", event); ++ } + + } else if (transition_graph->complete) { + desc = "arrived late"; +-- +1.8.3.1 + + +From dac669b390cfb1be265c71cb588fa0a67ecdd929 Mon Sep 17 00:00:00 2001 +From: "Gao,Yan" +Date: Thu, 9 May 2019 13:24:35 +0200 +Subject: [PATCH 10/96] Fix: controller: confirm cancel of failed monitors + +Usually after a monitor has been cancelled from executor, contoller +erases the corresponding lrm_rsc_op from the cib, and DC will confirm +the cancel action by process_op_deletion() according to the cib diff. + +But if a monitor has failed, the lrm_rsc_op will be recorded as +"last_failure". When cancelling it, the lrm_rsc_op won't get erased from +the cib given the logic on purpose in erase_lrm_history_by_op(). So that +the cancel action won't have a chance to get confirmed by DC with +process_op_deletion(). + +Previously cluster transition would get stuck waiting for the remaining +action timer to time out. + +This commit fixes the issue by directly acknowledging the cancel action +in this case and enabling DC to be able to confirm it. + +This also moves get_node_id() function into controld_utils.c for common +use. + +Producer: +``` + # Insert a 10s sleep in the monitor action of RA + # /usr/lib/ocf/resource.d/pacemaker/Stateful: + + stateful_monitor() { + + sleep 10 + stateful_check_state "master" + + # Add a promotable clone resource: + + crm configure primitive stateful ocf:pacemaker:Stateful \ + op monitor interval=5 role=Master \ + op monitor interval=10 role=Slave + crm configure clone p-clone stateful \ + meta promotable=true + + # Wait for the resource instance to be started, promoted to be master, + # and monitor for master role to complete. + + # Set is-managed=false for the promotable clone: + crm_resource --meta -p is-managed -v false -r p-clone + + # Change the status of the master instance to be slave and immediately + # enforce refresh of it: + echo slave > /var/run/Stateful-stateful.state; crm_resource --refresh -r stateful --force + + # Wait for probe to complete, and then monitor for slave role to be + # issued: + sleep 15 + + # While the monitor for slave role is still in progress, change the + # status to be master again: + echo master > /var/run/Stateful-stateful.state + + # The monitor for slave role returns error. Cluster issues monitor for + # master role instead and tries to cancel the failed one for slave role. + # But cluster transition gets stuck. Depending on the monitor timeout + # configured for the slave role plus cluster-delay, only after that + # controller eventually says: + + pacemaker-controld[21205] error: Node opensuse150 did not send cancel result (via controller) within 20000ms (action timeout plus cluster-delay) + pacemaker-controld[21205] error: [Action 1]: In-flight rsc op stateful_monitor_10000 on opensuse150 (priority: 0, waiting: none) + pacemaker-controld[21205] notice: Transition 6 aborted: Action lost + +``` +--- + crmd/crmd_utils.h | 2 ++ + crmd/lrm.c | 38 ++++++++++++++++++++++++++++++++++++++ + crmd/te_callbacks.c | 21 ++------------------- + crmd/te_events.c | 32 ++++++++++++++++++++++++++++++++ + crmd/tengine.h | 1 + + crmd/utils.c | 13 +++++++++++++ + 6 files changed, 88 insertions(+), 19 deletions(-) + +diff --git a/crmd/crmd_utils.h b/crmd/crmd_utils.h +index d49642f..a754487 100644 +--- a/crmd/crmd_utils.h ++++ b/crmd/crmd_utils.h +@@ -112,6 +112,8 @@ void crmd_peer_down(crm_node_t *peer, bool full); + unsigned int cib_op_timeout(void); + bool controld_action_is_recordable(const char *action); + ++const char *get_node_id(xmlNode *lrm_rsc_op); ++ + /* Convenience macro for registering a CIB callback + * (assumes that data can be freed with free()) + */ +diff --git a/crmd/lrm.c b/crmd/lrm.c +index 1c9a276..776c02b 100644 +--- a/crmd/lrm.c ++++ b/crmd/lrm.c +@@ -2487,6 +2487,30 @@ unescape_newlines(const char *string) + return ret; + } + ++static bool ++did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id, ++ const char * op_type, guint interval_ms) ++{ ++ rsc_history_t *entry = NULL; ++ ++ CRM_CHECK(lrm_state != NULL, return FALSE); ++ CRM_CHECK(rsc_id != NULL, return FALSE); ++ CRM_CHECK(op_type != NULL, return FALSE); ++ ++ entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id); ++ if (entry == NULL || entry->failed == NULL) { ++ return FALSE; ++ } ++ ++ if (crm_str_eq(entry->failed->rsc_id, rsc_id, TRUE) ++ && safe_str_eq(entry->failed->op_type, op_type) ++ && entry->failed->interval == interval_ms) { ++ return TRUE; ++ } ++ ++ return FALSE; ++} ++ + void + process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, + struct recurring_op_s *pending, xmlNode *action_xml) +@@ -2616,6 +2640,20 @@ process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, + erase_lrm_history_by_op(lrm_state, op); + } + ++ /* If the recurring operation had failed, the lrm_rsc_op is recorded as ++ * "last_failure" which won't get erased from the cib given the logic on ++ * purpose in erase_lrm_history_by_op(). So that the cancel action won't ++ * have a chance to get confirmed by DC with process_op_deletion(). ++ * Cluster transition would get stuck waiting for the remaining action ++ * timer to time out. ++ * ++ * Directly acknowledge the cancel operation in this case. ++ */ ++ if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id, ++ pending->op_type, pending->interval)) { ++ need_direct_ack = TRUE; ++ } ++ + } else if (op->rsc_deleted) { + /* This recurring operation was cancelled (but not by us, and the + * executor does not have resource information, likely due to resource +diff --git a/crmd/te_callbacks.c b/crmd/te_callbacks.c +index 087f3e1..9faf932 100644 +--- a/crmd/te_callbacks.c ++++ b/crmd/te_callbacks.c +@@ -42,19 +42,6 @@ static unsigned long int stonith_max_attempts = 10; + /* #define rsc_op_template "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_CIB_TAG_STATE"[@uname='%s']"//"XML_LRM_TAG_RSC_OP"[@id='%s]" */ + #define rsc_op_template "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_LRM_TAG_RSC_OP"[@id='%s']" + +-static const char * +-get_node_id(xmlNode * rsc_op) +-{ +- xmlNode *node = rsc_op; +- +- while (node != NULL && safe_str_neq(XML_CIB_TAG_STATE, TYPE(node))) { +- node = node->parent; +- } +- +- CRM_CHECK(node != NULL, return NULL); +- return ID(node); +-} +- + void + update_stonith_max_attempts(const char* value) + { +@@ -384,12 +371,8 @@ process_op_deletion(const char *xpath, xmlNode *change) + node_uuid = extract_node_uuid(xpath); + cancel = get_cancel_action(key, node_uuid); + if (cancel) { +- crm_info("Cancellation of %s on %s confirmed (%d)", +- key, node_uuid, cancel->id); +- stop_te_timer(cancel->timer); +- te_action_confirmed(cancel); +- update_graph(transition_graph, cancel); +- trigger_graph(); ++ confirm_cancel_action(cancel); ++ + } else { + abort_transition(INFINITY, tg_restart, "Resource operation removal", + change); +diff --git a/crmd/te_events.c b/crmd/te_events.c +index eb1a8ca..b398739 100644 +--- a/crmd/te_events.c ++++ b/crmd/te_events.c +@@ -373,6 +373,27 @@ get_cancel_action(const char *id, const char *node) + return NULL; + } + ++void ++confirm_cancel_action(crm_action_t *cancel) ++{ ++ const char *op_key = NULL; ++ const char *node_name = NULL; ++ ++ CRM_ASSERT(cancel != NULL); ++ ++ op_key = crm_element_value(cancel->xml, XML_LRM_ATTR_TASK_KEY); ++ node_name = crm_element_value(cancel->xml, XML_LRM_ATTR_TARGET); ++ ++ stop_te_timer(cancel->timer); ++ te_action_confirmed(cancel); ++ update_graph(transition_graph, cancel); ++ ++ crm_info("Cancellation of %s on %s confirmed (action %d)", ++ op_key, node_name, cancel->id); ++ ++ trigger_graph(); ++} ++ + /* downed nodes are listed like: ... */ + #define XPATH_DOWNED "//" XML_GRAPH_TAG_DOWNED \ + "/" XML_CIB_TAG_NODE "[@" XML_ATTR_UUID "='%s']" +@@ -493,6 +514,17 @@ process_graph_event(xmlNode *event, const char *event_node) + /* Recurring actions have the transition number they were first + * scheduled in. + */ ++ ++ if (status == PCMK_LRM_OP_CANCELLED) { ++ const char *node_id = get_node_id(event); ++ ++ action = get_cancel_action(id, node_id); ++ if (action) { ++ confirm_cancel_action(action); ++ } ++ goto bail; ++ } ++ + desc = "arrived after initial scheduling"; + abort_transition(INFINITY, tg_restart, "Change in recurring result", + event); +diff --git a/crmd/tengine.h b/crmd/tengine.h +index b5141a0..1a9b2d2 100644 +--- a/crmd/tengine.h ++++ b/crmd/tengine.h +@@ -35,6 +35,7 @@ void execute_stonith_cleanup(void); + /* tengine */ + extern crm_action_t *match_down_event(const char *target, bool quiet); + extern crm_action_t *get_cancel_action(const char *id, const char *node); ++void confirm_cancel_action(crm_action_t *cancel); + + void controld_record_action_timeout(crm_action_t *action); + extern gboolean fail_incompletable_actions(crm_graph_t * graph, const char *down_node); +diff --git a/crmd/utils.c b/crmd/utils.c +index 08abc6e..761f5a7 100644 +--- a/crmd/utils.c ++++ b/crmd/utils.c +@@ -1054,3 +1054,16 @@ cib_op_timeout() + } + return calculated_timeout; + } ++ ++const char * ++get_node_id(xmlNode *lrm_rsc_op) ++{ ++ xmlNode *node = lrm_rsc_op; ++ ++ while (node != NULL && safe_str_neq(XML_CIB_TAG_STATE, TYPE(node))) { ++ node = node->parent; ++ } ++ ++ CRM_CHECK(node != NULL, return NULL); ++ return ID(node); ++} +-- +1.8.3.1 + + +From a81ca9625e8d1ccd7f79fbe464b9f4221e8671f2 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 9 May 2019 20:26:08 -0500 +Subject: [PATCH 11/96] Refactor: libpe_status: functionize unfencing digest + code more + +... for readability, reusability, and avoiding unnecessary function calls or +memory allocation. +--- + lib/pengine/utils.c | 159 ++++++++++++++++++++++++++++++++++++++-------------- + 1 file changed, 118 insertions(+), 41 deletions(-) + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index d09b0d8..b6a31d1 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -2091,57 +2091,134 @@ rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, + return data; + } + ++/*! ++ * \internal ++ * \brief Create an unfencing summary for use in special node attribute ++ * ++ * Create a string combining a fence device's resource ID, agent type, and ++ * parameter digest (whether for all parameters or just non-private parameters). ++ * This can be stored in a special node attribute, allowing us to detect changes ++ * in either the agent type or parameters, to know whether unfencing must be ++ * redone or can be safely skipped when the device's history is cleaned. ++ * ++ * \param[in] rsc_id Fence device resource ID ++ * \param[in] agent_type Fence device agent ++ * \param[in] param_digest Fence device parameter digest ++ * ++ * \return Newly allocated string with unfencing digest ++ * \note The caller is responsible for freeing the result. ++ */ ++static inline char * ++create_unfencing_summary(const char *rsc_id, const char *agent_type, ++ const char *param_digest) ++{ ++ return crm_strdup_printf("%s:%s:%s", rsc_id, agent_type, param_digest); ++} ++ ++/*! ++ * \internal ++ * \brief Check whether a node can skip unfencing ++ * ++ * Check whether a fence device's current definition matches a node's ++ * stored summary of when it was last unfenced by the device. ++ * ++ * \param[in] rsc_id Fence device's resource ID ++ * \param[in] agent Fence device's agent type ++ * \param[in] digest_calc Fence device's current parameter digest ++ * \param[in] node_summary Value of node's special unfencing node attribute ++ * (a comma-separated list of unfencing summaries for ++ * all devices that have unfenced this node) ++ * ++ * \return TRUE if digest matches, FALSE otherwise ++ */ ++static bool ++unfencing_digest_matches(const char *rsc_id, const char *agent, ++ const char *digest_calc, const char *node_summary) ++{ ++ bool matches = FALSE; ++ ++ if (rsc_id && agent && digest_calc && node_summary) { ++ char *search_secure = create_unfencing_summary(rsc_id, agent, ++ digest_calc); ++ ++ /* The digest was calculated including the device ID and agent, ++ * so there is no risk of collision using strstr(). ++ */ ++ matches = (strstr(node_summary, search_secure) != NULL); ++ crm_trace("Calculated unfencing digest '%s' %sfound in '%s'", ++ search_secure, matches? "" : "not ", node_summary); ++ free(search_secure); ++ } ++ return matches; ++} ++ ++/* Magic string to use as action name for digest cache entries used for ++ * unfencing checks. This is not a real action name (i.e. "on"), so ++ * check_action_definition() won't confuse these entries with real actions. ++ */ + #define STONITH_DIGEST_TASK "stonith-on" + ++/*! ++ * \internal ++ * \brief Calculate fence device digests and digest comparison result ++ * ++ * \param[in] rsc Fence device resource ++ * \param[in] agent Fence device's agent type ++ * \param[in] node Node with digest cache to use ++ * \param[in] data_set Cluster working set ++ * ++ * \return Node's digest cache entry ++ */ + static op_digest_cache_t * +-fencing_action_digest_cmp(resource_t * rsc, node_t * node, pe_working_set_t * data_set) ++fencing_action_digest_cmp(pe_resource_t *rsc, const char *agent, ++ pe_node_t *node, pe_working_set_t *data_set) + { +- char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); +- op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, node, NULL, data_set); ++ const char *node_summary = NULL; + +- const char *digest_all = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL); +- const char *digest_secure = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE); ++ // Calculate device's current parameter digests ++ char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); ++ op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, ++ node, NULL, data_set); + +- /* No 'reloads' for fencing device changes +- * +- * We use the resource id + agent + digest so that we can detect +- * changes to the agent and/or the parameters used +- */ +- char *search_all = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_all_calc); +- char *search_secure = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_secure_calc); ++ free(key); + +- data->rc = RSC_DIGEST_ALL; +- if (digest_all == NULL) { +- /* it is unknown what the previous op digest was */ ++ // Check whether node has special unfencing summary node attribute ++ node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL); ++ if (node_summary == NULL) { + data->rc = RSC_DIGEST_UNKNOWN; ++ return data; ++ } + +- } else if (strstr(digest_all, search_all)) { ++ // Check whether full parameter digest matches ++ if (unfencing_digest_matches(rsc->id, agent, data->digest_all_calc, ++ node_summary)) { + data->rc = RSC_DIGEST_MATCH; ++ return data; ++ } + +- } else if(digest_secure && data->digest_secure_calc) { +- if(strstr(digest_secure, search_secure)) { +- if (is_set(data_set->flags, pe_flag_stdout)) { +- printf("Only 'private' parameters to %s for unfencing %s changed\n", +- rsc->id, node->details->uname); +- } +- data->rc = RSC_DIGEST_MATCH; ++ // Check whether secure parameter digest matches ++ node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE); ++ if (unfencing_digest_matches(rsc->id, agent, data->digest_secure_calc, ++ node_summary)) { ++ data->rc = RSC_DIGEST_MATCH; ++ if (is_set(data_set->flags, pe_flag_stdout)) { ++ printf("Only 'private' parameters to %s for unfencing %s changed\n", ++ rsc->id, node->details->uname); + } ++ return data; + } + +- if (is_set(data_set->flags, pe_flag_sanitized) +- && is_set(data_set->flags, pe_flag_stdout) +- && (data->rc == RSC_DIGEST_ALL) ++ // Parameters don't match ++ data->rc = RSC_DIGEST_ALL; ++ if (is_set(data_set->flags, (pe_flag_sanitized|pe_flag_stdout)) + && data->digest_secure_calc) { +- printf("Parameters to %s for unfencing %s changed, try '%s:%s:%s'\n", +- rsc->id, node->details->uname, rsc->id, +- (const char *) g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), +- data->digest_secure_calc); +- } +- +- free(key); +- free(search_all); +- free(search_secure); ++ char *digest = create_unfencing_summary(rsc->id, agent, ++ data->digest_secure_calc); + ++ printf("Parameters to %s for unfencing %s changed, try '%s'\n", ++ rsc->id, node->details->uname, digest); ++ free(digest); ++ } + return data; + } + +@@ -2228,9 +2305,6 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe + * + * We may do this for all nodes in the future, but for now + * the check_action_definition() based stuff works fine. +- * +- * Use "stonith-on" to avoid creating cache entries for +- * operations check_action_definition() would look for. + */ + long max = 1024; + long digests_all_offset = 0; +@@ -2242,8 +2316,11 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe + + for (GListPtr gIter = matches; gIter != NULL; gIter = gIter->next) { + resource_t *match = gIter->data; +- op_digest_cache_t *data = fencing_action_digest_cmp(match, node, data_set); ++ const char *agent = g_hash_table_lookup(match->meta, ++ XML_ATTR_TYPE); ++ op_digest_cache_t *data = NULL; + ++ data = fencing_action_digest_cmp(match, agent, node, data_set); + if(data->rc == RSC_DIGEST_ALL) { + optional = FALSE; + crm_notice("Unfencing %s (remote): because the definition of %s changed", node->details->uname, match->id); +@@ -2254,11 +2331,11 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe + + digests_all_offset += snprintf( + digests_all+digests_all_offset, max-digests_all_offset, +- "%s:%s:%s,", match->id, (const char*)g_hash_table_lookup(match->meta, XML_ATTR_TYPE), data->digest_all_calc); ++ "%s:%s:%s,", match->id, agent, data->digest_all_calc); + + digests_secure_offset += snprintf( + digests_secure+digests_secure_offset, max-digests_secure_offset, +- "%s:%s:%s,", match->id, (const char*)g_hash_table_lookup(match->meta, XML_ATTR_TYPE), data->digest_secure_calc); ++ "%s:%s:%s,", match->id, agent, data->digest_secure_calc); + } + g_hash_table_insert(stonith_op->meta, + strdup(XML_OP_ATTR_DIGESTS_ALL), +-- +1.8.3.1 + + +From be34a73f9cfb6abdb3e2799593cb0358c01c2521 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 10 May 2019 11:57:31 -0500 +Subject: [PATCH 12/96] Fix: libpe_status: calculate secure digests for + unfencing ops + +The calculation of digests for detection of when unfencing is needed reused +rsc_action_digest(). However that would only add secure digests when the +pe_flag_sanitized flag was set, which is only set by crm_simulate, so secure +digests would never be added in normal cluster operation. This led to +node attributes like name="#digests-secure" +value="stonith-fence_compute-fence-nova:fence_compute:(null),". + +Now, rsc_action_digest() takes a new argument to select whether secure digests +are added, which is always set to TRUE when calculating unfencing digests. +--- + lib/pengine/utils.c | 27 ++++++++++++++++++++++----- + 1 file changed, 22 insertions(+), 5 deletions(-) + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index b6a31d1..f52f1c7 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -1948,9 +1948,24 @@ append_versioned_params(xmlNode *versioned_params, const char *ra_version, xmlNo + } + #endif + ++/*! ++ * \internal ++ * \brief Calculate action digests and store in node's digest cache ++ * ++ * \param[in] rsc Resource that action was for ++ * \param[in] task Name of action performed ++ * \param[in] key Action's task key ++ * \param[in] node Node action was performed on ++ * \param[in] xml_op XML of operation in CIB status (if available) ++ * \param[in] calc_secure Whether to calculate secure digest ++ * \param[in] data_set Cluster working set ++ * ++ * \return Pointer to node's digest cache entry ++ */ + static op_digest_cache_t * +-rsc_action_digest(resource_t * rsc, const char *task, const char *key, +- node_t * node, xmlNode * xml_op, pe_working_set_t * data_set) ++rsc_action_digest(pe_resource_t *rsc, const char *task, const char *key, ++ pe_node_t *node, xmlNode *xml_op, bool calc_secure, ++ pe_working_set_t *data_set) + { + op_digest_cache_t *data = NULL; + +@@ -2018,7 +2033,7 @@ rsc_action_digest(resource_t * rsc, const char *task, const char *key, + + data->digest_all_calc = calculate_operation_digest(data->params_all, op_version); + +- if (is_set(data_set->flags, pe_flag_sanitized)) { ++ if (calc_secure) { + data->params_secure = copy_xml(data->params_all); + if(secure_list) { + filter_parameters(data->params_secure, secure_list, FALSE); +@@ -2064,7 +2079,9 @@ rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, + + interval = crm_parse_int(interval_s, "0"); + key = generate_op_key(rsc->id, task, interval); +- data = rsc_action_digest(rsc, task, key, node, xml_op, data_set); ++ data = rsc_action_digest(rsc, task, key, node, xml_op, ++ is_set(data_set->flags, pe_flag_sanitized), ++ data_set); + + data->rc = RSC_DIGEST_MATCH; + if (digest_restart && data->digest_restart_calc && strcmp(data->digest_restart_calc, digest_restart) != 0) { +@@ -2178,7 +2195,7 @@ fencing_action_digest_cmp(pe_resource_t *rsc, const char *agent, + // Calculate device's current parameter digests + char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); + op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, +- node, NULL, data_set); ++ node, NULL, TRUE, data_set); + + free(key); + +-- +1.8.3.1 + + +From 8819c2f96f74ab4b4979df5ed04c16dd6bdad5f1 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Sat, 8 Jun 2019 16:25:04 -0500 +Subject: [PATCH 13/96] Refactor: libpe_status: add function for checking + shutdown attribute + +... to reduce code duplication and allow further reuse +--- + include/crm/pengine/internal.h | 2 ++ + lib/pengine/unpack.c | 8 ++------ + lib/pengine/utils.c | 20 ++++++++++++++++++++ + 3 files changed, 24 insertions(+), 6 deletions(-) + +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index c40b075..c3f9f70 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -362,4 +362,6 @@ void pe__foreach_param_check(pe_working_set_t *data_set, + enum pe_check_parameters, + pe_working_set_t*)); + void pe__free_param_checks(pe_working_set_t *data_set); ++ ++bool pe__shutdown_requested(pe_node_t *node); + #endif +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 619ccbf..cf725a1 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -1013,7 +1013,6 @@ unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * + const char *resource_discovery_enabled = NULL; + xmlNode *attrs = NULL; + resource_t *rsc = NULL; +- const char *shutdown = NULL; + + if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) { + return; +@@ -1035,8 +1034,7 @@ unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * + attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE); + add_node_attrs(attrs, this_node, TRUE, data_set); + +- shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); +- if (shutdown != NULL && safe_str_neq("0", shutdown)) { ++ if (pe__shutdown_requested(this_node)) { + crm_info("Node %s is shutting down", this_node->details->uname); + this_node->details->shutdown = TRUE; + if (rsc) { +@@ -1512,7 +1510,6 @@ gboolean + determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set_t * data_set) + { + gboolean online = FALSE; +- const char *shutdown = NULL; + const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED); + + if (this_node == NULL) { +@@ -1522,9 +1519,8 @@ determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set + + this_node->details->shutdown = FALSE; + this_node->details->expected_up = FALSE; +- shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); + +- if (shutdown != NULL && safe_str_neq("0", shutdown)) { ++ if (pe__shutdown_requested(this_node)) { + this_node->details->shutdown = TRUE; + + } else if (safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) { +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index f52f1c7..8eac2ce 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -2522,3 +2522,23 @@ void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrit + } + } + } ++ ++/*! ++ * \internal ++ * \brief Check whether shutdown has been requested for a node ++ * ++ * \param[in] node Node to check ++ * ++ * \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise ++ * \note This differs from simply using node->details->shutdown in that it can ++ * be used before that has been determined (and in fact to determine it), ++ * and it can also be used to distinguish requested shutdown from implicit ++ * shutdown of remote nodes by virtue of their connection stopping. ++ */ ++bool ++pe__shutdown_requested(pe_node_t *node) ++{ ++ const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN); ++ ++ return shutdown && strcmp(shutdown, "0"); ++} +-- +1.8.3.1 + + +From 938e99f29ed5faaeb4015247e363ddc7e77208a3 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 5 Jun 2019 16:37:26 -0500 +Subject: [PATCH 14/96] Fix: scheduler: remote state is failed if node is + shutting down with connection failure + +When determining remote state, if the connection resource is failed and not +being started again, we consider the state to be unknown if the connection has +a reconnect interval, because we won't know whether the connection can be +recovered until the interval expires and we re-attempt connection. + +However, if the node is shutting down at the time, we won't re-attempt +connection, so consider the state failed in that case. (Note that we check the +actual shutdown node attribute, rather than node->details->shutdown, since that +is set for remote nodes whenever the connection is stopping.) + +This avoids a situation where actions that cannot succeed can be scheduled on a +remote node that's shutting down. +--- + pengine/allocate.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 578db2f..c9877a4 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1998,7 +1998,8 @@ get_remote_node_state(pe_node_t *node) + + if ((remote_rsc->next_role == RSC_ROLE_STOPPED) + && remote_rsc->remote_reconnect_interval +- && node->details->remote_was_fenced) { ++ && node->details->remote_was_fenced ++ && !pe__shutdown_requested(node)) { + + /* We won't know whether the connection is recoverable until the + * reconnect interval expires and we reattempt connection. +-- +1.8.3.1 + + +From c20f8920634f47bbdf699d80dafd50c6a72eac8b Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 5 Jun 2019 16:43:19 -0500 +Subject: [PATCH 15/96] Fix: libpe_status: don't order implied stops relative + to a remote connection + +Actions behind a remote connection are ordered relative to any start or stop of +the remote connection. However, if the action is a stop implied due to fencing, +it does not require the remote connection, and the ordering should not be done. + +This avoids a delay in the remote connection recovery if it is failed, e.g. +previously the ordering would look like: + + fence remote node -> implied stop of resource on remote -> stop connection + +Now, the connection stop can proceed simultaneously with the remote node +fencing. +--- + pengine/allocate.c | 11 +++++------ + 1 file changed, 5 insertions(+), 6 deletions(-) + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index c9877a4..c7c68f8 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -2091,14 +2091,13 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) + pe_order_implies_first, data_set); + + } else if(state == remote_state_failed) { +- /* We would only be here if the resource is +- * running on the remote node. Since we have no +- * way to stop it, it is necessary to fence the +- * node. ++ /* The resource is active on the node, but since we don't have a ++ * valid connection, the only way to stop the resource is by ++ * fencing the node. There is no need to order the stop relative ++ * to the remote connection, since the stop will become implied ++ * by the fencing. + */ + pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable"); +- order_action_then_stop(action, remote_rsc, +- pe_order_implies_first, data_set); + + } else if(remote_rsc->next_role == RSC_ROLE_STOPPED) { + /* State must be remote_state_unknown or remote_state_stopped. +-- +1.8.3.1 + + +From 26a28ee80b7fc110125eedac377dfa4c0a8e8294 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 14 Jun 2019 14:08:47 -0500 +Subject: [PATCH 16/96] Test: pengine: update regression tests for remote + connection ordering change + +--- + pengine/test10/remote-connection-unrecoverable.dot | 2 -- + pengine/test10/remote-connection-unrecoverable.exp | 6 ------ + pengine/test10/remote-connection-unrecoverable.summary | 2 +- + pengine/test10/remote-fence-before-reconnect.dot | 1 - + pengine/test10/remote-fence-before-reconnect.exp | 6 +----- + pengine/test10/remote-fence-before-reconnect.summary | 2 +- + pengine/test10/remote-recover-all.dot | 2 -- + pengine/test10/remote-recover-all.exp | 6 ------ + pengine/test10/remote-recover-all.summary | 4 ++-- + pengine/test10/remote-recover-no-resources.dot | 1 - + pengine/test10/remote-recover-no-resources.exp | 3 --- + pengine/test10/remote-recover-no-resources.summary | 2 +- + pengine/test10/remote-recover-unknown.dot | 1 - + pengine/test10/remote-recover-unknown.exp | 3 --- + pengine/test10/remote-recover-unknown.summary | 2 +- + 15 files changed, 7 insertions(+), 36 deletions(-) + +diff --git a/pengine/test10/remote-connection-unrecoverable.dot b/pengine/test10/remote-connection-unrecoverable.dot +index 0360cd0..b5caca6 100644 +--- a/pengine/test10/remote-connection-unrecoverable.dot ++++ b/pengine/test10/remote-connection-unrecoverable.dot +@@ -7,14 +7,12 @@ digraph "g" { + "remote1_stop_0 node1" [ style=bold color="green" fontcolor="orange"] + "rsc1_delete_0 remote1" -> "rsc1_start_0 node2" [ style = dashed] + "rsc1_delete_0 remote1" [ style=dashed color="red" fontcolor="black"] +-"rsc1_monitor_0 node2" -> "remote1_stop_0 node1" [ style = bold] + "rsc1_monitor_0 node2" -> "rsc1_start_0 node2" [ style = bold] + "rsc1_monitor_0 node2" -> "rsc2-master_demote_0" [ style = bold] + "rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black"] + "rsc1_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] + "rsc1_start_0 node2" -> "rsc1_monitor_10000 node2" [ style = bold] + "rsc1_start_0 node2" [ style=bold color="green" fontcolor="black"] +-"rsc1_stop_0 remote1" -> "remote1_stop_0 node1" [ style = bold] + "rsc1_stop_0 remote1" -> "rsc1_delete_0 remote1" [ style = dashed] + "rsc1_stop_0 remote1" -> "rsc1_start_0 node2" [ style = bold] + "rsc1_stop_0 remote1" -> "rsc2-master_demote_0" [ style = bold] +diff --git a/pengine/test10/remote-connection-unrecoverable.exp b/pengine/test10/remote-connection-unrecoverable.exp +index 73fa7a1..339ad56 100644 +--- a/pengine/test10/remote-connection-unrecoverable.exp ++++ b/pengine/test10/remote-connection-unrecoverable.exp +@@ -9,12 +9,6 @@ + + + +- +- +- +- +- +- + + + +diff --git a/pengine/test10/remote-connection-unrecoverable.summary b/pengine/test10/remote-connection-unrecoverable.summary +index efeb765..18f7dc7 100644 +--- a/pengine/test10/remote-connection-unrecoverable.summary ++++ b/pengine/test10/remote-connection-unrecoverable.summary +@@ -24,12 +24,12 @@ Executing cluster transition: + * Resource action: killer stop on node2 + * Resource action: rsc1 monitor on node2 + * Fencing node1 (reboot) ++ * Pseudo action: remote1_stop_0 + * Fencing remote1 (reboot) + * Resource action: killer start on node2 + * Resource action: killer monitor=60000 on node2 + * Pseudo action: rsc1_stop_0 + * Pseudo action: rsc2-master_demote_0 +- * Pseudo action: remote1_stop_0 + * Resource action: rsc1 start on node2 + * Pseudo action: rsc2_demote_0 + * Pseudo action: rsc2-master_demoted_0 +diff --git a/pengine/test10/remote-fence-before-reconnect.dot b/pengine/test10/remote-fence-before-reconnect.dot +index 4ced43e..5812b7f 100644 +--- a/pengine/test10/remote-fence-before-reconnect.dot ++++ b/pengine/test10/remote-fence-before-reconnect.dot +@@ -3,7 +3,6 @@ + "fake2_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"] + "fake2_start_0 c7auto1" -> "fake2_monitor_10000 c7auto1" [ style = bold] + "fake2_start_0 c7auto1" [ style=bold color="green" fontcolor="black"] +-"fake2_stop_0 c7auto4" -> "c7auto4_stop_0 c7auto1" [ style = bold] + "fake2_stop_0 c7auto4" -> "fake2_start_0 c7auto1" [ style = bold] + "fake2_stop_0 c7auto4" [ style=bold color="green" fontcolor="orange"] + "stonith 'reboot' c7auto4" -> "fake2_start_0 c7auto1" [ style = bold] +diff --git a/pengine/test10/remote-fence-before-reconnect.exp b/pengine/test10/remote-fence-before-reconnect.exp +index f99d9ef..f506f85 100644 +--- a/pengine/test10/remote-fence-before-reconnect.exp ++++ b/pengine/test10/remote-fence-before-reconnect.exp +@@ -9,11 +9,7 @@ + + + +- +- +- +- +- ++ + + + +diff --git a/pengine/test10/remote-fence-before-reconnect.summary b/pengine/test10/remote-fence-before-reconnect.summary +index f61e18b..03eac20 100644 +--- a/pengine/test10/remote-fence-before-reconnect.summary ++++ b/pengine/test10/remote-fence-before-reconnect.summary +@@ -17,9 +17,9 @@ Transition Summary: + * Move fake2 ( c7auto4 -> c7auto1 ) + + Executing cluster transition: ++ * Resource action: c7auto4 stop on c7auto1 + * Fencing c7auto4 (reboot) + * Pseudo action: fake2_stop_0 +- * Resource action: c7auto4 stop on c7auto1 + * Resource action: fake2 start on c7auto1 + * Resource action: fake2 monitor=10000 on c7auto1 + +diff --git a/pengine/test10/remote-recover-all.dot b/pengine/test10/remote-recover-all.dot +index 1f967c5..b48b04e 100644 +--- a/pengine/test10/remote-recover-all.dot ++++ b/pengine/test10/remote-recover-all.dot +@@ -19,7 +19,6 @@ digraph "g" { + "galera_demote_0 galera-2" -> "galera_stop_0 galera-2" [ style = bold] + "galera_demote_0 galera-2" [ style=bold color="green" fontcolor="orange"] + "galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"] +-"galera_stop_0 galera-2" -> "galera-2_stop_0 controller-1" [ style = bold] + "galera_stop_0 galera-2" -> "galera-master_stopped_0" [ style = bold] + "galera_stop_0 galera-2" [ style=bold color="green" fontcolor="orange"] + "haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold] +@@ -60,7 +59,6 @@ digraph "g" { + "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] + "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] + "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] +-"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] + "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] + "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] + "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] +diff --git a/pengine/test10/remote-recover-all.exp b/pengine/test10/remote-recover-all.exp +index 900781c..e61ad6a 100644 +--- a/pengine/test10/remote-recover-all.exp ++++ b/pengine/test10/remote-recover-all.exp +@@ -9,9 +9,6 @@ + + + +- +- +- + + + +@@ -64,9 +61,6 @@ + + + +- +- +- + + + +diff --git a/pengine/test10/remote-recover-all.summary b/pengine/test10/remote-recover-all.summary +index 865f39a..cfeac3a 100644 +--- a/pengine/test10/remote-recover-all.summary ++++ b/pengine/test10/remote-recover-all.summary +@@ -63,6 +63,8 @@ Executing cluster transition: + * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 + * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 + * Fencing controller-1 (reboot) ++ * Pseudo action: messaging-1_stop_0 ++ * Pseudo action: galera-2_stop_0 + * Pseudo action: redis_post_notify_stop_0 + * Resource action: redis notify on controller-0 + * Resource action: redis notify on controller-2 +@@ -94,7 +96,6 @@ Executing cluster transition: + * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 + * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 + * Resource action: galera-0 monitor=20000 on controller-2 +- * Pseudo action: galera-2_stop_0 + * Resource action: rabbitmq notify on messaging-2 + * Resource action: rabbitmq notify on messaging-0 + * Pseudo action: rabbitmq_notified_0 +@@ -107,7 +108,6 @@ Executing cluster transition: + * Resource action: ip-172.17.1.17 start on controller-2 + * Resource action: ip-172.17.4.11 start on controller-2 + * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 +- * Pseudo action: messaging-1_stop_0 + * Pseudo action: redis_notified_0 + * Resource action: ip-172.17.1.14 monitor=10000 on controller-2 + * Resource action: ip-172.17.1.17 monitor=10000 on controller-2 +diff --git a/pengine/test10/remote-recover-no-resources.dot b/pengine/test10/remote-recover-no-resources.dot +index a46c305..a0b1ecc 100644 +--- a/pengine/test10/remote-recover-no-resources.dot ++++ b/pengine/test10/remote-recover-no-resources.dot +@@ -45,7 +45,6 @@ digraph "g" { + "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] + "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] + "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] +-"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] + "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] + "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] + "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] +diff --git a/pengine/test10/remote-recover-no-resources.exp b/pengine/test10/remote-recover-no-resources.exp +index 4d82aa4..27f18b5 100644 +--- a/pengine/test10/remote-recover-no-resources.exp ++++ b/pengine/test10/remote-recover-no-resources.exp +@@ -9,9 +9,6 @@ + + + +- +- +- + + + +diff --git a/pengine/test10/remote-recover-no-resources.summary b/pengine/test10/remote-recover-no-resources.summary +index 9527161..c01eb87 100644 +--- a/pengine/test10/remote-recover-no-resources.summary ++++ b/pengine/test10/remote-recover-no-resources.summary +@@ -60,6 +60,7 @@ Executing cluster transition: + * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 + * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 + * Fencing controller-1 (reboot) ++ * Pseudo action: messaging-1_stop_0 + * Pseudo action: galera-2_stop_0 + * Pseudo action: redis_post_notify_stop_0 + * Resource action: redis notify on controller-0 +@@ -92,7 +93,6 @@ Executing cluster transition: + * Pseudo action: ip-172.17.1.17_stop_0 + * Pseudo action: ip-172.17.4.11_stop_0 + * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 +- * Pseudo action: messaging-1_stop_0 + * Resource action: redis notify on controller-0 + * Resource action: redis notify on controller-2 + * Pseudo action: redis-master_confirmed-post_notify_stopped_0 +diff --git a/pengine/test10/remote-recover-unknown.dot b/pengine/test10/remote-recover-unknown.dot +index a883eb4..1d13e50 100644 +--- a/pengine/test10/remote-recover-unknown.dot ++++ b/pengine/test10/remote-recover-unknown.dot +@@ -46,7 +46,6 @@ digraph "g" { + "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] + "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] + "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] +-"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] + "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] + "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] + "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] +diff --git a/pengine/test10/remote-recover-unknown.exp b/pengine/test10/remote-recover-unknown.exp +index 65677b4..13bd295 100644 +--- a/pengine/test10/remote-recover-unknown.exp ++++ b/pengine/test10/remote-recover-unknown.exp +@@ -9,9 +9,6 @@ + + + +- +- +- + + + +diff --git a/pengine/test10/remote-recover-unknown.summary b/pengine/test10/remote-recover-unknown.summary +index 78a60d0..64f37cb 100644 +--- a/pengine/test10/remote-recover-unknown.summary ++++ b/pengine/test10/remote-recover-unknown.summary +@@ -61,6 +61,7 @@ Executing cluster transition: + * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 + * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 + * Fencing controller-1 (reboot) ++ * Pseudo action: messaging-1_stop_0 + * Pseudo action: galera-2_stop_0 + * Pseudo action: redis_post_notify_stop_0 + * Resource action: redis notify on controller-0 +@@ -94,7 +95,6 @@ Executing cluster transition: + * Pseudo action: ip-172.17.1.17_stop_0 + * Pseudo action: ip-172.17.4.11_stop_0 + * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 +- * Pseudo action: messaging-1_stop_0 + * Resource action: redis notify on controller-0 + * Resource action: redis notify on controller-2 + * Pseudo action: redis-master_confirmed-post_notify_stopped_0 +-- +1.8.3.1 + + +From 71142273e6b5108224ecdb0082b36f533b604fad Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Mon, 8 Jul 2019 18:17:28 +0200 +Subject: [PATCH 17/96] Fix: fence-history: fail leftover pending-actions after + stonithd-restart + +--- + fencing/history.c | 15 +++++++++++++++ + fencing/internal.h | 8 ++++++++ + fencing/remote.c | 6 +++--- + 3 files changed, 26 insertions(+), 3 deletions(-) + +diff --git a/fencing/history.c b/fencing/history.c +index 0f98058..c487848 100644 +--- a/fencing/history.c ++++ b/fencing/history.c +@@ -347,6 +347,21 @@ stonith_merge_in_history_list(GHashTable *history) + + updated = TRUE; + g_hash_table_iter_steal(&iter); ++ ++ if ((op->state != st_failed) && ++ (op->state != st_done) && ++ safe_str_eq(op->originator, stonith_our_uname)) { ++ crm_warn("received pending action we are supposed to be the " ++ "owner but it's not in our records -> fail it"); ++ op->state = st_failed; ++ op->completed = time(NULL); ++ /* use -EHOSTUNREACH to not introduce a new return-code that might ++ trigger unexpected results at other places and to prevent ++ remote_op_done from setting the delegate if not present ++ */ ++ stonith_bcast_result_to_peers(op, -EHOSTUNREACH); ++ } ++ + g_hash_table_insert(stonith_remote_op_list, op->id, op); + /* we could trim the history here but if we bail + * out after trim we might miss more recent entries +diff --git a/fencing/internal.h b/fencing/internal.h +index 028137f..cd48b53 100644 +--- a/fencing/internal.h ++++ b/fencing/internal.h +@@ -142,6 +142,14 @@ typedef struct remote_fencing_op_s { + + } remote_fencing_op_t; + ++/*! ++ * \internal ++ * \brief Broadcast the result of an operation to the peers. ++ * \param op, Operation whose result should be broadcast ++ * \param rc, Result of the operation ++ */ ++void stonith_bcast_result_to_peers(remote_fencing_op_t * op, int rc); ++ + enum st_callback_flags { + st_callback_unknown = 0x0000, + st_callback_notify_fence = 0x0001, +diff --git a/fencing/remote.c b/fencing/remote.c +index 866112f..6c5b9b8 100644 +--- a/fencing/remote.c ++++ b/fencing/remote.c +@@ -379,8 +379,8 @@ create_op_done_notify(remote_fencing_op_t * op, int rc) + return notify_data; + } + +-static void +-bcast_result_to_peers(remote_fencing_op_t * op, int rc) ++void ++stonith_bcast_result_to_peers(remote_fencing_op_t * op, int rc) + { + static int count = 0; + xmlNode *bcast = create_xml_node(NULL, T_STONITH_REPLY); +@@ -518,7 +518,7 @@ remote_op_done(remote_fencing_op_t * op, xmlNode * data, int rc, int dup) + subt = crm_element_value(data, F_SUBTYPE); + if (dup == FALSE && safe_str_neq(subt, "broadcast")) { + /* Defer notification until the bcast message arrives */ +- bcast_result_to_peers(op, rc); ++ stonith_bcast_result_to_peers(op, rc); + goto remote_op_done_cleanup; + } + +-- +1.8.3.1 + + +From 89c87a604b7a318df5a6fd66d5e077362d6717aa Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Mon, 8 Jul 2019 18:49:18 +0200 +Subject: [PATCH 18/96] Fix: st_client: make safe to remove notifications from + notifications + +While cycling over the notification-list just mark for deletion +and delete afterwards. +--- + lib/fencing/st_client.c | 58 +++++++++++++++++++++++++++++++++++++++++++++---- + 1 file changed, 54 insertions(+), 4 deletions(-) + +diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c +index c38f356..feb8c73 100644 +--- a/lib/fencing/st_client.c ++++ b/lib/fencing/st_client.c +@@ -77,6 +77,8 @@ typedef struct stonith_private_s { + mainloop_io_t *source; + GHashTable *stonith_op_callback_table; + GList *notify_list; ++ int notify_refcnt; ++ bool notify_deletes; + + void (*op_callback) (stonith_t * st, stonith_callback_data_t * data); + +@@ -87,6 +89,7 @@ typedef struct stonith_notify_client_s { + const char *obj_id; /* implement one day */ + const char *obj_type; /* implement one day */ + void (*notify) (stonith_t * st, stonith_event_t * e); ++ bool delete; + + } stonith_notify_client_t; + +@@ -223,6 +226,38 @@ log_action(stonith_action_t *action, pid_t pid) + } + } + ++/* when cycling through the list we don't want to delete items ++ so just mark them and when we know nobody is using the list ++ loop over it to remove the marked items ++ */ ++static void ++foreach_notify_entry (stonith_private_t *private, ++ GFunc func, ++ gpointer user_data) ++{ ++ private->notify_refcnt++; ++ g_list_foreach(private->notify_list, func, user_data); ++ private->notify_refcnt--; ++ if ((private->notify_refcnt == 0) && ++ private->notify_deletes) { ++ GList *list_item = private->notify_list; ++ ++ private->notify_deletes = FALSE; ++ while (list_item != NULL) ++ { ++ stonith_notify_client_t *list_client = list_item->data; ++ GList *next = g_list_next(list_item); ++ ++ if (list_client->delete) { ++ free(list_client); ++ private->notify_list = ++ g_list_delete_link(private->notify_list, list_item); ++ } ++ list_item = next; ++ } ++ } ++} ++ + static void + stonith_connection_destroy(gpointer user_data) + { +@@ -242,7 +277,7 @@ stonith_connection_destroy(gpointer user_data) + crm_xml_add(blob.xml, F_TYPE, T_STONITH_NOTIFY); + crm_xml_add(blob.xml, F_SUBTYPE, T_STONITH_NOTIFY_DISCONNECT); + +- g_list_foreach(native->notify_list, stonith_send_notification, &blob); ++ foreach_notify_entry(native, stonith_send_notification, &blob); + free_xml(blob.xml); + } + +@@ -1244,6 +1279,10 @@ stonithlib_GCompareFunc(gconstpointer a, gconstpointer b) + const stonith_notify_client_t *a_client = a; + const stonith_notify_client_t *b_client = b; + ++ if (a_client->delete || b_client->delete) { ++ /* make entries marked for deletion not findable */ ++ return -1; ++ } + CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0); + rc = strcmp(a_client->event, b_client->event); + if (rc == 0) { +@@ -1502,8 +1541,13 @@ stonith_api_del_notification(stonith_t * stonith, const char *event) + if (list_item != NULL) { + stonith_notify_client_t *list_client = list_item->data; + +- private->notify_list = g_list_remove(private->notify_list, list_client); +- free(list_client); ++ if (private->notify_refcnt) { ++ list_client->delete = TRUE; ++ private->notify_deletes = TRUE; ++ } else { ++ private->notify_list = g_list_remove(private->notify_list, list_client); ++ free(list_client); ++ } + + crm_trace("Removed callback"); + +@@ -1807,6 +1851,10 @@ stonith_send_notification(gpointer data, gpointer user_data) + crm_warn("Skipping callback - NULL callback client"); + return; + ++ } else if (entry->delete) { ++ crm_trace("Skipping callback - marked for deletion"); ++ return; ++ + } else if (entry->notify == NULL) { + crm_warn("Skipping callback - NULL callback"); + return; +@@ -1988,7 +2036,7 @@ stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata) + stonith_perform_callback(st, blob.xml, 0, 0); + + } else if (safe_str_eq(type, T_STONITH_NOTIFY)) { +- g_list_foreach(private->notify_list, stonith_send_notification, &blob); ++ foreach_notify_entry(private, stonith_send_notification, &blob); + } else if (safe_str_eq(type, T_STONITH_TIMEOUT_VALUE)) { + int call_id = 0; + int timeout = 0; +@@ -2135,6 +2183,8 @@ stonith_api_new(void) + private->stonith_op_callback_table = g_hash_table_new_full(g_direct_hash, g_direct_equal, + NULL, stonith_destroy_op_callback); + private->notify_list = NULL; ++ private->notify_refcnt = 0; ++ private->notify_deletes = FALSE; + + new_stonith->call_id = 1; + new_stonith->state = stonith_disconnected; +-- +1.8.3.1 + + +From 4625dd976d9e9cd08e0fefa0bbe057fb91510d98 Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Mon, 8 Jul 2019 19:12:53 +0200 +Subject: [PATCH 19/96] Feature: fence-history: add notification upon + history-synced + +--- + fencing/history.c | 5 +++++ + fencing/internal.h | 11 ++++++----- + fencing/main.c | 3 +++ + include/crm/stonith-ng.h | 1 + + 4 files changed, 15 insertions(+), 5 deletions(-) + +diff --git a/fencing/history.c b/fencing/history.c +index c487848..4f1fc4a 100644 +--- a/fencing/history.c ++++ b/fencing/history.c +@@ -420,6 +420,11 @@ stonith_fence_history(xmlNode *msg, xmlNode **output, + stonith_fence_history_cleanup(target, + crm_element_value(msg, F_STONITH_CALLID) != NULL); + } else if (options & st_opt_broadcast) { ++ /* there is no clear sign atm for when a history sync ++ is done so send a notification for anything ++ that smells like history-sync ++ */ ++ do_stonith_notify(0, T_STONITH_NOTIFY_HISTORY_SYNCED, 0, NULL); + if (crm_element_value(msg, F_STONITH_CALLID)) { + /* this is coming from the stonith-API + * +diff --git a/fencing/internal.h b/fencing/internal.h +index cd48b53..a51b0e6 100644 +--- a/fencing/internal.h ++++ b/fencing/internal.h +@@ -151,11 +151,12 @@ typedef struct remote_fencing_op_s { + void stonith_bcast_result_to_peers(remote_fencing_op_t * op, int rc); + + enum st_callback_flags { +- st_callback_unknown = 0x0000, +- st_callback_notify_fence = 0x0001, +- st_callback_device_add = 0x0004, +- st_callback_device_del = 0x0010, +- st_callback_notify_history = 0x0020 ++ st_callback_unknown = 0x0000, ++ st_callback_notify_fence = 0x0001, ++ st_callback_device_add = 0x0004, ++ st_callback_device_del = 0x0010, ++ st_callback_notify_history = 0x0020, ++ st_callback_notify_history_synced = 0x0040 + }; + + /* +diff --git a/fencing/main.c b/fencing/main.c +index 82bee86..624937e 100644 +--- a/fencing/main.c ++++ b/fencing/main.c +@@ -301,6 +301,9 @@ get_stonith_flag(const char *name) + } else if (safe_str_eq(name, T_STONITH_NOTIFY_HISTORY)) { + return st_callback_notify_history; + ++ } else if (safe_str_eq(name, T_STONITH_NOTIFY_HISTORY_SYNCED)) { ++ return st_callback_notify_history_synced; ++ + } + return st_callback_unknown; + } +diff --git a/include/crm/stonith-ng.h b/include/crm/stonith-ng.h +index 045521a..23f879b 100644 +--- a/include/crm/stonith-ng.h ++++ b/include/crm/stonith-ng.h +@@ -35,6 +35,7 @@ + # define T_STONITH_NOTIFY_DISCONNECT "st_notify_disconnect" + # define T_STONITH_NOTIFY_FENCE "st_notify_fence" + # define T_STONITH_NOTIFY_HISTORY "st_notify_history" ++# define T_STONITH_NOTIFY_HISTORY_SYNCED "st_notify_history_synced" + + /* *INDENT-OFF* */ + enum stonith_state { +-- +1.8.3.1 + + +From 732f069557e490a9ce1d1a5adfc74081c3e309c1 Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Mon, 8 Jul 2019 19:32:28 +0200 +Subject: [PATCH 20/96] Fix: crmd: remove-stonith-notifications upon + connection-destroy + +--- + crmd/te_utils.c | 11 +++++++++-- + 1 file changed, 9 insertions(+), 2 deletions(-) + +diff --git a/crmd/te_utils.c b/crmd/te_utils.c +index 280fc95..f5bcb84 100644 +--- a/crmd/te_utils.c ++++ b/crmd/te_utils.c +@@ -183,8 +183,15 @@ tengine_stonith_connection_destroy(stonith_t * st, stonith_event_t * e) + } + + /* cbchan will be garbage at this point, arrange for it to be reset */ +- if(stonith_api) { +- stonith_api->state = stonith_disconnected; ++ if (stonith_api) { ++ /* the client API won't properly reconnect notifications ++ * if they are still in the table - so remove them ++ */ ++ stonith_api->cmds->remove_notification(st, T_STONITH_NOTIFY_DISCONNECT); ++ stonith_api->cmds->remove_notification(st, T_STONITH_NOTIFY_FENCE); ++ if (stonith_api->state != stonith_disconnected) { ++ stonith_api->cmds->disconnect(st); ++ } + } + + if (AM_I_DC) { +-- +1.8.3.1 + + +From 199d9df653e8264fb09655dd5f8f1e94533612ee Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Mon, 8 Jul 2019 19:36:02 +0200 +Subject: [PATCH 21/96] Fix: crmd: add notice-log for successful fencer-connect + +--- + crmd/te_utils.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/crmd/te_utils.c b/crmd/te_utils.c +index f5bcb84..2805ea9 100644 +--- a/crmd/te_utils.c ++++ b/crmd/te_utils.c +@@ -496,7 +496,7 @@ te_connect_stonith(gpointer user_data) + stonith_api->cmds->register_notification(stonith_api, T_STONITH_NOTIFY_FENCE, + tengine_stonith_notify); + +- crm_trace("Connected"); ++ crm_notice("Fencer successfully connected"); + return TRUE; + } + +-- +1.8.3.1 + + +From ec0bb46caba533fa03b9bf4376f0f1ac5f70d393 Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Mon, 8 Jul 2019 19:40:42 +0200 +Subject: [PATCH 22/96] Test: CTS: new pattern to identify fenced reconnected + +Now that we are removing notifications upon disconnect a duplicate +notification can't be used as sign for reconnection any more. +--- + cts/patterns.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/cts/patterns.py b/cts/patterns.py +index ccd753d..87b44a9 100644 +--- a/cts/patterns.py ++++ b/cts/patterns.py +@@ -376,7 +376,7 @@ class crm_cs_v0(BasePatterns): + "LRMD lost STONITH connection", + "Connection to stonith-ng.* closed", + "Fencing daemon connection failed", +- r"crmd.*:\s*warn.*:\s*Callback already present", ++ r"pacemaker-controld.*Fencer successfully connected", + ] + self.components["stonith-ignore"] = [ + r"pengine.*: Recover Fencing", +-- +1.8.3.1 + + +From 5141b03fe7bffd493cd26c20412bbcf3041fa495 Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Mon, 8 Jul 2019 21:15:51 +0200 +Subject: [PATCH 23/96] Fix: fence-history: resync fence-history after stonithd + crash + +Setting up a 30s fallback timer to trigger history-sync if the +sync via DC doesn't happen +--- + crmd/callbacks.c | 2 +- + crmd/control.c | 9 ++++++- + crmd/te_utils.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++-------- + crmd/tengine.h | 3 ++- + 4 files changed, 79 insertions(+), 13 deletions(-) + +diff --git a/crmd/callbacks.c b/crmd/callbacks.c +index c51b215..7560470 100644 +--- a/crmd/callbacks.c ++++ b/crmd/callbacks.c +@@ -212,7 +212,7 @@ peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *d + } else { + crm_info("New peer %s we want to sync fence history with", + node->uname); +- te_trigger_stonith_history_sync(); ++ te_trigger_stonith_history_sync(FALSE); + } + } + break; +diff --git a/crmd/control.c b/crmd/control.c +index 488ea88..04935c7 100644 +--- a/crmd/control.c ++++ b/crmd/control.c +@@ -192,7 +192,12 @@ do_shutdown(long long action, + clear_bit(fsa_input_register, R_ST_REQUIRED); + + crm_info("Disconnecting STONITH..."); +- stonith_api->cmds->disconnect(stonith_api); ++ if (stonith_api->state != stonith_disconnected) { ++ stonith_api->cmds->disconnect(stonith_api); ++ } ++ stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT); ++ stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_FENCE); ++ stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_HISTORY_SYNCED); + } + } + +@@ -369,6 +374,8 @@ crmd_exit(int rc) + crm_timer_stop(wait_timer); + crm_timer_stop(recheck_timer); + ++ te_cleanup_stonith_history_sync(NULL, TRUE); ++ + free(transition_timer); transition_timer = NULL; + free(integration_timer); integration_timer = NULL; + free(finalization_timer); finalization_timer = NULL; +diff --git a/crmd/te_utils.c b/crmd/te_utils.c +index 2805ea9..29b411f 100644 +--- a/crmd/te_utils.c ++++ b/crmd/te_utils.c +@@ -33,7 +33,33 @@ + + crm_trigger_t *stonith_reconnect = NULL; + static crm_trigger_t *stonith_history_sync_trigger = NULL; +-static mainloop_timer_t *stonith_history_sync_timer = NULL; ++static mainloop_timer_t *stonith_history_sync_timer_short = NULL; ++static mainloop_timer_t *stonith_history_sync_timer_long = NULL; ++ ++void ++te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers) ++{ ++ if (free_timers) { ++ mainloop_timer_del(stonith_history_sync_timer_short); ++ stonith_history_sync_timer_short = NULL; ++ mainloop_timer_del(stonith_history_sync_timer_long); ++ stonith_history_sync_timer_long = NULL; ++ } else { ++ mainloop_timer_stop(stonith_history_sync_timer_short); ++ mainloop_timer_stop(stonith_history_sync_timer_long); ++ } ++ ++ if (st) { ++ st->cmds->remove_notification(st, T_STONITH_NOTIFY_HISTORY_SYNCED); ++ } ++} ++ ++static void ++tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event) ++{ ++ te_cleanup_stonith_history_sync(st, FALSE); ++ crm_debug("Fence-history synced - cancel all timers"); ++} + + /* + * stonith cleanup list +@@ -174,6 +200,8 @@ fail_incompletable_stonith(crm_graph_t * graph) + static void + tengine_stonith_connection_destroy(stonith_t * st, stonith_event_t * e) + { ++ te_cleanup_stonith_history_sync(st, FALSE); ++ + if (is_set(fsa_input_register, R_ST_REQUIRED)) { + crm_crit("Fencing daemon connection failed"); + mainloop_set_trigger(stonith_reconnect); +@@ -187,11 +215,12 @@ tengine_stonith_connection_destroy(stonith_t * st, stonith_event_t * e) + /* the client API won't properly reconnect notifications + * if they are still in the table - so remove them + */ +- stonith_api->cmds->remove_notification(st, T_STONITH_NOTIFY_DISCONNECT); +- stonith_api->cmds->remove_notification(st, T_STONITH_NOTIFY_FENCE); + if (stonith_api->state != stonith_disconnected) { + stonith_api->cmds->disconnect(st); + } ++ stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT); ++ stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_FENCE); ++ stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_HISTORY_SYNCED); + } + + if (AM_I_DC) { +@@ -212,6 +241,9 @@ char *te_client_id = NULL; + #endif + + static void ++tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event); ++ ++static void + tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event) + { + if(te_client_id == NULL) { +@@ -404,6 +436,7 @@ do_stonith_history_sync(gpointer user_data) + if (stonith_api && (stonith_api->state != stonith_disconnected)) { + stonith_history_t *history = NULL; + ++ te_cleanup_stonith_history_sync(stonith_api, FALSE); + stonith_api->cmds->history(stonith_api, + st_opt_sync_call | st_opt_broadcast, + NULL, &history, 5); +@@ -423,11 +456,18 @@ stonith_history_sync_set_trigger(gpointer user_data) + } + + void +-te_trigger_stonith_history_sync(void) ++te_trigger_stonith_history_sync(bool long_timeout) + { + /* trigger a sync in 5s to give more nodes the + * chance to show up so that we don't create + * unnecessary stonith-history-sync traffic ++ * ++ * the long timeout of 30s is there as a fallback ++ * so that after a successful connection to fenced ++ * we will wait for 30s for the DC to trigger a ++ * history-sync ++ * if this doesn't happen we trigger a sync locally ++ * (e.g. fenced segfaults and is restarted by pacemakerd) + */ + + /* as we are finally checking the stonith-connection +@@ -441,13 +481,25 @@ te_trigger_stonith_history_sync(void) + do_stonith_history_sync, NULL); + } + +- if(stonith_history_sync_timer == NULL) { +- stonith_history_sync_timer = +- mainloop_timer_add("history_sync", 5000, +- FALSE, stonith_history_sync_set_trigger, +- NULL); ++ if (long_timeout) { ++ if(stonith_history_sync_timer_long == NULL) { ++ stonith_history_sync_timer_long = ++ mainloop_timer_add("history_sync_long", 30000, ++ FALSE, stonith_history_sync_set_trigger, ++ NULL); ++ } ++ crm_info("Fence history will be synchronized cluster-wide within 30 seconds"); ++ mainloop_timer_start(stonith_history_sync_timer_long); ++ } else { ++ if(stonith_history_sync_timer_short == NULL) { ++ stonith_history_sync_timer_short = ++ mainloop_timer_add("history_sync_short", 5000, ++ FALSE, stonith_history_sync_set_trigger, ++ NULL); ++ } ++ crm_info("Fence history will be synchronized cluster-wide within 5 seconds"); ++ mainloop_timer_start(stonith_history_sync_timer_short); + } +- mainloop_timer_start(stonith_history_sync_timer); + } + + gboolean +@@ -496,6 +548,12 @@ te_connect_stonith(gpointer user_data) + stonith_api->cmds->register_notification(stonith_api, T_STONITH_NOTIFY_FENCE, + tengine_stonith_notify); + ++ stonith_api->cmds->register_notification(stonith_api, ++ T_STONITH_NOTIFY_HISTORY_SYNCED, ++ tengine_stonith_history_synced); ++ ++ te_trigger_stonith_history_sync(TRUE); ++ + crm_notice("Fencer successfully connected"); + return TRUE; + } +diff --git a/crmd/tengine.h b/crmd/tengine.h +index 1a9b2d2..a20760c 100644 +--- a/crmd/tengine.h ++++ b/crmd/tengine.h +@@ -72,7 +72,8 @@ extern void abort_transition_graph(int abort_priority, enum transition_action ab + + extern gboolean te_connect_stonith(gpointer user_data); + +-extern void te_trigger_stonith_history_sync(void); ++extern void te_trigger_stonith_history_sync(bool long_timeout); ++extern void te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers); + + extern crm_trigger_t *transition_trigger; + extern crm_trigger_t *stonith_reconnect; +-- +1.8.3.1 + + +From 926ae820d6406885dce8ba794215002b57bce17f Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Wed, 10 Jul 2019 17:57:02 +0200 +Subject: [PATCH 24/96] Fix: st_client: cleanup token whenever setting api to + disconnected + +--- + lib/fencing/st_client.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c +index feb8c73..60c07d5 100644 +--- a/lib/fencing/st_client.c ++++ b/lib/fencing/st_client.c +@@ -273,6 +273,7 @@ stonith_connection_destroy(gpointer user_data) + native->ipc = NULL; + native->source = NULL; + ++ free(native->token); native->token = NULL; + stonith->state = stonith_disconnected; + crm_xml_add(blob.xml, F_TYPE, T_STONITH_NOTIFY); + crm_xml_add(blob.xml, F_SUBTYPE, T_STONITH_NOTIFY_DISCONNECT); +@@ -1975,6 +1976,7 @@ stonith_send_command(stonith_t * stonith, const char *op, xmlNode * data, xmlNod + done: + if (crm_ipc_connected(native->ipc) == FALSE) { + crm_err("STONITH disconnected"); ++ free(native->token); native->token = NULL; + stonith->state = stonith_disconnected; + } + +-- +1.8.3.1 + + +From 365c29581f0a88f106622c000e8fe4b3e9edd024 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 27 Feb 2019 15:39:59 -0600 +Subject: [PATCH 25/96] Low: executor: consider stonith resource stopped only + if stop succeeded + +Previously, the executor would consider a stonith resource stopped whenever a +stop was requested, regardless of the actual result of remove_device(). One +could make a case for ignoring device unregistration failures, but it would be +unusual enough result that it makes sense to draw attention to it. +--- + lrmd/lrmd.c | 9 ++++++--- + 1 file changed, 6 insertions(+), 3 deletions(-) + +diff --git a/lrmd/lrmd.c b/lrmd/lrmd.c +index d6a431d..ecf0cc7 100644 +--- a/lrmd/lrmd.c ++++ b/lrmd/lrmd.c +@@ -1057,8 +1057,12 @@ stonith_action_complete(lrmd_cmd_t * cmd, int rc) + } else { + /* command successful */ + cmd->lrmd_op_status = PCMK_LRM_OP_DONE; +- if (safe_str_eq(cmd->action, "start") && rsc) { +- rsc->stonith_started = 1; ++ if (rsc) { ++ if (safe_str_eq(cmd->action, "start")) { ++ rsc->stonith_started = 1; ++ } else if (safe_str_eq(cmd->action, "stop")) { ++ rsc->stonith_started = 0; ++ } + } + } + +@@ -1160,7 +1164,6 @@ lrmd_rsc_execute_stonith(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd) + } + } else if (safe_str_eq(cmd->action, "stop")) { + rc = stonith_api->cmds->remove_device(stonith_api, st_opt_sync_call, cmd->rsc_id); +- rsc->stonith_started = 0; + } else if (safe_str_eq(cmd->action, "monitor")) { + if (cmd->interval) { + do_monitor = 1; +-- +1.8.3.1 + + +From 4e535413aaacb21aa11de3d1b48a90ed3a173825 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 10 Apr 2019 12:43:21 -0500 +Subject: [PATCH 26/96] Fix: executor: return error for stonith probes if + stonith connection was lost + +Previously, stonith probes could return only PCMK_OCF_OK (if the executor had +registered the device with the fencer) or PCMK_OCF_NOT_RUNNING (if the executor +had unregistered or not yet registered the device). + +However if the stonith connection is lost, the executor doesn't know whether +the device is still registered or not, and thus could be giving wrong +information back to the controller. + +This fixes that by refactoring lrmd_rsc_t's stonith_started member from a +boolean (0 = not started, 1 = started) to an rc code (pcmk_ok = started, +-ENODEV = not started, pcmk_err_generic = stonith connection lost). +stonith_rc2status() will map these to PCMK_OCF_OK, PCMK_OCF_NOT_RUNNING, or +PCMK_OCF_UNKNOWN_ERROR. + +This ensures that probes after the connection is lost will fail, which is +especially important if the controller respawned at the same time as the fencer +and so didn't receive client notification of failed monitors. + +This means that if the executor loses its stonith connection, probes for *all* +stonith devices on that node will fail and require a stop on that node, which +may be unexpected for users accustomed to the old behavior, but is more +correct. +--- + lrmd/lrmd.c | 15 ++++++++++++--- + lrmd/lrmd_private.h | 2 +- + 2 files changed, 13 insertions(+), 4 deletions(-) + +diff --git a/lrmd/lrmd.c b/lrmd/lrmd.c +index ecf0cc7..59751ec 100644 +--- a/lrmd/lrmd.c ++++ b/lrmd/lrmd.c +@@ -158,6 +158,7 @@ build_rsc_from_xml(xmlNode * msg) + rsc->provider = crm_element_value_copy(rsc_xml, F_LRMD_PROVIDER); + rsc->type = crm_element_value_copy(rsc_xml, F_LRMD_TYPE); + rsc->work = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_rsc_dispatch, rsc); ++ rsc->st_probe_rc = -ENODEV; // if stonith, initialize to "not running" + return rsc; + } + +@@ -1059,9 +1060,9 @@ stonith_action_complete(lrmd_cmd_t * cmd, int rc) + cmd->lrmd_op_status = PCMK_LRM_OP_DONE; + if (rsc) { + if (safe_str_eq(cmd->action, "start")) { +- rsc->stonith_started = 1; ++ rsc->st_probe_rc = pcmk_ok; // maps to PCMK_OCF_OK + } else if (safe_str_eq(cmd->action, "stop")) { +- rsc->stonith_started = 0; ++ rsc->st_probe_rc = -ENODEV; // maps to PCMK_OCF_NOT_RUNNING + } + } + } +@@ -1094,6 +1095,14 @@ stonith_connection_failed(void) + g_hash_table_iter_init(&iter, rsc_list); + while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & rsc)) { + if (safe_str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH)) { ++ /* This will cause future probes to return PCMK_OCF_UNKNOWN_ERROR ++ * until the resource is stopped or started successfully. This is ++ * especially important if the controller also went away (possibly ++ * due to a cluster layer restart) and won't receive our client ++ * notification of any monitors finalized below. ++ */ ++ rsc->st_probe_rc = pcmk_err_generic; ++ + if (rsc->active) { + cmd_list = g_list_append(cmd_list, rsc->active); + } +@@ -1168,7 +1177,7 @@ lrmd_rsc_execute_stonith(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd) + if (cmd->interval) { + do_monitor = 1; + } else { +- rc = rsc->stonith_started ? 0 : -ENODEV; ++ rc = rsc->st_probe_rc; + } + } + +diff --git a/lrmd/lrmd_private.h b/lrmd/lrmd_private.h +index 4449bb0..44de453 100644 +--- a/lrmd/lrmd_private.h ++++ b/lrmd/lrmd_private.h +@@ -54,7 +54,7 @@ typedef struct lrmd_rsc_s { + * that have been handed off from the pending ops list. */ + GList *recurring_ops; + +- int stonith_started; ++ int st_probe_rc; // What value should be returned for a probe if stonith + + crm_trigger_t *work; + } lrmd_rsc_t; +-- +1.8.3.1 + + +From 1268ce78d1be9841f5861892e4ed3d9013d38fdb Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 10 Apr 2019 12:51:48 -0500 +Subject: [PATCH 27/96] Fix: executor: don't cancel stonith monitors when + device is not registered + +This essentially reverts 53532a7. Now that failed stonith connections will +cause failures of future probes, we can properly leave the decision to cancel +monitors to the controller. Otherwise, the controller won't know that the +recurring action is no longer active, and could get action timeouts in some +cases. +--- + lrmd/lrmd.c | 12 +----------- + 1 file changed, 1 insertion(+), 11 deletions(-) + +diff --git a/lrmd/lrmd.c b/lrmd/lrmd.c +index 59751ec..5d33324 100644 +--- a/lrmd/lrmd.c ++++ b/lrmd/lrmd.c +@@ -1029,17 +1029,7 @@ stonith_action_complete(lrmd_cmd_t * cmd, int rc) + } else if (rc == -ENODEV && safe_str_eq(cmd->action, "monitor")) { + // The device is not registered with the fencer + +- if (recurring) { +- /* If we get here, the fencer somehow lost the registration of a +- * previously active device (possibly due to crash and respawn). In +- * that case, we need to indicate that the recurring monitor needs +- * to be cancelled. +- */ +- cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED; +- recurring = FALSE; +- } else { +- cmd->lrmd_op_status = PCMK_LRM_OP_DONE; +- } ++ cmd->lrmd_op_status = PCMK_LRM_OP_ERROR; + cmd->exec_rc = PCMK_OCF_NOT_RUNNING; + + } else if (rc) { +-- +1.8.3.1 + + +From ffd57df21eff2e0c96966be7069152b5b57c1870 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 28 Jun 2019 15:27:56 -0500 +Subject: [PATCH 28/96] Refactor: controller: functionize access to last + scheduler request ID + +This will come in handy for adding a new timer +--- + crmd/control.c | 3 +-- + crmd/crmd_utils.h | 2 ++ + crmd/pengine.c | 49 ++++++++++++++++++++++++++++++++++++++----------- + crmd/te_utils.c | 5 +---- + 4 files changed, 42 insertions(+), 17 deletions(-) + +diff --git a/crmd/control.c b/crmd/control.c +index 04935c7..73a2b08 100644 +--- a/crmd/control.c ++++ b/crmd/control.c +@@ -339,7 +339,6 @@ crmd_exit(int rc) + clear_bit(fsa_input_register, R_MEMBERSHIP); + g_list_free(fsa_message_queue); fsa_message_queue = NULL; + +- free(pe_subsystem); pe_subsystem = NULL; + free(te_subsystem); te_subsystem = NULL; + free(cib_subsystem); cib_subsystem = NULL; + +@@ -375,6 +374,7 @@ crmd_exit(int rc) + crm_timer_stop(recheck_timer); + + te_cleanup_stonith_history_sync(NULL, TRUE); ++ controld_sched_cleanup(); + + free(transition_timer); transition_timer = NULL; + free(integration_timer); integration_timer = NULL; +@@ -393,7 +393,6 @@ crmd_exit(int rc) + + free(te_uuid); te_uuid = NULL; + free(te_client_id); te_client_id = NULL; +- free(fsa_pe_ref); fsa_pe_ref = NULL; + free(failed_stop_offset); failed_stop_offset = NULL; + free(failed_start_offset); failed_start_offset = NULL; + +diff --git a/crmd/crmd_utils.h b/crmd/crmd_utils.h +index a754487..ce856b7 100644 +--- a/crmd/crmd_utils.h ++++ b/crmd/crmd_utils.h +@@ -83,6 +83,8 @@ int crmd_exit(int rc); + int crmd_fast_exit(int rc); + gboolean stop_subsystem(struct crm_subsystem_s *centry, gboolean force_quit); + gboolean start_subsystem(struct crm_subsystem_s *centry); ++void controld_expect_sched_reply(xmlNode *msg); ++void controld_sched_cleanup(void); + + void fsa_dump_actions(long long action, const char *text); + void fsa_dump_inputs(int log_level, const char *text, long long input_register); +diff --git a/crmd/pengine.c b/crmd/pengine.c +index 8ecb21d..b7d996a 100644 +--- a/crmd/pengine.c ++++ b/crmd/pengine.c +@@ -1,5 +1,7 @@ + /* +- * Copyright (C) 2004 Andrew Beekhof ++ * Copyright 2004-2019 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public +@@ -172,6 +174,36 @@ do_pe_control(long long action, + int fsa_pe_query = 0; + char *fsa_pe_ref = NULL; + ++/*! ++ * \internal ++ * \brief Set the scheduler request currently being waited on ++ * ++ * \param[in] msg Request to expect reply to (or NULL for none) ++ */ ++void ++controld_expect_sched_reply(xmlNode *msg) ++{ ++ char *ref = NULL; ++ ++ if (msg) { ++ ref = crm_element_value_copy(msg, XML_ATTR_REFERENCE); ++ CRM_ASSERT(ref != NULL); ++ } ++ free(fsa_pe_ref); ++ fsa_pe_ref = ref; ++} ++ ++/*! ++ * \internal ++ * \brief Clean up all memory used by controller scheduler handling ++ */ ++void ++controld_sched_cleanup() ++{ ++ free(pe_subsystem); pe_subsystem = NULL; ++ controld_expect_sched_reply(NULL); ++} ++ + /* A_PE_INVOKE */ + void + do_pe_invoke(long long action, +@@ -216,10 +248,7 @@ do_pe_invoke(long long action, + crm_debug("Query %d: Requesting the current CIB: %s", fsa_pe_query, + fsa_state2string(fsa_state)); + +- /* Make sure any queued calculations are discarded */ +- free(fsa_pe_ref); +- fsa_pe_ref = NULL; +- ++ controld_expect_sched_reply(NULL); + fsa_register_cib_callback(fsa_pe_query, FALSE, NULL, do_pe_invoke_callback); + } + +@@ -335,16 +364,14 @@ do_pe_invoke_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void + + cmd = create_request(CRM_OP_PECALC, output, NULL, CRM_SYSTEM_PENGINE, CRM_SYSTEM_DC, NULL); + +- free(fsa_pe_ref); +- fsa_pe_ref = crm_element_value_copy(cmd, XML_ATTR_REFERENCE); +- + sent = crm_ipc_send(mainloop_get_ipc_client(pe_subsystem->source), cmd, 0, 0, NULL); + if (sent <= 0) { + crm_err("Could not contact the pengine: %d", sent); + register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __FUNCTION__); ++ } else { ++ controld_expect_sched_reply(cmd); ++ crm_debug("Invoking the PE: query=%d, ref=%s, seq=%llu, quorate=%d", ++ fsa_pe_query, fsa_pe_ref, crm_peer_seq, fsa_has_quorum); + } +- +- crm_debug("Invoking the PE: query=%d, ref=%s, seq=%llu, quorate=%d", +- fsa_pe_query, fsa_pe_ref, crm_peer_seq, fsa_has_quorum); + free_xml(cmd); + } +diff --git a/crmd/te_utils.c b/crmd/te_utils.c +index 29b411f..14570cd 100644 +--- a/crmd/te_utils.c ++++ b/crmd/te_utils.c +@@ -717,10 +717,7 @@ abort_transition_graph(int abort_priority, enum transition_action abort_action, + } + + abort_timer.aborted = TRUE; +- +- /* Make sure any queued calculations are discarded ASAP */ +- free(fsa_pe_ref); +- fsa_pe_ref = NULL; ++ controld_expect_sched_reply(NULL); + + if (transition_graph->complete == FALSE) { + if(update_abort_priority(transition_graph, abort_priority, abort_action, abort_text)) { +-- +1.8.3.1 + + +From 61000ec5247d77ee539cc6988f826c20a6903105 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 28 Jun 2019 16:20:03 -0500 +Subject: [PATCH 29/96] Fix: controller: set timeout on scheduler responses + +Previously, once the DC successfully read the CIB and sent a calculation +request to the scheduler, it wouldn't do anything further with the request, +aside from the message handler for the scheduler's response. + +This meant that if the scheduler successfully accepted the request, but then +was unable to reply (such as not getting enough CPU cycles), the controller +would never detect anything wrong, and the cluster would be blocked. + +Now, the controller sets a 2-minute timer after handing off the request to the +scheduler, and if it doesn't get a response in that time, it exits and stays +down (if a node is elected DC but can't run the scheduler, we want to ensure it +doesn't interfere with further elections). +--- + crmd/crmd_utils.h | 1 + + crmd/election.c | 1 + + crmd/messages.c | 2 +- + crmd/pengine.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ + 4 files changed, 56 insertions(+), 1 deletion(-) + +diff --git a/crmd/crmd_utils.h b/crmd/crmd_utils.h +index ce856b7..a704380 100644 +--- a/crmd/crmd_utils.h ++++ b/crmd/crmd_utils.h +@@ -83,6 +83,7 @@ int crmd_exit(int rc); + int crmd_fast_exit(int rc); + gboolean stop_subsystem(struct crm_subsystem_s *centry, gboolean force_quit); + gboolean start_subsystem(struct crm_subsystem_s *centry); ++void controld_stop_sched_timer(void); + void controld_expect_sched_reply(xmlNode *msg); + void controld_sched_cleanup(void); + +diff --git a/crmd/election.c b/crmd/election.c +index cef82d0..a4313cc 100644 +--- a/crmd/election.c ++++ b/crmd/election.c +@@ -280,6 +280,7 @@ do_dc_release(long long action, + if (action & A_DC_RELEASE) { + crm_debug("Releasing the role of DC"); + clear_bit(fsa_input_register, R_THE_DC); ++ controld_expect_sched_reply(NULL); + + } else if (action & A_DC_RELEASED) { + crm_info("DC role released"); +diff --git a/crmd/messages.c b/crmd/messages.c +index 24ffac6..f1599ab 100644 +--- a/crmd/messages.c ++++ b/crmd/messages.c +@@ -1000,9 +1000,9 @@ handle_response(xmlNode * stored_msg) + } else if (safe_str_eq(msg_ref, fsa_pe_ref)) { + ha_msg_input_t fsa_input; + ++ controld_stop_sched_timer(); + fsa_input.msg = stored_msg; + register_fsa_input_later(C_IPC_MESSAGE, I_PE_SUCCESS, &fsa_input); +- crm_trace("Completed: %s...", fsa_pe_ref); + + } else { + crm_info("%s calculation %s is obsolete", op, msg_ref); +diff --git a/crmd/pengine.c b/crmd/pengine.c +index b7d996a..1630e7b 100644 +--- a/crmd/pengine.c ++++ b/crmd/pengine.c +@@ -173,6 +173,45 @@ do_pe_control(long long action, + + int fsa_pe_query = 0; + char *fsa_pe_ref = NULL; ++static mainloop_timer_t *controld_sched_timer = NULL; ++ ++// @TODO Make this a configurable cluster option if there's demand for it ++#define SCHED_TIMEOUT_MS (120000) ++ ++/*! ++ * \internal ++ * \brief Handle a timeout waiting for scheduler reply ++ * ++ * \param[in] user_data Ignored ++ * ++ * \return FALSE (indicating that timer should not be restarted) ++ */ ++static gboolean ++controld_sched_timeout(gpointer user_data) ++{ ++ if (AM_I_DC) { ++ /* If this node is the DC but can't communicate with the scheduler, just ++ * exit (and likely get fenced) so this node doesn't interfere with any ++ * further DC elections. ++ * ++ * @TODO We could try something less drastic first, like disconnecting ++ * and reconnecting to the scheduler, but something is likely going ++ * seriously wrong, so perhaps it's better to just fail as quickly as ++ * possible. ++ */ ++ crmd_exit(DAEMON_RESPAWN_STOP); ++ } ++ return FALSE; ++} ++ ++void ++controld_stop_sched_timer() ++{ ++ if (controld_sched_timer && fsa_pe_ref) { ++ crm_trace("Stopping timer for scheduler reply %s", fsa_pe_ref); ++ } ++ mainloop_timer_stop(controld_sched_timer); ++} + + /*! + * \internal +@@ -188,6 +227,16 @@ controld_expect_sched_reply(xmlNode *msg) + if (msg) { + ref = crm_element_value_copy(msg, XML_ATTR_REFERENCE); + CRM_ASSERT(ref != NULL); ++ ++ if (controld_sched_timer == NULL) { ++ controld_sched_timer = mainloop_timer_add("scheduler_reply_timer", ++ SCHED_TIMEOUT_MS, FALSE, ++ controld_sched_timeout, ++ NULL); ++ } ++ mainloop_timer_start(controld_sched_timer); ++ } else { ++ controld_stop_sched_timer(); + } + free(fsa_pe_ref); + fsa_pe_ref = ref; +@@ -200,6 +249,10 @@ controld_expect_sched_reply(xmlNode *msg) + void + controld_sched_cleanup() + { ++ if (controld_sched_timer != NULL) { ++ mainloop_timer_del(controld_sched_timer); ++ controld_sched_timer = NULL; ++ } + free(pe_subsystem); pe_subsystem = NULL; + controld_expect_sched_reply(NULL); + } +-- +1.8.3.1 + + +From ab4ddb17c9b69f182a71026ee9752eec519fb10c Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 30 May 2019 08:37:52 -0500 +Subject: [PATCH 30/96] Low: libpe_status: offer compile-time option to change + concurrent-fencing default + +We most likely want to make concurrent-fencing default to true at some point. +For now, offer that possibility via a compile-time constant, for experimenting. +--- + lib/pengine/common.c | 8 +++++++- + lib/pengine/status.c | 3 +++ + 2 files changed, 10 insertions(+), 1 deletion(-) + +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index d03a6aa..e82434a 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -111,7 +111,13 @@ pe_cluster_option pe_opts[] = { + "How long to wait for the STONITH action (reboot,on,off) to complete", NULL }, + { XML_ATTR_HAVE_WATCHDOG, NULL, "boolean", NULL, "false", &check_boolean, + "Enable watchdog integration", "Set automatically by the cluster if SBD is detected. User configured values are ignored." }, +- { "concurrent-fencing", NULL, "boolean", NULL, "false", &check_boolean, ++ { "concurrent-fencing", NULL, "boolean", NULL, ++#ifdef DEFAULT_CONCURRENT_FENCING_TRUE ++ "true", ++#else ++ "false", ++#endif ++ &check_boolean, + "Allow performing fencing operations in parallel", NULL }, + { "startup-fencing", "startup_fencing", "boolean", NULL, "true", &check_boolean, + "STONITH unseen nodes", "Advanced Use Only! Not using the default is very unsafe!" }, +diff --git a/lib/pengine/status.c b/lib/pengine/status.c +index 6810c78..fdebaa2 100644 +--- a/lib/pengine/status.c ++++ b/lib/pengine/status.c +@@ -376,6 +376,9 @@ set_working_set_defaults(pe_working_set_t * data_set) + set_bit(data_set->flags, pe_flag_symmetric_cluster); + set_bit(data_set->flags, pe_flag_is_managed_default); + set_bit(data_set->flags, pe_flag_stop_action_orphans); ++#ifdef DEFAULT_CONCURRENT_FENCING_TRUE ++ set_bit(data_set->flags, pe_flag_concurrent_fencing); ++#endif + } + + resource_t * +-- +1.8.3.1 + + +From deac9823e08c51a2683c127527a2b047b516f393 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 6 Jun 2019 14:18:37 -0500 +Subject: [PATCH 31/96] Test: scheduler: explicitly set concurrent-fencing in + relevant regression tests + +... since concurrent-fencing's default is likely to eventually change, +which would otherwise affect the results of these tests +--- + pengine/test10/rec-node-14.xml | 1 + + pengine/test10/remote-connection-unrecoverable.xml | 1 + + pengine/test10/remote-recover-all.xml | 1 + + pengine/test10/remote-recover-no-resources.xml | 1 + + pengine/test10/remote-recover-unknown.xml | 1 + + pengine/test10/stonith-4.xml | 1 + + pengine/test10/suicide-needed-inquorate.xml | 1 + + pengine/test10/ticket-clone-21.xml | 1 + + pengine/test10/ticket-clone-9.xml | 1 + + 9 files changed, 9 insertions(+) + +diff --git a/pengine/test10/rec-node-14.xml b/pengine/test10/rec-node-14.xml +index 456ea80..8582c17 100644 +--- a/pengine/test10/rec-node-14.xml ++++ b/pengine/test10/rec-node-14.xml +@@ -4,6 +4,7 @@ + + + ++ + + + +diff --git a/pengine/test10/remote-connection-unrecoverable.xml b/pengine/test10/remote-connection-unrecoverable.xml +index 4dda833..8096e25 100644 +--- a/pengine/test10/remote-connection-unrecoverable.xml ++++ b/pengine/test10/remote-connection-unrecoverable.xml +@@ -7,6 +7,7 @@ + + + ++ + + + +diff --git a/pengine/test10/remote-recover-all.xml b/pengine/test10/remote-recover-all.xml +index 30d2451..f56e641 100644 +--- a/pengine/test10/remote-recover-all.xml ++++ b/pengine/test10/remote-recover-all.xml +@@ -10,6 +10,7 @@ + + + ++ + + + +diff --git a/pengine/test10/remote-recover-no-resources.xml b/pengine/test10/remote-recover-no-resources.xml +index d2fa0df..36f424b 100644 +--- a/pengine/test10/remote-recover-no-resources.xml ++++ b/pengine/test10/remote-recover-no-resources.xml +@@ -10,6 +10,7 @@ + + + ++ + + + +diff --git a/pengine/test10/remote-recover-unknown.xml b/pengine/test10/remote-recover-unknown.xml +index 3992b03..dd7807c 100644 +--- a/pengine/test10/remote-recover-unknown.xml ++++ b/pengine/test10/remote-recover-unknown.xml +@@ -10,6 +10,7 @@ + + + ++ + + + +diff --git a/pengine/test10/stonith-4.xml b/pengine/test10/stonith-4.xml +index f9a4d44..4f185de 100644 +--- a/pengine/test10/stonith-4.xml ++++ b/pengine/test10/stonith-4.xml +@@ -4,6 +4,7 @@ + + + ++ + + + +diff --git a/pengine/test10/suicide-needed-inquorate.xml b/pengine/test10/suicide-needed-inquorate.xml +index 160af00..6add7fd 100644 +--- a/pengine/test10/suicide-needed-inquorate.xml ++++ b/pengine/test10/suicide-needed-inquorate.xml +@@ -7,6 +7,7 @@ + + + ++ + + + +diff --git a/pengine/test10/ticket-clone-21.xml b/pengine/test10/ticket-clone-21.xml +index c29d89f..4a9fce9 100644 +--- a/pengine/test10/ticket-clone-21.xml ++++ b/pengine/test10/ticket-clone-21.xml +@@ -4,6 +4,7 @@ + + + ++ + + + +diff --git a/pengine/test10/ticket-clone-9.xml b/pengine/test10/ticket-clone-9.xml +index 7b2a62f..c6139d3 100644 +--- a/pengine/test10/ticket-clone-9.xml ++++ b/pengine/test10/ticket-clone-9.xml +@@ -4,6 +4,7 @@ + + + ++ + + + +-- +1.8.3.1 + + +From 7c3bc762a9cede20a0193f64ca1a36f507aeeeb3 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 20 Apr 2018 13:23:10 -0500 +Subject: [PATCH 32/96] Build: libcrmcommon: configure option to specify GnuTLS + cipher priorities + +Default to current behavior, i.e. "NORMAL". Spec file overrides with "@SYSTEM" +on distros that have it. + +Pacemaker does not use option value as-is; it adds "+ANON-DH" for CIB remote +commands and "+DHE-PSK:+PSK" for Pacemaker Remote connections. In the longer +term, we could consider moving to certificate-based connections in both cases, +but that has backward compatibility issues as well as additional administrative +burden. +--- + configure.ac | 9 +++++++++ + lib/common/remote.c | 4 ++-- + pacemaker.spec.in | 4 ++++ + 3 files changed, 15 insertions(+), 2 deletions(-) + +diff --git a/configure.ac b/configure.ac +index ce02777..a7084e2 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -290,6 +290,12 @@ AC_ARG_WITH(cibsecrets, + [ SUPPORT_CIBSECRETS=no ], + ) + ++AC_ARG_WITH(gnutls-priorities, ++ [ --with-gnutls-priorities GnuTLS cipher priorities @<:@NORMAL@:>@ ], ++ [ PCMK_GNUTLS_PRIORITIES="$withval" ], ++ [ PCMK_GNUTLS_PRIORITIES="NORMAL" ], ++) ++ + CSPREFIX="" + AC_ARG_WITH(ais-prefix, + [ --with-ais-prefix=DIR Prefix used when Corosync was installed [$prefix]], +@@ -453,6 +459,9 @@ if test x"${BUG_URL}" = x""; then + fi + AC_SUBST(BUG_URL) + ++AC_DEFINE_UNQUOTED([PCMK_GNUTLS_PRIORITIES], ["$PCMK_GNUTLS_PRIORITIES"], ++ [GnuTLS cipher priorities]) ++ + for j in prefix exec_prefix bindir sbindir libexecdir datadir sysconfdir \ + sharedstatedir localstatedir libdir includedir oldincludedir infodir \ + mandir INITDIR docdir CONFIGDIR +diff --git a/lib/common/remote.c b/lib/common/remote.c +index 12d25fa..1e4f8d8 100644 +--- a/lib/common/remote.c ++++ b/lib/common/remote.c +@@ -244,9 +244,9 @@ pcmk__new_tls_session(int csock, unsigned int conn_type, + # ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT + if (cred_type == GNUTLS_CRD_ANON) { + // http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication +- prio = "NORMAL:+ANON-DH"; ++ prio = PCMK_GNUTLS_PRIORITIES ":+ANON-DH"; + } else { +- prio = "NORMAL:+DHE-PSK:+PSK"; ++ prio = PCMK_GNUTLS_PRIORITIES ":+DHE-PSK:+PSK"; + } + # endif + +diff --git a/pacemaker.spec.in b/pacemaker.spec.in +index 3a26572..fd0e3c8 100644 +--- a/pacemaker.spec.in ++++ b/pacemaker.spec.in +@@ -80,6 +80,9 @@ + } || %{?__transaction_systemd_inhibit:1}%{!?__transaction_systemd_inhibit:0}%{nil \ + } || %(test -f /usr/lib/os-release; test $? -ne 0; echo $?)) + ++%if 0%{?fedora} > 20 || 0%{?rhel} > 7 ++%global gnutls_priorities @SYSTEM ++%endif + + # Definitions for backward compatibility with older RPM versions + +@@ -403,6 +406,7 @@ export LDFLAGS_HARDENED_LIB="%{?_hardening_ldflags}" + --without-heartbeat \ + %{!?with_doc: --with-brand=} \ + %{!?with_hardening: --disable-hardening} \ ++ %{?gnutls_priorities: --with-gnutls-priorities="%{gnutls_priorities}"} \ + --with-initdir=%{_initrddir} \ + --localstatedir=%{_var} \ + --with-version=%{version}-%{release} +-- +1.8.3.1 + + +From 99a83b172544102ec32585514e5808585f2ce31c Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 8 Jul 2019 17:39:12 -0500 +Subject: [PATCH 33/96] Feature: remote: allow run-time configurable TLS + priorities + +This also restores compilability with GnuTLS <2.1.7 (not that anyone is still +using that ...), unintentionally broken in 5bded36 (1.1.20). +--- + lib/common/remote.c | 34 +++++++++++++++++++++++++++------- + mcp/pacemaker.sysconfig | 9 +++++++++ + 2 files changed, 36 insertions(+), 7 deletions(-) + +diff --git a/lib/common/remote.c b/lib/common/remote.c +index 1e4f8d8..ccd0342 100644 +--- a/lib/common/remote.c ++++ b/lib/common/remote.c +@@ -237,17 +237,25 @@ pcmk__new_tls_session(int csock, unsigned int conn_type, + { + int rc = GNUTLS_E_SUCCESS; + # ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT +- const char *prio = NULL; ++ const char *prio_base = NULL; ++ char *prio = NULL; + # endif + gnutls_session_t *session = NULL; + + # ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT +- if (cred_type == GNUTLS_CRD_ANON) { +- // http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication +- prio = PCMK_GNUTLS_PRIORITIES ":+ANON-DH"; +- } else { +- prio = PCMK_GNUTLS_PRIORITIES ":+DHE-PSK:+PSK"; ++ /* Determine list of acceptable ciphers, etc. Pacemaker always adds the ++ * values required for its functionality. ++ * ++ * For an example of anonymous authentication, see: ++ * http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication ++ */ ++ ++ prio_base = getenv("PCMK_tls_priorities"); ++ if (prio_base == NULL) { ++ prio_base = PCMK_GNUTLS_PRIORITIES; + } ++ prio = crm_strdup_printf("%s:%s", prio_base, ++ (cred_type == GNUTLS_CRD_ANON)? "+ANON-DH" : "+DHE-PSK:+PSK"); + # endif + + session = gnutls_malloc(sizeof(gnutls_session_t)); +@@ -285,6 +293,9 @@ pcmk__new_tls_session(int csock, unsigned int conn_type, + if (rc != GNUTLS_E_SUCCESS) { + goto error; + } ++# ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT ++ free(prio); ++# endif + return session; + + error: +@@ -292,7 +303,16 @@ error: + CRM_XS " rc=%d priority='%s'", + (cred_type == GNUTLS_CRD_ANON)? "anonymous" : "PSK", + (conn_type == GNUTLS_SERVER)? "server" : "client", +- gnutls_strerror(rc), rc, prio); ++ gnutls_strerror(rc), rc, ++# ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT ++ prio ++# else ++ "default" ++# endif ++ ); ++# ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT ++ free(prio); ++# endif + if (session != NULL) { + gnutls_free(session); + } +diff --git a/mcp/pacemaker.sysconfig b/mcp/pacemaker.sysconfig +index a983011..0da401e 100644 +--- a/mcp/pacemaker.sysconfig ++++ b/mcp/pacemaker.sysconfig +@@ -101,6 +101,15 @@ + # value must be the same on all nodes. The default is "3121". + # PCMK_remote_port=3121 + ++# Use these GnuTLS cipher priorities for TLS connections. See: ++# ++# https://gnutls.org/manual/html_node/Priority-Strings.html ++# ++# Pacemaker will append ":+ANON-DH" for remote CIB access (when enabled) and ++# ":+DHE-PSK:+PSK" for Pacemaker Remote connections, as they are required for ++# the respective functionality. ++# PCMK_tls_priorities="NORMAL" ++ + # Set bounds on the bit length of the prime number generated for Diffie-Hellman + # parameters needed by TLS connections. The default is not to set any bounds. + # +-- +1.8.3.1 + + +From bb7f4be166e4a8d9e851377aeb3b69a2a6b429a4 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 5 Jul 2019 15:34:30 -0500 +Subject: [PATCH 34/96] Low: controller: reset expected reply when + disconnecting from scheduler + +--- + crmd/control.c | 7 ++++++- + crmd/crmd_utils.h | 2 +- + crmd/pengine.c | 12 ++++++++---- + 3 files changed, 15 insertions(+), 6 deletions(-) + +diff --git a/crmd/control.c b/crmd/control.c +index 73a2b08..4b19114 100644 +--- a/crmd/control.c ++++ b/crmd/control.c +@@ -297,6 +297,10 @@ crmd_exit(int rc) + + if (pe_subsystem && pe_subsystem->client && pe_subsystem->client->ipcs) { + crm_trace("Disconnecting Policy Engine"); ++ ++ // If we aren't connected to the scheduler, we can't expect a reply ++ controld_expect_sched_reply(NULL); ++ + qb_ipcs_disconnect(pe_subsystem->client->ipcs); + } + +@@ -339,6 +343,7 @@ crmd_exit(int rc) + clear_bit(fsa_input_register, R_MEMBERSHIP); + g_list_free(fsa_message_queue); fsa_message_queue = NULL; + ++ free(pe_subsystem); pe_subsystem = NULL; + free(te_subsystem); te_subsystem = NULL; + free(cib_subsystem); cib_subsystem = NULL; + +@@ -374,7 +379,7 @@ crmd_exit(int rc) + crm_timer_stop(recheck_timer); + + te_cleanup_stonith_history_sync(NULL, TRUE); +- controld_sched_cleanup(); ++ controld_free_sched_timer(); + + free(transition_timer); transition_timer = NULL; + free(integration_timer); integration_timer = NULL; +diff --git a/crmd/crmd_utils.h b/crmd/crmd_utils.h +index a704380..955d859 100644 +--- a/crmd/crmd_utils.h ++++ b/crmd/crmd_utils.h +@@ -84,8 +84,8 @@ int crmd_fast_exit(int rc); + gboolean stop_subsystem(struct crm_subsystem_s *centry, gboolean force_quit); + gboolean start_subsystem(struct crm_subsystem_s *centry); + void controld_stop_sched_timer(void); ++void controld_free_sched_timer(void); + void controld_expect_sched_reply(xmlNode *msg); +-void controld_sched_cleanup(void); + + void fsa_dump_actions(long long action, const char *text); + void fsa_dump_inputs(int log_level, const char *text, long long input_register); +diff --git a/crmd/pengine.c b/crmd/pengine.c +index 1630e7b..3512952 100644 +--- a/crmd/pengine.c ++++ b/crmd/pengine.c +@@ -97,6 +97,9 @@ pe_ipc_destroy(gpointer user_data) + crm_info("Connection to the Policy Engine released"); + } + ++ // If we aren't connected to the scheduler, we can't expect a reply ++ controld_expect_sched_reply(NULL); ++ + clear_bit(fsa_input_register, pe_subsystem->flag_connected); + pe_subsystem->pid = -1; + pe_subsystem->source = NULL; +@@ -137,6 +140,9 @@ do_pe_control(long long action, + }; + + if (action & stop_actions) { ++ // If we aren't connected to the scheduler, we can't expect a reply ++ controld_expect_sched_reply(NULL); ++ + clear_bit(fsa_input_register, pe_subsystem->flag_required); + + mainloop_del_ipc_client(pe_subsystem->source); +@@ -244,17 +250,15 @@ controld_expect_sched_reply(xmlNode *msg) + + /*! + * \internal +- * \brief Clean up all memory used by controller scheduler handling ++ * \brief Free the scheduler reply timer + */ + void +-controld_sched_cleanup() ++controld_free_sched_timer() + { + if (controld_sched_timer != NULL) { + mainloop_timer_del(controld_sched_timer); + controld_sched_timer = NULL; + } +- free(pe_subsystem); pe_subsystem = NULL; +- controld_expect_sched_reply(NULL); + } + + /* A_PE_INVOKE */ +-- +1.8.3.1 + + +From e2981df8681d7721d576eacd443fa3cc08c17a02 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 11 Jul 2019 13:49:11 -0500 +Subject: [PATCH 35/96] Test: CTS: update pattern for 1.1 backport + +--- + cts/patterns.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/cts/patterns.py b/cts/patterns.py +index 87b44a9..00c26ff 100644 +--- a/cts/patterns.py ++++ b/cts/patterns.py +@@ -376,7 +376,7 @@ class crm_cs_v0(BasePatterns): + "LRMD lost STONITH connection", + "Connection to stonith-ng.* closed", + "Fencing daemon connection failed", +- r"pacemaker-controld.*Fencer successfully connected", ++ r"crmd:.*Fencer successfully connected", + ] + self.components["stonith-ignore"] = [ + r"pengine.*: Recover Fencing", +-- +1.8.3.1 + + +From 7903d2a0ad53f4906248d26e13d5aaf7c4c824e6 Mon Sep 17 00:00:00 2001 +From: "Gao,Yan" +Date: Mon, 8 Apr 2019 15:01:20 +0200 +Subject: [PATCH 36/96] Fix: scheduler: wait for probe actions to complete to + prevent unnecessary restart/re-promote of dependent resources + +This addresses the issue brought up from: +https://github.com/ClusterLabs/pacemaker/commit/faf44d811e4f5598dae085c61fdef410c8d18882#commitcomment-22262090 + +Given an ordering chain in a transition graph like: + +A.probe -> A.start -> [...] -> B.start + +, if B was already started, it would be scheduled to restart. + +Previously, B would be directly stopped, which could turn out to be +unnecessary if A was probed being already started as well. Such +unnecessary restart could be very expensive for heavy workload. + +With this commit, a new order will be created: + +A.probe -> B.stop + +So that any potential restart of B will wait for A.probe to complete. In +case that A is already started, transition will abort and restart of B +won't need to be performed any more. + +Similarly for an ordering chain like: + +A.probe -> A.start -> [...] -> B.promote + +A new order will be created to prevent unnecessary re-promote: +A.probe -> B.demote +--- + pengine/allocate.c | 204 ++++++++++++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 202 insertions(+), 2 deletions(-) + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index c7c68f8..d600bbf 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -2294,9 +2294,8 @@ order_first_probe_unneeded(pe_action_t * probe, pe_action_t * rh_action) + return FALSE; + } + +- + static void +-order_first_probes(pe_working_set_t * data_set) ++order_first_probes_imply_stops(pe_working_set_t * data_set) + { + GListPtr gIter = NULL; + +@@ -2424,6 +2423,207 @@ order_first_probes(pe_working_set_t * data_set) + } + + static void ++order_first_probe_then_restart_repromote(pe_action_t * probe, ++ pe_action_t * after, ++ pe_working_set_t * data_set) ++{ ++ GListPtr gIter = NULL; ++ bool interleave = FALSE; ++ pe_resource_t *compatible_rsc = NULL; ++ ++ if (probe == NULL ++ || probe->rsc == NULL ++ || probe->rsc->variant != pe_native) { ++ return; ++ } ++ ++ if (after == NULL ++ // Avoid running into any possible loop ++ || is_set(after->flags, pe_action_tracking)) { ++ return; ++ } ++ ++ if (safe_str_neq(probe->task, RSC_STATUS)) { ++ return; ++ } ++ ++ pe_set_action_bit(after, pe_action_tracking); ++ ++ crm_trace("Processing based on %s %s -> %s %s", ++ probe->uuid, ++ probe->node ? probe->node->details->uname: "", ++ after->uuid, ++ after->node ? after->node->details->uname : ""); ++ ++ if (after->rsc ++ /* Better not build a dependency directly with a clone/group. ++ * We are going to proceed through the ordering chain and build ++ * dependencies with its children. ++ */ ++ && after->rsc->variant == pe_native ++ && probe->rsc != after->rsc) { ++ ++ GListPtr then_actions = NULL; ++ enum pe_ordering probe_order_type = pe_order_optional; ++ ++ if (safe_str_eq(after->task, RSC_START)) { ++ char *key = generate_op_key(after->rsc->id, RSC_STOP, 0); ++ ++ then_actions = find_actions(after->rsc->actions, key, NULL); ++ free(key); ++ ++ } else if (safe_str_eq(after->task, RSC_PROMOTE)) { ++ char *key = generate_op_key(after->rsc->id, RSC_DEMOTE, 0); ++ ++ then_actions = find_actions(after->rsc->actions, key, NULL); ++ free(key); ++ } ++ ++ for (gIter = then_actions; gIter != NULL; gIter = gIter->next) { ++ pe_action_t *then = (pe_action_t *) gIter->data; ++ ++ // Skip any pseudo action which for example is implied by fencing ++ if (is_set(then->flags, pe_action_pseudo)) { ++ continue; ++ } ++ ++ order_actions(probe, then, probe_order_type); ++ } ++ g_list_free(then_actions); ++ } ++ ++ if (after->rsc ++ && after->rsc->variant > pe_group) { ++ const char *interleave_s = g_hash_table_lookup(after->rsc->meta, ++ XML_RSC_ATTR_INTERLEAVE); ++ ++ interleave = crm_is_true(interleave_s); ++ ++ if (interleave) { ++ /* For an interleaved clone, we should build a dependency only ++ * with the relevant clone child. ++ */ ++ compatible_rsc = find_compatible_child(probe->rsc, ++ after->rsc, ++ RSC_ROLE_UNKNOWN, ++ FALSE); ++ } ++ } ++ ++ for (gIter = after->actions_after; gIter != NULL; gIter = gIter->next) { ++ action_wrapper_t *after_wrapper = (action_wrapper_t *) gIter->data; ++ /* pe_order_implies_then is the reason why a required A.start ++ * implies/enforces B.start to be required too, which is the cause of ++ * B.restart/re-promote. ++ * ++ * Not sure about pe_order_implies_then_on_node though. It's now only ++ * used for unfencing case, which tends to introduce transition ++ * loops... ++ */ ++ ++ if (is_not_set(after_wrapper->type, pe_order_implies_then)) { ++ /* The order type between a group/clone and its child such as ++ * B.start-> B_child.start is: ++ * pe_order_implies_first_printed | pe_order_runnable_left ++ * ++ * Proceed through the ordering chain and build dependencies with ++ * its children. ++ */ ++ if (after->rsc == NULL ++ || after->rsc->variant < pe_group ++ || probe->rsc->parent == after->rsc ++ || after_wrapper->action->rsc == NULL ++ || after_wrapper->action->rsc->variant > pe_group ++ || after->rsc != after_wrapper->action->rsc->parent) { ++ continue; ++ } ++ ++ /* Proceed to the children of a group or a non-interleaved clone. ++ * For an interleaved clone, proceed only to the relevant child. ++ */ ++ if (after->rsc->variant > pe_group ++ && interleave == TRUE ++ && (compatible_rsc == NULL ++ || compatible_rsc != after_wrapper->action->rsc)) { ++ continue; ++ } ++ } ++ ++ crm_trace("Proceeding through %s %s -> %s %s (type=0x%.6x)", ++ after->uuid, ++ after->node ? after->node->details->uname: "", ++ after_wrapper->action->uuid, ++ after_wrapper->action->node ? after_wrapper->action->node->details->uname : "", ++ after_wrapper->type); ++ ++ order_first_probe_then_restart_repromote(probe, after_wrapper->action, data_set); ++ } ++} ++ ++static void clear_actions_tracking_flag(pe_working_set_t * data_set) ++{ ++ GListPtr gIter = NULL; ++ ++ for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { ++ pe_action_t *action = (pe_action_t *) gIter->data; ++ ++ if (is_set(action->flags, pe_action_tracking)) { ++ pe_clear_action_bit(action, pe_action_tracking); ++ } ++ } ++} ++ ++static void ++order_first_rsc_probes(pe_resource_t * rsc, pe_working_set_t * data_set) ++{ ++ GListPtr gIter = NULL; ++ GListPtr probes = NULL; ++ char *key = NULL; ++ ++ for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { ++ pe_resource_t * child = (pe_resource_t *) gIter->data; ++ ++ order_first_rsc_probes(child, data_set); ++ } ++ ++ if (rsc->variant != pe_native) { ++ return; ++ } ++ ++ key = generate_op_key(rsc->id, RSC_STATUS, 0); ++ probes = find_actions(rsc->actions, key, NULL); ++ free(key); ++ ++ for (gIter = probes; gIter != NULL; gIter= gIter->next) { ++ pe_action_t *probe = (pe_action_t *) gIter->data; ++ GListPtr aIter = NULL; ++ ++ for (aIter = probe->actions_after; aIter != NULL; aIter = aIter->next) { ++ action_wrapper_t *after_wrapper = (action_wrapper_t *) aIter->data; ++ ++ order_first_probe_then_restart_repromote(probe, after_wrapper->action, data_set); ++ clear_actions_tracking_flag(data_set); ++ } ++ } ++ ++ g_list_free(probes); ++} ++ ++static void ++order_first_probes(pe_working_set_t * data_set) ++{ ++ GListPtr gIter = NULL; ++ ++ for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) { ++ pe_resource_t *rsc = (pe_resource_t *) gIter->data; ++ ++ order_first_rsc_probes(rsc, data_set); ++ } ++ ++ order_first_probes_imply_stops(data_set); ++} ++ ++static void + order_then_probes(pe_working_set_t * data_set) + { + #if 0 +-- +1.8.3.1 + + +From 3228b6b4df624db4b72d40bd366433f673a8780e Mon Sep 17 00:00:00 2001 +From: "Gao,Yan" +Date: Tue, 30 Apr 2019 03:47:07 +0200 +Subject: [PATCH 37/96] Test: scheduler: wait for probe actions to complete to + prevent unnecessary restart/re-promote of dependent resources (update tests) + +--- + .../test10/11-a-then-bm-b-move-a-clone-starting.dot | 1 + + .../test10/11-a-then-bm-b-move-a-clone-starting.exp | 6 +++++- + pengine/test10/bug-n-387749.dot | 7 +++++++ + pengine/test10/bug-n-387749.exp | 21 +++++++++++++++++++++ + pengine/test10/group5.dot | 3 +++ + pengine/test10/group5.exp | 9 +++++++++ + pengine/test10/group6.dot | 6 ++++++ + pengine/test10/group6.exp | 18 ++++++++++++++++++ + pengine/test10/group9.dot | 6 ++++++ + pengine/test10/group9.exp | 18 ++++++++++++++++++ + pengine/test10/order6.dot | 2 ++ + pengine/test10/order6.exp | 12 ++++++++++-- + pengine/test10/reload-becomes-restart.dot | 1 + + pengine/test10/reload-becomes-restart.exp | 3 +++ + 14 files changed, 110 insertions(+), 3 deletions(-) + +diff --git a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.dot b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.dot +index 4a89db6..4fd6a7d 100644 +--- a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.dot ++++ b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.dot +@@ -12,6 +12,7 @@ + "myclone-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] + "myclone_monitor_0 f20node2" -> "myclone-clone_start_0" [ style = bold] + "myclone_monitor_0 f20node2" -> "myclone-clone_stopped_0" [ style = bold] ++"myclone_monitor_0 f20node2" -> "vm_stop_0 f20node1" [ style = bold] + "myclone_monitor_0 f20node2" [ style=bold color="green" fontcolor="black"] + "myclone_start_0 f20node2" -> "myclone-clone_running_0" [ style = bold] + "myclone_start_0 f20node2" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp +index 4eeb086..d3ce8b7 100644 +--- a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp ++++ b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp +@@ -120,6 +120,10 @@ + + + +- ++ ++ ++ ++ ++ + + +diff --git a/pengine/test10/bug-n-387749.dot b/pengine/test10/bug-n-387749.dot +index 5095351..a820108 100644 +--- a/pengine/test10/bug-n-387749.dot ++++ b/pengine/test10/bug-n-387749.dot +@@ -1,17 +1,23 @@ + digraph "g" { + "export_home_ocfs2:0_monitor_0 power720-1" -> "export_home_ocfs2_clone_set_start_0" [ style = bold] ++"export_home_ocfs2:0_monitor_0 power720-1" -> "resource_ipaddr1_single_stop_0 power720-2" [ style = bold] ++"export_home_ocfs2:0_monitor_0 power720-1" -> "resource_nfsserver_single_stop_0 power720-2" [ style = bold] + "export_home_ocfs2:0_monitor_0 power720-1" [ style=bold color="green" fontcolor="black" ] + "export_home_ocfs2:0_post_notify_start_0 power720-1" -> "export_home_ocfs2_clone_set_confirmed-post_notify_running_0" [ style = bold] + "export_home_ocfs2:0_post_notify_start_0 power720-1" [ style=bold color="green" fontcolor="black" ] + "export_home_ocfs2:0_start_0 power720-1" -> "export_home_ocfs2_clone_set_running_0" [ style = bold] + "export_home_ocfs2:0_start_0 power720-1" [ style=bold color="green" fontcolor="black" ] + "export_home_ocfs2:1_monitor_0 power720-1" -> "export_home_ocfs2_clone_set_start_0" [ style = bold] ++"export_home_ocfs2:1_monitor_0 power720-1" -> "resource_ipaddr1_single_stop_0 power720-2" [ style = bold] ++"export_home_ocfs2:1_monitor_0 power720-1" -> "resource_nfsserver_single_stop_0 power720-2" [ style = bold] + "export_home_ocfs2:1_monitor_0 power720-1" [ style=bold color="green" fontcolor="black" ] + "export_home_ocfs2:1_post_notify_start_0 power720-2" -> "export_home_ocfs2_clone_set_confirmed-post_notify_running_0" [ style = bold] + "export_home_ocfs2:1_post_notify_start_0 power720-2" [ style=bold color="green" fontcolor="black" ] + "export_home_ocfs2:1_pre_notify_start_0 power720-2" -> "export_home_ocfs2_clone_set_confirmed-pre_notify_start_0" [ style = bold] + "export_home_ocfs2:1_pre_notify_start_0 power720-2" [ style=bold color="green" fontcolor="black" ] + "export_home_ocfs2:2_monitor_0 power720-1" -> "export_home_ocfs2_clone_set_start_0" [ style = bold] ++"export_home_ocfs2:2_monitor_0 power720-1" -> "resource_ipaddr1_single_stop_0 power720-2" [ style = bold] ++"export_home_ocfs2:2_monitor_0 power720-1" -> "resource_nfsserver_single_stop_0 power720-2" [ style = bold] + "export_home_ocfs2:2_monitor_0 power720-1" [ style=bold color="green" fontcolor="black" ] + "export_home_ocfs2_clone_set_confirmed-post_notify_running_0" -> "group_nfs_start_0" [ style = bold] + "export_home_ocfs2_clone_set_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange" ] +@@ -43,6 +49,7 @@ digraph "g" { + "group_nfs_stopped_0" [ style=bold color="green" fontcolor="orange" ] + "resource_ipaddr1_single_monitor_0 power720-1" -> "group_nfs_stopped_0" [ style = bold] + "resource_ipaddr1_single_monitor_0 power720-1" -> "resource_ipaddr1_single_start_0 power720-1" [ style = bold] ++"resource_ipaddr1_single_monitor_0 power720-1" -> "resource_nfsserver_single_stop_0 power720-2" [ style = bold] + "resource_ipaddr1_single_monitor_0 power720-1" [ style=bold color="green" fontcolor="black" ] + "resource_ipaddr1_single_monitor_5000 power720-1" [ style=bold color="green" fontcolor="black" ] + "resource_ipaddr1_single_start_0 power720-1" -> "group_nfs_running_0" [ style = bold] +diff --git a/pengine/test10/bug-n-387749.exp b/pengine/test10/bug-n-387749.exp +index d6fe8e4..5b646bc 100644 +--- a/pengine/test10/bug-n-387749.exp ++++ b/pengine/test10/bug-n-387749.exp +@@ -276,6 +276,15 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -339,6 +348,18 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/group5.dot b/pengine/test10/group5.dot +index 3fe0193..4776b1e 100644 +--- a/pengine/test10/group5.dot ++++ b/pengine/test10/group5.dot +@@ -1,5 +1,7 @@ + digraph "g" { + "child_rsc1_monitor_0 node2" -> "child_rsc1_start_0 node2" [ style = bold] ++"child_rsc1_monitor_0 node2" -> "child_rsc2_stop_0 node1" [ style = bold] ++"child_rsc1_monitor_0 node2" -> "child_rsc3_stop_0 node1" [ style = bold] + "child_rsc1_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] + "child_rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "child_rsc1_start_0 node2" -> "child_rsc2_start_0 node2" [ style = bold] +@@ -10,6 +12,7 @@ + "child_rsc1_stop_0 node1" [ style=bold color="green" fontcolor="black" ] + "child_rsc2_monitor_0 node2" -> "child_rsc1_stop_0 node1" [ style = bold] + "child_rsc2_monitor_0 node2" -> "child_rsc2_start_0 node2" [ style = bold] ++"child_rsc2_monitor_0 node2" -> "child_rsc3_stop_0 node1" [ style = bold] + "child_rsc2_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] + "child_rsc2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "child_rsc2_start_0 node2" -> "child_rsc3_start_0 node2" [ style = bold] +diff --git a/pengine/test10/group5.exp b/pengine/test10/group5.exp +index 4ea2b08..0f55341 100644 +--- a/pengine/test10/group5.exp ++++ b/pengine/test10/group5.exp +@@ -196,6 +196,9 @@ + + + ++ ++ ++ + + + +@@ -246,6 +249,12 @@ + + + ++ ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/group6.dot b/pengine/test10/group6.dot +index a563e05..536f56b 100644 +--- a/pengine/test10/group6.dot ++++ b/pengine/test10/group6.dot +@@ -1,5 +1,7 @@ + digraph "g" { + "child_rsc1_monitor_0 node2" -> "child_rsc1_start_0 node2" [ style = bold] ++"child_rsc1_monitor_0 node2" -> "child_rsc2_stop_0 node1" [ style = bold] ++"child_rsc1_monitor_0 node2" -> "child_rsc3_stop_0 node1" [ style = bold] + "child_rsc1_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] + "child_rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "child_rsc1_start_0 node2" -> "child_rsc2_start_0 node2" [ style = bold] +@@ -10,6 +12,7 @@ + "child_rsc1_stop_0 node1" [ style=bold color="green" fontcolor="black" ] + "child_rsc2_monitor_0 node2" -> "child_rsc1_stop_0 node1" [ style = bold] + "child_rsc2_monitor_0 node2" -> "child_rsc2_start_0 node2" [ style = bold] ++"child_rsc2_monitor_0 node2" -> "child_rsc3_stop_0 node1" [ style = bold] + "child_rsc2_monitor_0 node2" -> "rsc1_stopped_0" [ style = bold] + "child_rsc2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "child_rsc2_start_0 node2" -> "child_rsc3_start_0 node2" [ style = bold] +@@ -30,6 +33,8 @@ + "child_rsc3_stop_0 node1" -> "rsc1_stopped_0" [ style = bold] + "child_rsc3_stop_0 node1" [ style=bold color="green" fontcolor="black" ] + "child_rsc4_monitor_0 node2" -> "child_rsc4_start_0 node2" [ style = bold] ++"child_rsc4_monitor_0 node2" -> "child_rsc5_stop_0 node1" [ style = bold] ++"child_rsc4_monitor_0 node2" -> "child_rsc6_stop_0 node1" [ style = bold] + "child_rsc4_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] + "child_rsc4_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "child_rsc4_start_0 node2" -> "child_rsc5_start_0 node2" [ style = bold] +@@ -40,6 +45,7 @@ + "child_rsc4_stop_0 node1" [ style=bold color="green" fontcolor="black" ] + "child_rsc5_monitor_0 node2" -> "child_rsc4_stop_0 node1" [ style = bold] + "child_rsc5_monitor_0 node2" -> "child_rsc5_start_0 node2" [ style = bold] ++"child_rsc5_monitor_0 node2" -> "child_rsc6_stop_0 node1" [ style = bold] + "child_rsc5_monitor_0 node2" -> "rsc2_stopped_0" [ style = bold] + "child_rsc5_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "child_rsc5_start_0 node2" -> "child_rsc6_start_0 node2" [ style = bold] +diff --git a/pengine/test10/group6.exp b/pengine/test10/group6.exp +index cddd6f4..097d23d 100644 +--- a/pengine/test10/group6.exp ++++ b/pengine/test10/group6.exp +@@ -152,6 +152,9 @@ + + + ++ ++ ++ + + + +@@ -202,6 +205,12 @@ + + + ++ ++ ++ ++ ++ ++ + + + +@@ -367,6 +376,9 @@ + + + ++ ++ ++ + + + +@@ -417,6 +429,12 @@ + + + ++ ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/group9.dot b/pengine/test10/group9.dot +index 610fe93..5a93a31 100644 +--- a/pengine/test10/group9.dot ++++ b/pengine/test10/group9.dot +@@ -26,9 +26,12 @@ + "rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "rsc2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "rsc3_monitor_0 node2" -> "foo_stopped_0" [ style = bold] ++"rsc3_monitor_0 node2" -> "rsc4_stop_0 node1" [ style = bold] ++"rsc3_monitor_0 node2" -> "rsc5_stop_0 node1" [ style = bold] + "rsc3_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "rsc4_monitor_0 node2" -> "foo_stopped_0" [ style = bold] + "rsc4_monitor_0 node2" -> "rsc4_start_0 node1" [ style = bold] ++"rsc4_monitor_0 node2" -> "rsc5_stop_0 node1" [ style = bold] + "rsc4_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "rsc4_start_0 node1" -> "foo_running_0" [ style = bold] + "rsc4_start_0 node1" -> "rsc5_start_0 node1" [ style = bold] +@@ -48,6 +51,8 @@ + "rsc5_stop_0 node1" [ style=bold color="green" fontcolor="black" ] + "rsc6_monitor_0 node2" -> "bar_stopped_0" [ style = bold] + "rsc6_monitor_0 node2" -> "rsc6_start_0 node2" [ style = bold] ++"rsc6_monitor_0 node2" -> "rsc7_stop_0 node1" [ style = bold] ++"rsc6_monitor_0 node2" -> "rsc8_stop_0 node1" [ style = bold] + "rsc6_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "rsc6_start_0 node2" -> "bar_running_0" [ style = bold] + "rsc6_start_0 node2" -> "rsc7_start_0 node2" [ style = bold] +@@ -58,6 +63,7 @@ + "rsc7_monitor_0 node2" -> "bar_stopped_0" [ style = bold] + "rsc7_monitor_0 node2" -> "rsc6_stop_0 node1" [ style = bold] + "rsc7_monitor_0 node2" -> "rsc7_start_0 node2" [ style = bold] ++"rsc7_monitor_0 node2" -> "rsc8_stop_0 node1" [ style = bold] + "rsc7_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "rsc7_start_0 node2" -> "bar_running_0" [ style = bold] + "rsc7_start_0 node2" -> "rsc8_start_0 node2" [ style = bold] +diff --git a/pengine/test10/group9.exp b/pengine/test10/group9.exp +index f05c2c2..ac82825 100644 +--- a/pengine/test10/group9.exp ++++ b/pengine/test10/group9.exp +@@ -128,6 +128,9 @@ + + + ++ ++ ++ + + + +@@ -169,6 +172,12 @@ + + + ++ ++ ++ ++ ++ ++ + + + +@@ -340,6 +349,9 @@ + + + ++ ++ ++ + + + +@@ -381,6 +393,12 @@ + + + ++ ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/order6.dot b/pengine/test10/order6.dot +index 74f1c5b..0dfd73f 100644 +--- a/pengine/test10/order6.dot ++++ b/pengine/test10/order6.dot +@@ -1,4 +1,5 @@ + digraph "g" { ++"rsc1_monitor_0 node2" -> "rsc2_stop_0 node1" [ style = bold] + "rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "rsc2_monitor_0 node2" -> "rsc2_start_0 node2" [ style = bold] + "rsc2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] +@@ -11,6 +12,7 @@ + "rsc4_start_0 node2" [ style=bold color="green" fontcolor="black" ] + "rsc4_stop_0 node1" -> "rsc4_start_0 node2" [ style = bold] + "rsc4_stop_0 node1" [ style=bold color="green" fontcolor="black" ] ++"rsc5_monitor_0 node1" -> "rsc6_stop_0 node2" [ style = bold] + "rsc5_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] + "rsc6_monitor_0 node1" -> "rsc6_start_0 node1" [ style = bold] + "rsc6_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] +diff --git a/pengine/test10/order6.exp b/pengine/test10/order6.exp +index 47dc227..c3d74e6 100644 +--- a/pengine/test10/order6.exp ++++ b/pengine/test10/order6.exp +@@ -31,7 +31,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +@@ -117,7 +121,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/reload-becomes-restart.dot b/pengine/test10/reload-becomes-restart.dot +index a6616f9..36f8372 100644 +--- a/pengine/test10/reload-becomes-restart.dot ++++ b/pengine/test10/reload-becomes-restart.dot +@@ -30,6 +30,7 @@ digraph "g" { + "rsc1:0_start_0 node2" -> "rsc2:1_start_0 node2" [ style = bold] + "rsc1:0_start_0 node2" [ style=bold color="green" fontcolor="black"] + "rsc1:1_monitor_0 node1" -> "cl-rsc1_start_0" [ style = bold] ++"rsc1:1_monitor_0 node1" -> "rsc2_stop_0 node1" [ style = bold] + "rsc1:1_monitor_0 node1" [ style=bold color="green" fontcolor="black"] + "rsc1:1_monitor_120000 node1" [ style=bold color="green" fontcolor="black"] + "rsc1:1_start_0 node1" -> "cl-rsc1_running_0" [ style = bold] +diff --git a/pengine/test10/reload-becomes-restart.exp b/pengine/test10/reload-becomes-restart.exp +index c3e3721..224b8d2 100644 +--- a/pengine/test10/reload-becomes-restart.exp ++++ b/pengine/test10/reload-becomes-restart.exp +@@ -177,6 +177,9 @@ + + + ++ ++ ++ + + + +-- +1.8.3.1 + + +From 95791c21987c82dd5c821569a7f029e876f9bed1 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 12 Jul 2019 11:08:23 -0500 +Subject: [PATCH 38/96] Test: CTS: update patterns for stonith probe change + +Since 343ae2a, ComponentFail tests that result in the fencer respawning (i.e. +when the target is the fencer or the CIB) will return additional (expected) +errors due to stonith probes (correctly) being considered failed. +--- + cts/CTStests.py | 8 ++++++-- + cts/patterns.py | 26 +++++++++++++++++++++++--- + 2 files changed, 29 insertions(+), 5 deletions(-) + +diff --git a/cts/CTStests.py b/cts/CTStests.py +index b6cb31d..e30fe86 100644 +--- a/cts/CTStests.py ++++ b/cts/CTStests.py +@@ -1422,14 +1422,18 @@ class ComponentFail(CTSTest): + self.okerrpatterns.append(self.templates["Pat:ChildRespawn"] %(node, chosen.name)) + self.okerrpatterns.append(self.templates["Pat:ChildExit"]) + +- if chosen.name == "stonith": +- # Ignore actions for STONITH resources ++ # @TODO this should be a flag in the Component ++ if chosen.name in [ "corosync", "cib", "stonith" ]: ++ # Ignore actions for fence devices if fencer will respawn ++ # (their registration will be lost, and probes will fail) + (rc, lines) = self.rsh(node, "crm_resource -c", None) + for line in lines: + if re.search("^Resource", line): + r = AuditResource(self.CM, line) + if r.rclass == "stonith": + self.okerrpatterns.append(self.templates["Pat:Fencing_recover"] % r.id) ++ self.okerrpatterns.append(self.templates["Pat:Fencing_active"] % r.id) ++ self.okerrpatterns.append(self.templates["Pat:Fencing_probe"] % r.id) + + # supply a copy so self.patterns doesn't end up empty + tmpPats = [] +diff --git a/cts/patterns.py b/cts/patterns.py +index 00c26ff..0f9982e 100644 +--- a/cts/patterns.py ++++ b/cts/patterns.py +@@ -59,9 +59,11 @@ class BasePatterns: + "Pat:They_dead" : "node %s.*: is dead", + "Pat:TransitionComplete" : "Transition status: Complete: complete", + +- "Pat:Fencing_start" : "(Initiating remote operation|Requesting peer fencing ).* (for|of) %s", +- "Pat:Fencing_ok" : r"stonith.*:\s*Operation .* of %s by .* for .*@.*: OK", +- "Pat:Fencing_recover" : r"pengine.*: Recover %s", ++ "Pat:Fencing_start" : r"(Initiating remote operation|Requesting peer fencing ).* (for|of) %s", ++ "Pat:Fencing_ok" : r"stonith.*:\s*Operation .* of %s by .* for .*@.*: OK", ++ "Pat:Fencing_recover" : r"pengine.*: Recover %s", ++ "Pat:Fencing_active" : r"pengine.*: Resource %s is active on .* nodes", ++ "Pat:Fencing_probe" : r"crmd.*: Result of probe operation for %s on .*: Error", + + "Pat:RscOpOK" : r"crmd.*:\s+Result of %s operation for %s.*: (0 \()?ok", + "Pat:RscRemoteOpOK" : r"crmd.*:\s+Result of %s operation for %s on %s: (0 \()?ok", +@@ -299,6 +301,12 @@ class crm_cs_v0(BasePatterns): + r"error:.*STONITH connection failed", + r"error: Connection to stonith-ng.* (failed|closed)", + r"crit: Fencing daemon connection failed", ++ # This is overbroad, but we don't have a way to say that only ++ # certain transition errors are acceptable (if the fencer respawns, ++ # fence devices may appear multiply active). We have to rely on ++ # other causes of a transition error logging their own error ++ # message, which is the usual practice. ++ r"pengine.* Calculated transition .*/pe-error", + ] + + self.components["corosync"] = [ +@@ -316,6 +324,12 @@ class crm_cs_v0(BasePatterns): + "lrmd.*Connection to stonith-ng.* closed", + "lrmd.*LRMD lost STONITH connection", + "lrmd.*STONITH connection failed, finalizing .* pending operations", ++ # This is overbroad, but we don't have a way to say that only ++ # certain transition errors are acceptable (if the fencer respawns, ++ # fence devices may appear multiply active). We have to rely on ++ # other causes of a transition error logging their own error ++ # message, which is the usual practice. ++ r"pengine.* Calculated transition .*/pe-error", + ] + + self.components["cib"] = [ +@@ -387,6 +401,12 @@ class crm_cs_v0(BasePatterns): + r"error:.*Sign-in failed: triggered a retry", + "STONITH connection failed, finalizing .* pending operations.", + r"crmd.*:\s+Result of .* operation for Fencing.*Error", ++ # This is overbroad, but we don't have a way to say that only ++ # certain transition errors are acceptable (if the fencer respawns, ++ # fence devices may appear multiply active). We have to rely on ++ # other causes of a transition error logging their own error ++ # message, which is the usual practice. ++ r"pengine.* Calculated transition .*/pe-error", + ] + self.components["stonith-ignore"].extend(self.components["common-ignore"]) + +-- +1.8.3.1 + + +From 49593689fbe40f02376f8d540c4cbd554f9e8e9f Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 16 Jul 2019 14:03:52 -0500 +Subject: [PATCH 39/96] Test: CTS: update execd ComponentFail ignore patterns + +attrd connects to execd for sending alerts, so connection failures for it are +expected when killing execd +--- + cts/patterns.py | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/cts/patterns.py b/cts/patterns.py +index 0f9982e..1e9f0e5 100644 +--- a/cts/patterns.py ++++ b/cts/patterns.py +@@ -358,7 +358,9 @@ class crm_cs_v0(BasePatterns): + r"crmd.*: Input I_TERMINATE .*from do_recover", + "crmd.*Could not recover from internal error", + ] +- self.components["lrmd-ignore"] = [] ++ self.components["lrmd-ignore"] = [ ++ r"attrd.*Connection to lrmd (failed|closed)", ++ ] + + self.components["crmd"] = [ + # "WARN: determine_online_status: Node .* is unclean", +-- +1.8.3.1 + + +From e9721446830e98f32be3082138c0a27ccbf6a452 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 23 Jul 2019 10:33:12 -0500 +Subject: [PATCH 40/96] Test: CTS: correct fencer connection pattern + +e2981df had a typo +--- + cts/patterns.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/cts/patterns.py b/cts/patterns.py +index 1e9f0e5..cf1860a 100644 +--- a/cts/patterns.py ++++ b/cts/patterns.py +@@ -392,7 +392,7 @@ class crm_cs_v0(BasePatterns): + "LRMD lost STONITH connection", + "Connection to stonith-ng.* closed", + "Fencing daemon connection failed", +- r"crmd:.*Fencer successfully connected", ++ r"crmd.*: Fencer successfully connected", + ] + self.components["stonith-ignore"] = [ + r"pengine.*: Recover Fencing", +-- +1.8.3.1 + + +From 9fe68fdb8c1355e3436934eb5812af696de39dad Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 19 Jul 2019 18:49:21 -0500 +Subject: [PATCH 41/96] Fix: controller: panic local host if notified of own + fencing + +Previously, we attempted to reboot, but that would always fail because the +controller doesn't run as root, so it would fall back to exiting CRM_EX_FATAL. + +Now, we exit CRM_EX_PANIC, to tell pacemakerd to panic the local host, which +is a better method of self-fencing. + +clbz#5386 +--- + crmd/te_utils.c | 41 +++++++++++------------------------------ + 1 file changed, 11 insertions(+), 30 deletions(-) + +diff --git a/crmd/te_utils.c b/crmd/te_utils.c +index 14570cd..6c7f9a0 100644 +--- a/crmd/te_utils.c ++++ b/crmd/te_utils.c +@@ -235,11 +235,6 @@ tengine_stonith_connection_destroy(stonith_t * st, stonith_event_t * e) + + char *te_client_id = NULL; + +-#ifdef HAVE_SYS_REBOOT_H +-# include +-# include +-#endif +- + static void + tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event); + +@@ -271,33 +266,19 @@ tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event) + return; + + } else if (st_event->result == pcmk_ok && crm_str_eq(st_event->target, fsa_our_uname, TRUE)) { +- crm_crit("We were allegedly just fenced by %s for %s!", +- st_event->executioner ? st_event->executioner : "", st_event->origin); /* Dumps blackbox if enabled */ +- +- qb_log_fini(); /* Try to get the above log message to disk - somehow */ +- +- /* Get out ASAP and do not come back up. ++ /* We were notified of our own fencing. Most likely, either fencing was ++ * misconfigured, or fabric fencing that doesn't cut cluster ++ * communication is in use. + * +- * Triggering a reboot is also not the worst idea either since +- * the rest of the cluster thinks we're safely down ++ * Either way, shutting down the local host is a good idea, to require ++ * administrator intervention. Also, other nodes would otherwise likely ++ * set our status to lost because of the fencing callback and discard ++ * our subsequent election votes as "not part of our cluster". + */ +- +-#ifdef RB_HALT_SYSTEM +- reboot(RB_HALT_SYSTEM); +-#endif +- +- /* +- * If reboot() fails or is not supported, coming back up will +- * probably lead to a situation where the other nodes set our +- * status to 'lost' because of the fencing callback and will +- * discard subsequent election votes with: +- * +- * Election 87 (current: 5171, owner: 103): Processed vote from east-03 (Peer is not part of our cluster) +- * +- * So just stay dead, something is seriously messed up anyway. +- * +- */ +- exit(100); /* None of our wrappers since we already called qb_log_fini() */ ++ crm_crit("We were allegedly just fenced by %s for %s!", ++ st_event->executioner? st_event->executioner : "the cluster", ++ st_event->origin); /* Dumps blackbox if enabled */ ++ pcmk_panic(__FUNCTION__); + return; + } + +-- +1.8.3.1 + + +From b7214e4b4229f4678e09a81f6afac15ef1690406 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 23 Jul 2019 13:49:33 -0500 +Subject: [PATCH 42/96] Low: executor: stonith probes should fail only if + previously registered + +343ae2a4 made stonith probes return an error if the executor's fencer +connection was lost. However this is broader than necessary; we only need +errors to be returned for devices that were registered. Any that weren't +registered can still be assumed to be not registered. + +There's a theoretical possibility that the fencer connection could somehow be +severed and some other entity register a device between then and when the +executor reconnects. But that is not a realistic scenario, whereas probing a +fence device on a node where the fencer respawned sometime in the past is. +--- + lrmd/lrmd.c | 16 ++++++++++------ + 1 file changed, 10 insertions(+), 6 deletions(-) + +diff --git a/lrmd/lrmd.c b/lrmd/lrmd.c +index 5d33324..2e8ea41 100644 +--- a/lrmd/lrmd.c ++++ b/lrmd/lrmd.c +@@ -1085,13 +1085,17 @@ stonith_connection_failed(void) + g_hash_table_iter_init(&iter, rsc_list); + while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & rsc)) { + if (safe_str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH)) { +- /* This will cause future probes to return PCMK_OCF_UNKNOWN_ERROR +- * until the resource is stopped or started successfully. This is +- * especially important if the controller also went away (possibly +- * due to a cluster layer restart) and won't receive our client +- * notification of any monitors finalized below. ++ /* If we registered this fence device, we don't know whether the ++ * fencer still has the registration or not. Cause future probes to ++ * return PCMK_OCF_UNKNOWN_ERROR until the resource is stopped or ++ * started successfully. This is especially important if the ++ * controller also went away (possibly due to a cluster layer ++ * restart) and won't receive our client notification of any ++ * monitors finalized below. + */ +- rsc->st_probe_rc = pcmk_err_generic; ++ if (rsc->st_probe_rc == pcmk_ok) { ++ rsc->st_probe_rc = pcmk_err_generic; ++ } + + if (rsc->active) { + cmd_list = g_list_append(cmd_list, rsc->active); +-- +1.8.3.1 + + +From f392caf5dc9b26a0ba55474eb79e68c90baaca16 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 23 Jul 2019 14:25:32 -0500 +Subject: [PATCH 43/96] Feature: crmd: allow configurable reaction to local + node fencing + +9fe68fd fixed a bug so that when the local node is notified of its own fencing, +it correctly panics. + +However, some users may have been relying on the previous behavior. In +particular, some users may configure fabric fencing because they don't +want the node ever intentionally hard-powered off. + +This creates a new cluster property, fence-reaction, that controls the +behavior ("stop" for the original behavior, "panic" for the more correct +behavior). It defaults to "stop", to preserve the current behavior. +--- + crmd/control.c | 10 ++++++++++ + crmd/te_utils.c | 22 +++++++++++++++++++++- + crmd/tengine.h | 3 +++ + include/crm/msg_xml.h | 1 + + 4 files changed, 35 insertions(+), 1 deletion(-) + +diff --git a/crmd/control.c b/crmd/control.c +index 4b19114..32340ec 100644 +--- a/crmd/control.c ++++ b/crmd/control.c +@@ -932,6 +932,14 @@ pe_cluster_option crmd_opts[] = { + }, + { "node-action-limit", NULL, "integer", NULL, "0", &check_number, + "The maximum number of jobs that can be scheduled per node. Defaults to 2x cores"}, ++ { XML_CONFIG_ATTR_FENCE_REACTION, NULL, "string", NULL, "stop", NULL, ++ "How a cluster node should react if notified of its own fencing", ++ "A cluster node may receive notification of its own fencing if fencing " ++ "is misconfigured, or if fabric fencing is in use that doesn't cut " ++ "cluster communication. Allowed values are \"stop\" to attempt to " ++ "immediately stop pacemaker and stay stopped, or \"panic\" to attempt " ++ "to immediately reboot the local node, falling back to stop on failure." ++ }, + { XML_CONFIG_ATTR_ELECTION_FAIL, "election_timeout", "time", NULL, "2min", &check_timer, + "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." + }, +@@ -1053,6 +1061,8 @@ config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void + no_quorum_suicide_escalation = TRUE; + } + ++ set_fence_reaction(crmd_pref(config_hash, XML_CONFIG_ATTR_FENCE_REACTION)); ++ + value = crmd_pref(config_hash,"stonith-max-attempts"); + update_stonith_max_attempts(value); + +diff --git a/crmd/te_utils.c b/crmd/te_utils.c +index 6c7f9a0..6052bc0 100644 +--- a/crmd/te_utils.c ++++ b/crmd/te_utils.c +@@ -35,6 +35,7 @@ crm_trigger_t *stonith_reconnect = NULL; + static crm_trigger_t *stonith_history_sync_trigger = NULL; + static mainloop_timer_t *stonith_history_sync_timer_short = NULL; + static mainloop_timer_t *stonith_history_sync_timer_long = NULL; ++static bool fence_reaction_panic = FALSE; + + void + te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers) +@@ -54,6 +55,21 @@ te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers) + } + } + ++void ++set_fence_reaction(const char *reaction_s) ++{ ++ if (safe_str_eq(reaction_s, "panic")) { ++ fence_reaction_panic = TRUE; ++ ++ } else { ++ if (safe_str_neq(reaction_s, "stop")) { ++ crm_warn("Invalid value '%s' for %s, using 'stop'", ++ reaction_s, XML_CONFIG_ATTR_FENCE_REACTION); ++ } ++ fence_reaction_panic = FALSE; ++ } ++} ++ + static void + tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event) + { +@@ -278,7 +294,11 @@ tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event) + crm_crit("We were allegedly just fenced by %s for %s!", + st_event->executioner? st_event->executioner : "the cluster", + st_event->origin); /* Dumps blackbox if enabled */ +- pcmk_panic(__FUNCTION__); ++ if (fence_reaction_panic) { ++ pcmk_panic(__FUNCTION__); ++ } else { ++ crm_exit(DAEMON_RESPAWN_STOP); ++ } + return; + } + +diff --git a/crmd/tengine.h b/crmd/tengine.h +index a20760c..f5491a2 100644 +--- a/crmd/tengine.h ++++ b/crmd/tengine.h +@@ -32,6 +32,9 @@ void remove_stonith_cleanup(const char *target); + void purge_stonith_cleanup(void); + void execute_stonith_cleanup(void); + ++// reaction to notification of local node being fenced ++void set_fence_reaction(const char *reaction_s); ++ + /* tengine */ + extern crm_action_t *match_down_event(const char *target, bool quiet); + extern crm_action_t *get_cancel_action(const char *id, const char *node); +diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h +index 55f42c4..de99959 100644 +--- a/include/crm/msg_xml.h ++++ b/include/crm/msg_xml.h +@@ -377,6 +377,7 @@ + # define XML_CONFIG_ATTR_ELECTION_FAIL "election-timeout" + # define XML_CONFIG_ATTR_FORCE_QUIT "shutdown-escalation" + # define XML_CONFIG_ATTR_RECHECK "cluster-recheck-interval" ++# define XML_CONFIG_ATTR_FENCE_REACTION "fence-reaction" + + # define XML_ALERT_ATTR_PATH "path" + # define XML_ALERT_ATTR_TIMEOUT "timeout" +-- +1.8.3.1 + + +From 4fe2b0d9fac09a3228a558aaf5c1adadc7187217 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 25 Jul 2019 10:49:44 -0500 +Subject: [PATCH 44/96] Log: pacemakerd: backport pcmk_child_exit() log message + changes + +... from 2.0's f7e5558, along with corresponding CTS changes, to make patching +easier. +--- + cts/CM_lha.py | 27 +++++++++++++-------------- + cts/patterns.py | 25 +++++++++++-------------- + mcp/pacemaker.c | 22 +++++++++++++--------- + 3 files changed, 37 insertions(+), 37 deletions(-) + +diff --git a/cts/CM_lha.py b/cts/CM_lha.py +index 0ba4ba1..2f39063 100755 +--- a/cts/CM_lha.py ++++ b/cts/CM_lha.py +@@ -377,9 +377,9 @@ class crm_lha(ClusterManager): + "Exiting to recover from CCM connection failure", + r"crmd.*: Could not recover from internal error", + "crmd.*I_ERROR.*(ccm_dispatch|crmd_cib_connection_destroy)", +- "crmd.*exited with return code 2.", +- "attrd.*exited with return code 1.", +- "cib.*exited with return code 2.", ++ "crmd.*exited with status 2", ++ "attrd.*exited with status 1", ++ "cib.*exited with status 2", + + # Not if it was fenced + # "A new node joined the cluster", +@@ -400,8 +400,8 @@ class crm_lha(ClusterManager): + r"crmd.*: Input I_TERMINATE .*from do_recover", + "crmd.*I_ERROR.*crmd_cib_connection_destroy", + r"crmd.*: Could not recover from internal error", +- "crmd.*exited with return code 2.", +- "attrd.*exited with return code 1.", ++ "crmd.*exited with status 2", ++ "attrd.*exited with status 1", + ], badnews_ignore = common_ignore) + + lrmd = Process(self, "lrmd", triggersreboot=self.fastfail, pats = [ +@@ -411,7 +411,7 @@ class crm_lha(ClusterManager): + "State transition S_STARTING -> S_PENDING", + r"crmd.*: Input I_TERMINATE .*from do_recover", + r"crmd.*: Could not recover from internal error", +- "crmd.*exited with return code 2.", ++ "crmd.*exited with status 2", + ], badnews_ignore = common_ignore) + + crmd = Process(self, "crmd", triggersreboot=self.fastfail, pats = [ +@@ -425,12 +425,11 @@ class crm_lha(ClusterManager): + + pengine = Process(self, "pengine", triggersreboot=self.fastfail, pats = [ + "State transition .* S_RECOVERY", +- "crmd.*exited with return code 2.", + r"crmd.*: Input I_TERMINATE .*from do_recover", + r"crmd.*: Could not recover from internal error", + r"crmd.*CRIT.*: Connection to the Policy Engine failed", + "crmd.*I_ERROR.*save_cib_contents", +- "crmd.*exited with return code 2.", ++ "crmd.*exited with status 2", + ], badnews_ignore = common_ignore, dc_only=1) + + if self.Env["DoFencing"] == 1 : +@@ -441,21 +440,21 @@ class crm_lha(ClusterManager): + + if self.fastfail == 0: + ccm.pats.extend([ +- "attrd .* exited with return code 1", ++ "attrd .* exited with status 1", + "(ERROR|error): Respawning client .*attrd", +- "cib.* exited with return code 2", ++ "cib.* exited with status 2", + "(ERROR|error): Respawning client .*cib", +- "crmd.* exited with return code 2", ++ "crmd.* exited with status 2", + "(ERROR|error): Respawning client .*crmd" + ]) + cib.pats.extend([ +- "attrd.* exited with return code 1", ++ "attrd.* exited with status 1", + "(ERROR|error): Respawning client .*attrd", +- "crmd.* exited with return code 2", ++ "crmd.* exited with status 2", + "(ERROR|error): Respawning client .*crmd" + ]) + lrmd.pats.extend([ +- "crmd.* exited with return code 2", ++ "crmd.* exited with status 2", + "(ERROR|error): Respawning client .*crmd" + ]) + pengine.pats.extend([ +diff --git a/cts/patterns.py b/cts/patterns.py +index cf1860a..c7f0035 100644 +--- a/cts/patterns.py ++++ b/cts/patterns.py +@@ -130,7 +130,7 @@ class crm_lha(BasePatterns): + r"input=I_INTEGRATED cause=C_TIMER_POPPED", + r"input=I_FINALIZED cause=C_TIMER_POPPED", + r"input=I_ERROR", +- r", exiting\.", ++ r"(pacemakerd|lrmd|crmd):.*, exiting", + r"WARN.*Ignoring HA message.*vote.*not in our membership list", + r"pengine.*Attempting recovery of resource", + r"is taking more than 2x its timeout", +@@ -210,7 +210,7 @@ class crm_cs_v0(BasePatterns): + r"input=I_INTEGRATED cause=C_TIMER_POPPED", + r"input=I_FINALIZED cause=C_TIMER_POPPED", + r"input=I_ERROR", +- r", exiting\.", ++ r"(pacemakerd|lrmd|crmd):.*, exiting", + r"(WARN|warn).*Ignoring HA message.*vote.*not in our membership list", + r"pengine.*Attempting recovery of resource", + r"is taking more than 2x its timeout", +@@ -224,7 +224,7 @@ class crm_cs_v0(BasePatterns): + r"Faking parameter digest creation", + r"Parameters to .* action changed:", + r"Parameters to .* changed", +- r"The .* process .* terminated with signal", ++ r"\[[0-9]+\] terminated with signal [0-9]+ \(", + r"Child process .* terminated with signal", + r"pengine:.*Recover .*\(.* -\> .*\)", + r"rsyslogd.* imuxsock lost .* messages from pid .* due to rate-limiting", +@@ -281,13 +281,10 @@ class crm_cs_v0(BasePatterns): + + self.components["corosync-ignore"] = [ + r"error:.*Connection to the CPG API failed: Library error", +- r"The .* process .* exited", ++ r"\[[0-9]+\] exited with status [0-9]+ \(", + r"pacemakerd.*error:.*Child process .* exited", + r"cib.*error:.*Corosync connection lost", + r"stonith-ng.*error:.*Corosync connection terminated", +- r"The cib process .* exited: Invalid argument", +- r"The attrd process .* exited: Transport endpoint is not connected", +- r"The crmd process .* exited: Link has been severed", + r"error:.*Child process cib .* exited: Invalid argument", + r"error:.*Child process attrd .* exited: Transport endpoint is not connected", + r"error:.*Child process crmd .* exited: Link has been severed", +@@ -340,8 +337,8 @@ class crm_cs_v0(BasePatterns): + "Connection to cib_.* closed", + r"crmd.*:.*Connection to the CIB terminated...", + r"attrd.*:.*(Lost connection to CIB service|Connection to the CIB terminated)", +- "(Child process|The) crmd .* exited: Generic Pacemaker error", +- "(Child process|The) attrd .* exited: (Connection reset by peer|Transport endpoint is not connected)", ++ r"crmd\[[0-9]+\] exited with status 2", ++ r"attrd\[[0-9]+\] exited with status 1", + r"crmd.*: Input I_TERMINATE .*from do_recover", + "crmd.*I_ERROR.*crmd_cib_connection_destroy", + "crmd.*Could not recover from internal error", +@@ -354,7 +351,7 @@ class crm_cs_v0(BasePatterns): + "Connection to lrmd failed", + "Connection to lrmd.* closed", + "crmd.*I_ERROR.*lrm_connection_destroy", +- "(Child process|The) crmd .* exited: Generic Pacemaker error", ++ r"crmd\[[0-9]+\] exited with status 2", + r"crmd.*: Input I_TERMINATE .*from do_recover", + "crmd.*Could not recover from internal error", + ] +@@ -377,7 +374,7 @@ class crm_cs_v0(BasePatterns): + self.components["pengine"] = [ + "State transition .* S_RECOVERY", + "Respawning .* crmd", +- "(The|Child process) crmd .* exited: Generic Pacemaker error", ++ r"crmd\[[0-9]+\] exited with status 2", + "Connection to pengine failed", + "Connection to pengine.* closed", + "Connection to the Policy Engine failed", +@@ -437,8 +434,8 @@ class crm_mcp(crm_cs_v0): + "Pat:They_stopped" : "%s\W.*crmd.*Node %s(\[|\s).*state is now lost", + "Pat:They_dead" : "crmd.*Node %s(\[|\s).*state is now lost", + +- "Pat:ChildExit" : "The .* process exited", +- "Pat:ChildKilled" : "%s\W.*pacemakerd.*The %s process .* terminated with signal 9", ++ "Pat:ChildExit" : r"\[[0-9]+\] exited with status [0-9]+ \(", ++ "Pat:ChildKilled" : r"%s\W.*pacemakerd.*%s\[[0-9]+\] terminated with signal 9", + "Pat:ChildRespawn" : "%s\W.*pacemakerd.*Respawning failed child process: %s", + + "Pat:PacemakerUp" : "%s\W.*pacemakerd.*Starting Pacemaker", +@@ -487,7 +484,7 @@ class crm_cman(crm_cs_v0): + "Pat:They_stopped" : "%s\W.*crmd.*Node %s(\[|\s).*state is now lost", + "Pat:They_dead" : "crmd.*Node %s(\[|\s).*state is now lost", + +- "Pat:ChildKilled" : "%s\W.*pacemakerd.*The %s process .* terminated with signal 9", ++ "Pat:ChildKilled" : r"%s\W.*pacemakerd.*%s\[[0-9]+\] terminated with signal 9", + "Pat:ChildRespawn" : "%s\W.*pacemakerd.*Respawning failed child process: %s", + + "Pat:PacemakerUp" : "%s\W.*pacemakerd.*Starting Pacemaker", +diff --git a/mcp/pacemaker.c b/mcp/pacemaker.c +index 86df216..6a67b59 100644 +--- a/mcp/pacemaker.c ++++ b/mcp/pacemaker.c +@@ -217,27 +217,30 @@ pcmk_child_exit(mainloop_child_t * p, pid_t pid, int core, int signo, int exitco + pcmk_child_t *child = mainloop_child_userdata(p); + const char *name = mainloop_child_name(p); + +- if (signo && signo == SIGKILL) { +- crm_warn("The %s process (%d) terminated with signal %d (core=%d)", name, pid, signo, core); +- +- } else if (signo) { +- crm_err("The %s process (%d) terminated with signal %d (core=%d)", name, pid, signo, core); ++ if (signo) { ++ do_crm_log(((signo == SIGKILL)? LOG_WARNING : LOG_ERR), ++ "%s[%d] terminated with signal %d (core=%d)", ++ name, pid, signo, core); + + } else { + switch(exitcode) { + case pcmk_ok: +- crm_info("The %s process (%d) exited: %s (%d)", name, pid, pcmk_strerror(exitcode), exitcode); ++ crm_info("%s[%d] exited with status %d (%s)", ++ name, pid, exitcode, pcmk_strerror(exitcode)); + break; + + case DAEMON_RESPAWN_STOP: +- crm_warn("The %s process (%d) can no longer be respawned, shutting the cluster down.", name, pid); ++ crm_warn("Shutting cluster down because %s[%d] had fatal failure", ++ name, pid); + child->respawn = FALSE; + fatal_error = TRUE; + pcmk_shutdown(SIGTERM); + break; + + case pcmk_err_panic: +- do_crm_log_always(LOG_EMERG, "The %s process (%d) instructed the machine to reset", name, pid); ++ do_crm_log_always(LOG_EMERG, ++ "%s[%d] instructed the machine to reset", ++ name, pid); + child->respawn = FALSE; + fatal_error = TRUE; + pcmk_panic(__FUNCTION__); +@@ -245,7 +248,8 @@ pcmk_child_exit(mainloop_child_t * p, pid_t pid, int core, int signo, int exitco + break; + + default: +- crm_err("The %s process (%d) exited: %s (%d)", name, pid, pcmk_strerror(exitcode), exitcode); ++ crm_err("%s[%d] exited with status %d (%s)", ++ name, pid, exitcode, pcmk_strerror(exitcode)); + break; + } + } +-- +1.8.3.1 + + +From 2cc1d945e53ea8a8bae38b1d8fe36b78c731bef0 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 25 Jul 2019 10:00:14 -0500 +Subject: [PATCH 45/96] Log: pacemakerd: make daemon exit detection messages + more consistent + +Lost daemons may be detected via pcmk_child_exit() if the current pacemakerd +launched the daemon, or check_active_before_startup_processes() if a previous +pacemakerd launched it. + +Update the log messages in these cases to be more consistent, and update CTS +to detect exit via either path. +--- + cts/patterns.py | 10 ++++++---- + mcp/pacemaker.c | 15 +++++---------- + 2 files changed, 11 insertions(+), 14 deletions(-) + +diff --git a/cts/patterns.py b/cts/patterns.py +index c7f0035..e50daae 100644 +--- a/cts/patterns.py ++++ b/cts/patterns.py +@@ -224,9 +224,9 @@ class crm_cs_v0(BasePatterns): + r"Faking parameter digest creation", + r"Parameters to .* action changed:", + r"Parameters to .* changed", +- r"\[[0-9]+\] terminated with signal [0-9]+ \(", ++ r"pacemakerd.*\[[0-9]+\] terminated( with signal| as IPC server|$)", + r"Child process .* terminated with signal", +- r"pengine:.*Recover .*\(.* -\> .*\)", ++ r"pengine.*Recover .*\(.* -\> .*\)", + r"rsyslogd.* imuxsock lost .* messages from pid .* due to rate-limiting", + r"Peer is not part of our cluster", + r"We appear to be in an election loop", +@@ -435,7 +435,8 @@ class crm_mcp(crm_cs_v0): + "Pat:They_dead" : "crmd.*Node %s(\[|\s).*state is now lost", + + "Pat:ChildExit" : r"\[[0-9]+\] exited with status [0-9]+ \(", +- "Pat:ChildKilled" : r"%s\W.*pacemakerd.*%s\[[0-9]+\] terminated with signal 9", ++ # "with signal 9" == pcmk_child_exit(), "$" == check_active_before_startup_processes() ++ "Pat:ChildKilled" : r"%s\W.*pacemakerd.*%s\[[0-9]+\] terminated( with signal 9|$)", + "Pat:ChildRespawn" : "%s\W.*pacemakerd.*Respawning failed child process: %s", + + "Pat:PacemakerUp" : "%s\W.*pacemakerd.*Starting Pacemaker", +@@ -484,7 +485,8 @@ class crm_cman(crm_cs_v0): + "Pat:They_stopped" : "%s\W.*crmd.*Node %s(\[|\s).*state is now lost", + "Pat:They_dead" : "crmd.*Node %s(\[|\s).*state is now lost", + +- "Pat:ChildKilled" : r"%s\W.*pacemakerd.*%s\[[0-9]+\] terminated with signal 9", ++ # "with signal 9" == pcmk_child_exit(), "$" == check_active_before_startup_processes() ++ "Pat:ChildKilled" : r"%s\W.*pacemakerd.*%s\[[0-9]+\] terminated( with signal 9|$)", + "Pat:ChildRespawn" : "%s\W.*pacemakerd.*Respawning failed child process: %s", + + "Pat:PacemakerUp" : "%s\W.*pacemakerd.*Starting Pacemaker", +diff --git a/mcp/pacemaker.c b/mcp/pacemaker.c +index 6a67b59..1cc72af 100644 +--- a/mcp/pacemaker.c ++++ b/mcp/pacemaker.c +@@ -897,19 +897,14 @@ check_active_before_startup_processes(gpointer user_data) + case 0: + case 2: /* this very case: it was OK once already */ + if (pcmk_children[lpc].respawn == TRUE) { +- /* presumably after crash, hence critical */ +- crm_crit("Process %s terminated (pid=%lld)%s", \ +- name, (long long) +- PCMK__SPECIAL_PID_AS_0(pcmk_children[lpc].pid), +- ret ? ", at least per IPC end-point that went AWOL" +- : ""); ++ crm_err("%s[%d] terminated%s", name, ++ PCMK__SPECIAL_PID_AS_0(pcmk_children[lpc].pid), ++ ret ? " as IPC server" : ""); + } else { + /* orderly shutdown */ +- crm_notice("Process %s terminated (pid=%lld)%s", \ +- name, (long long) ++ crm_notice("%s[%d] terminated%s", name, + PCMK__SPECIAL_PID_AS_0(pcmk_children[lpc].pid), +- ret ? ", at least per IPC end-point that went AWOL" +- : ""); ++ ret ? " as IPC server" : ""); + } + pcmk_process_exit(&(pcmk_children[lpc])); + continue; +-- +1.8.3.1 + + +From ab09f0afd0d383da60411c6a719830f415e6102a Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 25 Jul 2019 11:20:04 -0500 +Subject: [PATCH 46/96] Test: CTS: alert failures are expected when executor is + killed + +--- + cts/patterns.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/cts/patterns.py b/cts/patterns.py +index e50daae..87418d4 100644 +--- a/cts/patterns.py ++++ b/cts/patterns.py +@@ -357,6 +357,7 @@ class crm_cs_v0(BasePatterns): + ] + self.components["lrmd-ignore"] = [ + r"attrd.*Connection to lrmd (failed|closed)", ++ r"(attrd|controld).*Could not execute alert", + ] + + self.components["crmd"] = [ +-- +1.8.3.1 + + +From 0654ed94b44930cd2762cad361ee19b229b6bd38 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 24 Jul 2019 12:06:21 -0500 +Subject: [PATCH 47/96] Doc: controller: document the cluster-name cluster + property + +--- + crmd/control.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/crmd/control.c b/crmd/control.c +index 32340ec..cd4223f 100644 +--- a/crmd/control.c ++++ b/crmd/control.c +@@ -897,6 +897,13 @@ pe_cluster_option crmd_opts[] = { + { "cluster-infrastructure", NULL, "string", NULL, "heartbeat", NULL, + "The messaging stack on which Pacemaker is currently running.", + "Used for informational and diagnostic purposes." }, ++ { "cluster-name", NULL, "string", NULL, NULL, NULL, ++ "An arbitrary name for the cluster", ++ "This optional value is mostly for users' convenience as desired " ++ "in administration, but may also be used in Pacemaker configuration " ++ "rules via the #cluster-name node attribute, and by higher-level tools " ++ "and resource agents." ++ }, + { XML_CONFIG_ATTR_DC_DEADTIME, "dc_deadtime", "time", NULL, "20s", &check_time, + "How long to wait for a response from other nodes during startup.", + "The \"correct\" value will depend on the speed/load of your network and the type of switches used." +-- +1.8.3.1 + + +From 065fabee559a62a400709957f514f33150a91442 Mon Sep 17 00:00:00 2001 +From: aleksei-burlakov +Date: Fri, 17 May 2019 18:13:01 +0200 +Subject: [PATCH 48/96] Low: stonith_admin --help: specify the usage of + --cleanup + +--- + fencing/admin.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/fencing/admin.c b/fencing/admin.c +index 8aef093..7da43e9 100644 +--- a/fencing/admin.c ++++ b/fencing/admin.c +@@ -58,7 +58,7 @@ static struct crm_option long_options[] = { + "\tBe less descriptive in output." + }, + { "cleanup", no_argument, NULL, 'c', +- "\tCleanup wherever appropriate." ++ "\tCleanup wherever appropriate. Requires: --history." + }, + { "broadcast", no_argument, NULL, 'b', + "Broadcast wherever appropriate." +-- +1.8.3.1 + + +From a87da1dc8512e210e892332326684aad60ec7ee0 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 15 Aug 2019 13:48:28 -0500 +Subject: [PATCH 49/96] Test: cts: check correct variable in bandwidth test + +found by static analysis (backport of 3b9dc32e from master branch) +--- + cts/CTStests.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/cts/CTStests.py b/cts/CTStests.py +index e30fe86..f328170 100644 +--- a/cts/CTStests.py ++++ b/cts/CTStests.py +@@ -1051,7 +1051,7 @@ class BandwidthTest(CTSTest): + linessplit = string.split(line," ") + for j in range(len(linessplit)-1): + if linessplit[j] == "udp": break +- if linesplit[j] == "length:": break ++ if linessplit[j] == "length:": break + try: + sum = int(linessplit[j+1]) + sum + except ValueError: +-- +1.8.3.1 + + +From 719c34f4b2da87970058326cb14d8488f10f90f5 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 17 Jun 2019 20:30:19 -0500 +Subject: [PATCH 50/96] Refactor: libcrmcommon: functionize freeing an XML + subtree + +... to reduce code duplication, and draw a distinction between this and freeing +the entire XML document the element is in (which free_xml() does). +--- + include/crm/common/xml.h | 1 + + lib/common/acl.c | 7 ++++--- + lib/common/xml.c | 22 ++++++++++++++-------- + 3 files changed, 19 insertions(+), 11 deletions(-) + +diff --git a/include/crm/common/xml.h b/include/crm/common/xml.h +index 11a03f4..a9af963 100644 +--- a/include/crm/common/xml.h ++++ b/include/crm/common/xml.h +@@ -271,6 +271,7 @@ __xml_next_element(xmlNode * child) + return NULL; + } + ++void pcmk_free_xml_subtree(xmlNode *xml); + void free_xml(xmlNode * child); + + xmlNode *first_named_child(xmlNode * parent, const char *name); +diff --git a/lib/common/acl.c b/lib/common/acl.c +index 80b1f6f..30adad8 100644 +--- a/lib/common/acl.c ++++ b/lib/common/acl.c +@@ -1,5 +1,7 @@ + /* +- * Copyright 2004-2018 Andrew Beekhof ++ * Copyright 2004-2019 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. +@@ -501,8 +503,7 @@ pcmk__post_process_acl(xmlNode *xml) + crm_element_name(xml), path); + + if (xml != xmlDocGetRootElement(xml->doc)) { +- xmlUnlinkNode(xml); +- xmlFreeNode(xml); ++ pcmk_free_xml_subtree(xml); + } + free(path); + return; +diff --git a/lib/common/xml.c b/lib/common/xml.c +index 91c0edb..dfa2d77 100644 +--- a/lib/common/xml.c ++++ b/lib/common/xml.c +@@ -2021,6 +2021,18 @@ xml_get_path(xmlNode *xml) + return NULL; + } + ++/*! ++ * Free an XML element and all of its children, removing it from its parent ++ * ++ * \param[in] xml XML element to free ++ */ ++void ++pcmk_free_xml_subtree(xmlNode *xml) ++{ ++ xmlUnlinkNode(xml); // Detaches from parent and siblings ++ xmlFreeNode(xml); // Frees ++} ++ + static void + free_xml_with_position(xmlNode * child, int position) + { +@@ -2075,12 +2087,7 @@ free_xml_with_position(xmlNode * child, int position) + pcmk__set_xml_flag(child, xpf_dirty); + } + } +- +- /* Free this particular subtree +- * Make sure to unlink it from the parent first +- */ +- xmlUnlinkNode(child); +- xmlFreeNode(child); ++ pcmk_free_xml_subtree(child); + } + } + } +@@ -2296,8 +2303,7 @@ strip_text_nodes(xmlNode * xml) + switch (iter->type) { + case XML_TEXT_NODE: + /* Remove it */ +- xmlUnlinkNode(iter); +- xmlFreeNode(iter); ++ pcmk_free_xml_subtree(iter); + break; + + case XML_ELEMENT_NODE: +-- +1.8.3.1 + + +From 310459de07dc6b3bb6e5a851fff1f25559caee2e Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 17 Jun 2019 21:08:57 -0500 +Subject: [PATCH 51/96] Refactor: libcrmcommon: make ACL creation checks more + efficient + +This does the simplest checks first, avoids doing the same check repeatedly, +and doesn't gather information unless needed. + +Trace log messages are also improved, and the regression tests updated to +match. +--- + lib/common/acl.c | 76 +++++++++++++++++++++++++++++++++++--------------------- + 1 file changed, 48 insertions(+), 28 deletions(-) + +diff --git a/lib/common/acl.c b/lib/common/acl.c +index 30adad8..dea67cf 100644 +--- a/lib/common/acl.c ++++ b/lib/common/acl.c +@@ -471,6 +471,41 @@ xml_acl_filtered_copy(const char *user, xmlNode *acl_source, xmlNode *xml, + return TRUE; + } + ++/*! ++ * \internal ++ * \brief Check whether creation of an XML element is implicitly allowed ++ * ++ * Check whether XML is a "scaffolding" element whose creation is implicitly ++ * allowed regardless of ACLs (that is, it is not in the ACL section and has ++ * no attributes other than "id"). ++ * ++ * \param[in] xml XML element to check ++ * ++ * \return TRUE if XML element is implicitly allowed, FALSE otherwise ++ */ ++static bool ++implicitly_allowed(xmlNode *xml) ++{ ++ char *path = NULL; ++ ++ for (xmlAttr *prop = xml->properties; prop != NULL; prop = prop->next) { ++ if (strcmp((const char *) prop->name, XML_ATTR_ID) != 0) { ++ return FALSE; ++ } ++ } ++ ++ path = xml_get_path(xml); ++ if (strstr(path, "/" XML_CIB_TAG_ACLS "/") != NULL) { ++ free(path); ++ return FALSE; ++ } ++ free(path); ++ ++ return TRUE; ++} ++ ++#define display_id(xml) (ID(xml)? ID(xml) : "") ++ + void + pcmk__post_process_acl(xmlNode *xml) + { +@@ -478,38 +513,23 @@ pcmk__post_process_acl(xmlNode *xml) + xml_private_t *p = xml->_private; + + if (is_set(p->flags, xpf_created)) { +- xmlAttr *xIter = NULL; +- char *path = xml_get_path(xml); ++ if (implicitly_allowed(xml)) { ++ crm_trace("Creation of <%s> scaffolding with id=\"%s\"" ++ " is implicitly allowed", ++ crm_element_name(xml), display_id(xml)); + +- /* Always allow new scaffolding (e.g. node with no attributes or only an +- * 'id'), except in the ACLs section +- */ +- +- for (xIter = xml->properties; xIter != NULL; xIter = xIter->next) { +- const char *prop_name = (const char *)xIter->name; +- +- if (!strcmp(prop_name, XML_ATTR_ID) +- && !strstr(path, "/"XML_CIB_TAG_ACLS"/")) { +- /* Delay the acl check */ +- continue; ++ } else if (pcmk__check_acl(xml, NULL, xpf_acl_write)) { ++ crm_trace("ACLs allow creation of <%s> with id=\"%s\"", ++ crm_element_name(xml), display_id(xml)); + +- } else if (pcmk__check_acl(xml, NULL, xpf_acl_write)) { +- crm_trace("Creation of %s=%s is allowed", +- crm_element_name(xml), ID(xml)); +- break; +- +- } else { +- crm_trace("Cannot add new node %s at %s", +- crm_element_name(xml), path); +- +- if (xml != xmlDocGetRootElement(xml->doc)) { +- pcmk_free_xml_subtree(xml); +- } +- free(path); +- return; ++ } else { ++ crm_trace("ACLs disallow creation of <%s> with id=\"%s\"", ++ crm_element_name(xml), display_id(xml)); ++ if (xml != xmlDocGetRootElement(xml->doc)) { ++ pcmk_free_xml_subtree(xml); + } ++ return; + } +- free(path); + } + + while (cIter != NULL) { +-- +1.8.3.1 + + +From fec0a2a12b099a14b7db9397fdba08d5b6c22456 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 15 Aug 2019 14:00:48 -0500 +Subject: [PATCH 52/96] Test: tools: update regression tests for ACL trace + message changes + +--- + tools/regression.acls.exp | 70 +++++++++++++++++++++++++---------------------- + 1 file changed, 37 insertions(+), 33 deletions(-) + +diff --git a/tools/regression.acls.exp b/tools/regression.acls.exp +index 6bc6062..6508b2c 100644 +--- a/tools/regression.acls.exp ++++ b/tools/regression.acls.exp +@@ -257,6 +257,7 @@ Error performing operation: Permission denied + ( acl.c:NNN ) trace: pcmk__check_acl: Ordinary user unknownguy cannot access the CIB without any defined ACLs + ( acl.c:NNN ) trace: pcmk__check_acl: Ordinary user unknownguy cannot access the CIB without any defined ACLs + ( acl.c:NNN ) trace: pcmk__check_acl: Ordinary user unknownguy cannot access the CIB without any defined ACLs ++( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of scaffolding with id="" is implicitly allowed + Call failed: Permission denied + =#=#=#= End test: unknownguy: Create a resource - Permission denied (13) =#=#=#= + * Passed: cibadmin - unknownguy: Create a resource +@@ -274,7 +275,7 @@ Error performing operation: Permission denied + * Passed: crm_attribute - l33t-haxor: Set stonith-enabled + =#=#=#= Begin test: l33t-haxor: Create a resource =#=#=#= + ( acl.c:NNN ) trace: pcmk__check_acl: 400 access denied to /cib/configuration/resources/primitive[@id='dummy']: parent +-( acl.c:NNN ) trace: pcmk__post_process_acl: Cannot add new node primitive at /cib/configuration/resources/primitive[@id='dummy'] ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs disallow creation of with id="dummy" + Call failed: Permission denied + =#=#=#= End test: l33t-haxor: Create a resource - Permission denied (13) =#=#=#= + * Passed: cibadmin - l33t-haxor: Create a resource +@@ -329,7 +330,7 @@ Error setting enable-acl=false (section=crm_config, set=): Permission deni + =#=#=#= End test: niceguy: Set enable-acl - Permission denied (13) =#=#=#= + * Passed: crm_attribute - niceguy: Set enable-acl + =#=#=#= Begin test: niceguy: Set stonith-enabled =#=#=#= +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of nvpair=cib-bootstrap-options-stonith-enabled is allowed ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="cib-bootstrap-options-stonith-enabled" + =#=#=#= Current cib after: niceguy: Set stonith-enabled =#=#=#= + + +@@ -377,7 +378,7 @@ Error setting enable-acl=false (section=crm_config, set=): Permission deni + * Passed: crm_attribute - niceguy: Set stonith-enabled + =#=#=#= Begin test: niceguy: Create a resource =#=#=#= + ( acl.c:NNN ) trace: pcmk__check_acl: 400 access denied to /cib/configuration/resources/primitive[@id='dummy']: default +-( acl.c:NNN ) trace: pcmk__post_process_acl: Cannot add new node primitive at /cib/configuration/resources/primitive[@id='dummy'] ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs disallow creation of with id="dummy" + Call failed: Permission denied + =#=#=#= End test: niceguy: Create a resource - Permission denied (13) =#=#=#= + * Passed: cibadmin - niceguy: Create a resource +@@ -536,7 +537,8 @@ Error performing operation: Permission denied + error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined + error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option + error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of nvpair=dummy-meta_attributes-target-role is allowed ++( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of scaffolding with id="dummy-meta_attributes" is implicitly allowed ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="dummy-meta_attributes-target-role" + + Set 'dummy' option: id=dummy-meta_attributes-target-role set=dummy-meta_attributes name=target-role=Stopped + =#=#=#= Current cib after: niceguy: Create a resource meta attribute =#=#=#= +@@ -704,7 +706,7 @@ Deleted 'dummy' option: id=dummy-meta_attributes-target-role name=target-role + error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined + error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option + error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of nvpair=dummy-meta_attributes-target-role is allowed ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="dummy-meta_attributes-target-role" + + Set 'dummy' option: id=dummy-meta_attributes-target-role set=dummy-meta_attributes name=target-role=Started + =#=#=#= Current cib after: niceguy: Create a resource meta attribute =#=#=#= +@@ -865,7 +867,7 @@ Call failed: Permission denied + =#=#=#= Begin test: niceguy: Replace - create resource =#=#=#= + ( acl.c:NNN ) trace: pcmk__check_acl: 400 access denied to /cib[@epoch]: default + ( acl.c:NNN ) trace: pcmk__check_acl: 400 access denied to /cib/configuration/resources/primitive[@id='dummy2']: default +-( acl.c:NNN ) trace: pcmk__post_process_acl: Cannot add new node primitive at /cib/configuration/resources/primitive[@id='dummy2'] ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs disallow creation of with id="dummy2" + Call failed: Permission denied + =#=#=#= End test: niceguy: Replace - create resource - Permission denied (13) =#=#=#= + * Passed: cibadmin - niceguy: Replace - create resource +@@ -1184,28 +1186,28 @@ Call failed: Permission denied + + !#!#!#!#! Upgrading to pacemaker-2.0 and retesting !#!#!#!#! + =#=#=#= Begin test: root: Upgrade to pacemaker-2.0 =#=#=#= +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_permission=observer-read-1 is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_permission=observer-write-1 is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_permission=observer-write-2 is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_permission=admin-read-1 is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_permission=admin-write-1 is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_target=l33t-haxor is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of role=auto-l33t-haxor is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_role=auto-l33t-haxor is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_permission=crook-nothing is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_target=niceguy is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of role=observer is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_target=bob is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of role=admin is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_target=badidea is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of role=auto-badidea is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_role=auto-badidea is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_permission=badidea-resources is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_target=betteridea is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of role=auto-betteridea is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_role=auto-betteridea is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_permission=betteridea-nothing is allowed +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of acl_permission=betteridea-resources is allowed ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="observer-read-1" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="observer-write-1" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="observer-write-2" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="admin-read-1" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="admin-write-1" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="l33t-haxor" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="auto-l33t-haxor" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="auto-l33t-haxor" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="crook-nothing" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="niceguy" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="observer" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="bob" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="admin" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="badidea" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="auto-badidea" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="auto-badidea" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="badidea-resources" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="betteridea" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="auto-betteridea" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="auto-betteridea" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="betteridea-nothing" ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="betteridea-resources" + =#=#=#= Current cib after: root: Upgrade to pacemaker-2.0 =#=#=#= + + +@@ -1279,6 +1281,7 @@ Error performing operation: Permission denied + ( acl.c:NNN ) trace: pcmk__check_acl: Ordinary user unknownguy cannot access the CIB without any defined ACLs + ( acl.c:NNN ) trace: pcmk__check_acl: Ordinary user unknownguy cannot access the CIB without any defined ACLs + ( acl.c:NNN ) trace: pcmk__check_acl: Ordinary user unknownguy cannot access the CIB without any defined ACLs ++( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of scaffolding with id="" is implicitly allowed + Call failed: Permission denied + =#=#=#= End test: unknownguy: Create a resource - Permission denied (13) =#=#=#= + * Passed: cibadmin - unknownguy: Create a resource +@@ -1296,7 +1299,7 @@ Error performing operation: Permission denied + * Passed: crm_attribute - l33t-haxor: Set stonith-enabled + =#=#=#= Begin test: l33t-haxor: Create a resource =#=#=#= + ( acl.c:NNN ) trace: pcmk__check_acl: 400 access denied to /cib/configuration/resources/primitive[@id='dummy']: parent +-( acl.c:NNN ) trace: pcmk__post_process_acl: Cannot add new node primitive at /cib/configuration/resources/primitive[@id='dummy'] ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs disallow creation of with id="dummy" + Call failed: Permission denied + =#=#=#= End test: l33t-haxor: Create a resource - Permission denied (13) =#=#=#= + * Passed: cibadmin - l33t-haxor: Create a resource +@@ -1417,7 +1420,7 @@ Error setting enable-acl=false (section=crm_config, set=): Permission deni + * Passed: crm_attribute - niceguy: Set stonith-enabled + =#=#=#= Begin test: niceguy: Create a resource =#=#=#= + ( acl.c:NNN ) trace: pcmk__check_acl: 400 access denied to /cib/configuration/resources/primitive[@id='dummy']: default +-( acl.c:NNN ) trace: pcmk__post_process_acl: Cannot add new node primitive at /cib/configuration/resources/primitive[@id='dummy'] ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs disallow creation of with id="dummy" + Call failed: Permission denied + =#=#=#= End test: niceguy: Create a resource - Permission denied (13) =#=#=#= + * Passed: cibadmin - niceguy: Create a resource +@@ -1603,7 +1606,8 @@ Error performing operation: Permission denied + error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined + error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option + error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of nvpair=dummy-meta_attributes-target-role is allowed ++( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of scaffolding with id="dummy-meta_attributes" is implicitly allowed ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="dummy-meta_attributes-target-role" + + Set 'dummy' option: id=dummy-meta_attributes-target-role set=dummy-meta_attributes name=target-role=Stopped + =#=#=#= Current cib after: niceguy: Create a resource meta attribute =#=#=#= +@@ -1798,7 +1802,7 @@ Deleted 'dummy' option: id=dummy-meta_attributes-target-role name=target-role + error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined + error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option + error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity +-( acl.c:NNN ) trace: pcmk__post_process_acl: Creation of nvpair=dummy-meta_attributes-target-role is allowed ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs allow creation of with id="dummy-meta_attributes-target-role" + + Set 'dummy' option: id=dummy-meta_attributes-target-role set=dummy-meta_attributes name=target-role=Started + =#=#=#= Current cib after: niceguy: Create a resource meta attribute =#=#=#= +@@ -1977,7 +1981,7 @@ Call failed: Permission denied + =#=#=#= Begin test: niceguy: Replace - create resource =#=#=#= + ( acl.c:NNN ) trace: pcmk__check_acl: 400 access denied to /cib[@epoch]: default + ( acl.c:NNN ) trace: pcmk__check_acl: 400 access denied to /cib/configuration/resources/primitive[@id='dummy2']: default +-( acl.c:NNN ) trace: pcmk__post_process_acl: Cannot add new node primitive at /cib/configuration/resources/primitive[@id='dummy2'] ++( acl.c:NNN ) trace: pcmk__post_process_acl: ACLs disallow creation of with id="dummy2" + Call failed: Permission denied + =#=#=#= End test: niceguy: Replace - create resource - Permission denied (13) =#=#=#= + * Passed: cibadmin - niceguy: Replace - create resource +-- +1.8.3.1 + + +From ecefc149f2a7f1212e678ba2824755b2d13d848a Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 18 Jun 2019 17:00:51 -0500 +Subject: [PATCH 53/96] Low: libcrmcommon: avoid use-after-free when enforcing + creation ACLs + +As detected by static analysis, pcmk__post_process_acl() can free its argument, +yet some callers continued to use the argument afterward. + +The existing code apparently tried to get around this by freeing the argument +only if it wasn't the root element of the XML document. However some callers +do pass non-root elements. + +The circumstances where the use-after-free could occur seem limited enough +that it hasn't been seen in practice. + +This avoids any chance of use-after-free by adding an argument indicating +whether the argument itself should be checked, or just its children (replacing +the root element check). All callers specify just children, except +pcmk__post_process_acl()'s recursive calls for the children. + +__xml_diff_object() gets a similar argument to pass along to +pcmk__post_process_acl(). +--- + lib/common/acl.c | 32 +++++++++++++++++++++++--------- + lib/common/crmcommon_private.h | 6 ++++-- + lib/common/xml.c | 10 +++++----- + 3 files changed, 32 insertions(+), 16 deletions(-) + +diff --git a/lib/common/acl.c b/lib/common/acl.c +index dea67cf..72dc707 100644 +--- a/lib/common/acl.c ++++ b/lib/common/acl.c +@@ -506,10 +506,22 @@ implicitly_allowed(xmlNode *xml) + + #define display_id(xml) (ID(xml)? ID(xml) : "") + ++/*! ++ * \internal ++ * \brief Drop XML nodes created in violation of ACLs ++ * ++ * Given an XML element, free all of its descendent nodes created in violation ++ * of ACLs, with the exception of allowing "scaffolding" elements (i.e. those ++ * that aren't in the ACL section and don't have any attributes other than ++ * "id"). ++ * ++ * \param[in,out] xml XML to check ++ * \param[in] check_top Whether to apply checks to argument itself ++ * (if TRUE, xml might get freed) ++ */ + void +-pcmk__post_process_acl(xmlNode *xml) ++pcmk__post_process_acl(xmlNode *xml, bool check_top) + { +- xmlNode *cIter = __xml_first_child(xml); + xml_private_t *p = xml->_private; + + if (is_set(p->flags, xpf_created)) { +@@ -522,20 +534,22 @@ pcmk__post_process_acl(xmlNode *xml) + crm_trace("ACLs allow creation of <%s> with id=\"%s\"", + crm_element_name(xml), display_id(xml)); + +- } else { ++ } else if (check_top) { + crm_trace("ACLs disallow creation of <%s> with id=\"%s\"", + crm_element_name(xml), display_id(xml)); +- if (xml != xmlDocGetRootElement(xml->doc)) { +- pcmk_free_xml_subtree(xml); +- } ++ pcmk_free_xml_subtree(xml); + return; ++ ++ } else { ++ crm_trace("ACLs would disallow creation of <%s> with id=\"%s\"", ++ crm_element_name(xml), display_id(xml)); + } + } + +- while (cIter != NULL) { ++ for (xmlNode *cIter = __xml_first_child(xml); cIter != NULL; ) { + xmlNode *child = cIter; + cIter = __xml_next(cIter); /* In case it is free'd */ +- pcmk__post_process_acl(child); ++ pcmk__post_process_acl(child, TRUE); + } + } + +@@ -558,7 +572,7 @@ xml_acl_disable(xmlNode *xml) + + /* Catch anything that was created but shouldn't have been */ + pcmk__apply_acl(xml); +- pcmk__post_process_acl(xml); ++ pcmk__post_process_acl(xml, FALSE); + clear_bit(p->flags, xpf_acl_enabled); + } + } +diff --git a/lib/common/crmcommon_private.h b/lib/common/crmcommon_private.h +index 113f525..b153873 100644 +--- a/lib/common/crmcommon_private.h ++++ b/lib/common/crmcommon_private.h +@@ -1,5 +1,7 @@ + /* +- * Copyright 2018-2019 Andrew Beekhof ++ * Copyright 2018-2019 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. +@@ -66,7 +68,7 @@ G_GNUC_INTERNAL + void pcmk__apply_acl(xmlNode *xml); + + G_GNUC_INTERNAL +-void pcmk__post_process_acl(xmlNode *xml); ++void pcmk__post_process_acl(xmlNode *xml, bool check_top); + + G_GNUC_INTERNAL + void pcmk__mark_xml_attr_dirty(xmlAttr *a); +diff --git a/lib/common/xml.c b/lib/common/xml.c +index dfa2d77..9fd83a8 100644 +--- a/lib/common/xml.c ++++ b/lib/common/xml.c +@@ -3368,7 +3368,7 @@ apply_xml_diff(xmlNode * old, xmlNode * diff, xmlNode ** new) + } + + static void +-__xml_diff_object(xmlNode * old, xmlNode * new) ++__xml_diff_object(xmlNode * old, xmlNode * new, bool check_top) + { + xmlNode *cIter = NULL; + xmlAttr *pIter = NULL; +@@ -3376,7 +3376,7 @@ __xml_diff_object(xmlNode * old, xmlNode * new) + CRM_CHECK(new != NULL, return); + if(old == NULL) { + crm_node_created(new); +- pcmk__post_process_acl(new); // Check creation is allowed ++ pcmk__post_process_acl(new, check_top); // Check creation is allowed + return; + + } else { +@@ -3483,7 +3483,7 @@ __xml_diff_object(xmlNode * old, xmlNode * new) + + cIter = __xml_next(cIter); + if(new_child) { +- __xml_diff_object(old_child, new_child); ++ __xml_diff_object(old_child, new_child, TRUE); + + } else { + /* Create then free (which will check the acls if necessary) */ +@@ -3511,7 +3511,7 @@ __xml_diff_object(xmlNode * old, xmlNode * new) + if(old_child == NULL) { + xml_private_t *p = new_child->_private; + p->flags |= xpf_skip; +- __xml_diff_object(old_child, new_child); ++ __xml_diff_object(old_child, new_child, TRUE); + + } else { + /* Check for movement, we already checked for differences */ +@@ -3554,7 +3554,7 @@ xml_calculate_changes(xmlNode * old, xmlNode * new) + xml_track_changes(new, NULL, NULL, FALSE); + } + +- __xml_diff_object(old, new); ++ __xml_diff_object(old, new, FALSE); + } + + xmlNode * +-- +1.8.3.1 + + +From b24cd86fa98fa042d40e6f0820c36d12e14ff3ed Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 15 Aug 2019 14:13:32 -0500 +Subject: [PATCH 54/96] Fix: extra: handle run-as-user properly in ClusterMon + +7b303943 improperly searched for the crm_mon process when the user option was +set (regression since 1.1.16) +--- + extra/resources/ClusterMon | 18 ++++++++++++++---- + 1 file changed, 14 insertions(+), 4 deletions(-) + +diff --git a/extra/resources/ClusterMon b/extra/resources/ClusterMon +index 0604a26..0f4eb55 100755 +--- a/extra/resources/ClusterMon ++++ b/extra/resources/ClusterMon +@@ -152,14 +152,24 @@ ClusterMon_stop() { + } + + ClusterMon_monitor() { ++ local USERARG="" ++ local header ++ local pid ++ + if [ -f $OCF_RESKEY_pidfile ]; then + pid=`cat $OCF_RESKEY_pidfile` + if [ ! -z $pid ]; then +- str=$(echo "su - $OCF_RESKEY_user -c \"$CMON_CMD\"" | tr 'crmon, \t' 'xxxxxxxx') +- ps -o "args=${str}" -p $pid 2>/dev/null | \ ++ if [ -n "$OCF_RESKEY_user" ]; then ++ USERARG="-u $OCF_RESKEY_user" ++ fi ++ ++ # Use column header wide as command, to ensure it's shown in full ++ header=$(echo $CMON_CMD | tr 'crmon, \t' 'xxxxxxxx') ++ ++ ps $USERARG -o "args=${header}" -p $pid 2>/dev/null | \ + grep -qE "[c]rm_mon.*${OCF_RESKEY_pidfile}" +- rc=$? +- case $rc in ++ ++ case $? in + 0) exit $OCF_SUCCESS;; + 1) exit $OCF_NOT_RUNNING;; + *) exit $OCF_ERR_GENERIC;; +-- +1.8.3.1 + + +From cdf23bd35084a13a02a394cada2d5baae857f47f Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 18 Jul 2019 20:36:16 -0500 +Subject: [PATCH 55/96] Fix: extra: calculate #health_disk correctly in SysInfo + +Previously, if SysInfo monitored multiple disks, the status of the last disk in +the list was used as the value of #health_disk. Now, #health_disk matches the +documentation in setting #health_disk red if any disk in the list is low on +space. +--- + extra/resources/SysInfo | 10 ++++------ + 1 file changed, 4 insertions(+), 6 deletions(-) + +diff --git a/extra/resources/SysInfo b/extra/resources/SysInfo +index 7d1c0a3..4441026 100755 +--- a/extra/resources/SysInfo ++++ b/extra/resources/SysInfo +@@ -157,6 +157,7 @@ UpdateStat() { + } + + SysInfoStats() { ++ local DISK_STATUS="green" + + UpdateStat arch "`uname -m`" + UpdateStat os "`uname -s`-`uname -r`" +@@ -241,15 +242,12 @@ SysInfoStats() { + disk_label=`echo $disk | sed -e 's#^/$#root#;s#^/*##;s#/#_#g'` + disk_free=`SysInfo_hdd_units $disk_free` + UpdateStat ${disk_label}_free $disk_free +- if [ -n "$MIN_FREE" ]; then +- if [ $disk_free -le $MIN_FREE ]; then +- UpdateStat "#health_disk" "red" +- else +- UpdateStat "#health_disk" "green" +- fi ++ if [ -n "$MIN_FREE" ] && [ $disk_free -le $MIN_FREE ]; then ++ DISK_STATUS="red" + fi + fi + done ++ UpdateStat "#health_disk" "$DISK_STATUS" + } + + SysInfo_megabytes() { +-- +1.8.3.1 + + +From dd8c31f794eb3cbbc37bf1abd6a2ec90374b7c36 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 15 Aug 2019 14:43:09 -0500 +Subject: [PATCH 56/96] Test: cts-cli: add test for crm_diff + +--- + tools/Makefile.am | 8 ++++-- + tools/crm_diff_new.xml | 54 +++++++++++++++++++++++++++++++++++++ + tools/crm_diff_old.xml | 54 +++++++++++++++++++++++++++++++++++++ + tools/regression.sh | 5 ++++ + tools/regression.tools.exp | 67 ++++++++++++++++++++++++++++++++++++++++++++-- + 5 files changed, 184 insertions(+), 4 deletions(-) + create mode 100644 tools/crm_diff_new.xml + create mode 100644 tools/crm_diff_old.xml + +diff --git a/tools/Makefile.am b/tools/Makefile.am +index 3548035..d8c3215 100644 +--- a/tools/Makefile.am ++++ b/tools/Makefile.am +@@ -42,8 +42,12 @@ sbin_PROGRAMS = crm_simulate crmadmin cibadmin crm_node crm_attribute crm_resou + + testdir = $(datadir)/$(PACKAGE)/tests/cli + test_SCRIPTS = regression.sh +-test_DATA = regression.dates.exp regression.tools.exp regression.acls.exp \ +- regression.validity.exp ++test_DATA = regression.dates.exp \ ++ regression.tools.exp \ ++ regression.acls.exp \ ++ regression.validity.exp \ ++ crm_diff_new.xml \ ++ crm_diff_old.xml + + if BUILD_HEARTBEAT_SUPPORT + sbin_PROGRAMS += crm_uuid +diff --git a/tools/crm_diff_new.xml b/tools/crm_diff_new.xml +new file mode 100644 +index 0000000..7c2ec22 +--- /dev/null ++++ b/tools/crm_diff_new.xml +@@ -0,0 +1,54 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/tools/crm_diff_old.xml b/tools/crm_diff_old.xml +new file mode 100644 +index 0000000..8a92edd +--- /dev/null ++++ b/tools/crm_diff_old.xml +@@ -0,0 +1,54 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/tools/regression.sh b/tools/regression.sh +index 3680f13..2765595 100755 +--- a/tools/regression.sh ++++ b/tools/regression.sh +@@ -9,6 +9,7 @@ verbose=0 + tests="dates tools acls validity" + + CRM_EX_OK=0 ++CRM_EX_ERROR=1 + + function test_assert() { + target=$1; shift +@@ -388,6 +389,10 @@ function test_tools() { + test_assert $CRM_EX_OK + + rm -f /tmp/$$.existing.xml /tmp/$$.resources.xml ++ ++ desc="Create an XML patchset" ++ cmd="crm_diff -o $test_home/crm_diff_old.xml -n $test_home/crm_diff_new.xml" ++ test_assert $CRM_EX_ERROR 0 + } + + function test_dates() { +diff --git a/tools/regression.tools.exp b/tools/regression.tools.exp +index 5be42c8..900544a 100644 +--- a/tools/regression.tools.exp ++++ b/tools/regression.tools.exp +@@ -3074,8 +3074,8 @@ Migration will take effect until: + * Passed: crm_resource - Try to move a resource previously moved with a lifetime + =#=#=#= Begin test: Ban dummy from node1 for a short time =#=#=#= + WARNING: Creating rsc_location constraint 'cli-ban-dummy-on-node1' with a score of -INFINITY for resource dummy on node1. +- This will prevent dummy from running on node1 until the constraint is removed using the clear option or by editing the CIB with an appropriate tool +- This will be the case even if node1 is the last node in the cluster ++ This will prevent dummy from running on node1 until the constraint is removed using the clear option or by editing the CIB with an appropriate tool ++ This will be the case even if node1 is the last node in the cluster + Migration will take effect until: + =#=#=#= Current cib after: Ban dummy from node1 for a short time =#=#=#= + +@@ -3174,3 +3174,66 @@ Migration will take effect until: + + =#=#=#= End test: Remove expired constraints - OK (0) =#=#=#= + * Passed: crm_resource - Remove expired constraints ++=#=#=#= Begin test: Create an XML patchset =#=#=#= ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++=#=#=#= End test: Create an XML patchset - Operation not permitted (1) =#=#=#= ++* Passed: crm_diff - Create an XML patchset +-- +1.8.3.1 + + +From d542a0579095471a5e3e21d7de2918051ab95ef1 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 15 Aug 2019 15:04:24 -0500 +Subject: [PATCH 57/96] Refactor: libcrmcommon: add assertion + +not really needed, but will hopefully make static analysis happy +--- + lib/common/xml.c | 15 ++++++++------- + 1 file changed, 8 insertions(+), 7 deletions(-) + +diff --git a/lib/common/xml.c b/lib/common/xml.c +index 9fd83a8..ae4cc6a 100644 +--- a/lib/common/xml.c ++++ b/lib/common/xml.c +@@ -3372,22 +3372,23 @@ __xml_diff_object(xmlNode * old, xmlNode * new, bool check_top) + { + xmlNode *cIter = NULL; + xmlAttr *pIter = NULL; ++ xml_private_t *p = NULL; + + CRM_CHECK(new != NULL, return); + if(old == NULL) { + crm_node_created(new); + pcmk__post_process_acl(new, check_top); // Check creation is allowed + return; ++ } + +- } else { +- xml_private_t *p = new->_private; ++ p = new->_private; ++ CRM_CHECK(p != NULL, return); + +- if(p->flags & xpf_processed) { +- /* Avoid re-comparing nodes */ +- return; +- } +- p->flags |= xpf_processed; ++ if(p->flags & xpf_processed) { ++ /* Avoid re-comparing nodes */ ++ return; + } ++ p->flags |= xpf_processed; + + for (pIter = pcmk__first_xml_attr(new); pIter != NULL; pIter = pIter->next) { + xml_private_t *p = pIter->_private; +-- +1.8.3.1 + + +From 01f56916870c72f054829b2276111729fc9e52de Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 12 Aug 2019 13:18:24 -0500 +Subject: [PATCH 58/96] Log: pacemakerd: log a better warning if unable to + create /var/run/crm + +also makes static analysis happy +--- + mcp/pacemaker.c | 8 ++++++-- + 1 file changed, 6 insertions(+), 2 deletions(-) + +diff --git a/mcp/pacemaker.c b/mcp/pacemaker.c +index 1cc72af..dbe6220 100644 +--- a/mcp/pacemaker.c ++++ b/mcp/pacemaker.c +@@ -1346,8 +1346,12 @@ main(int argc, char **argv) + crm_exit(ENOKEY); + } + +- mkdir(CRM_STATE_DIR, 0750); +- mcp_chown(CRM_STATE_DIR, pcmk_uid, pcmk_gid); ++ // Used by some resource agents ++ if ((mkdir(CRM_STATE_DIR, 0750) < 0) && (errno != EEXIST)) { ++ crm_warn("Could not create " CRM_STATE_DIR ": %s", pcmk_strerror(errno)); ++ } else { ++ mcp_chown(CRM_STATE_DIR, pcmk_uid, pcmk_gid); ++ } + + /* Used to store core/blackbox/pengine/cib files in */ + crm_build_path(CRM_PACEMAKER_DIR, 0750); +-- +1.8.3.1 + + +From b7b7e5ca6df11a9a21b9b61352741f4c3d9f5bf6 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 12 Aug 2019 13:29:44 -0500 +Subject: [PATCH 59/96] Log: pacemakerd: tweak messages for checking for + existing instance + +and silence a static analysis warning about unused return value +--- + mcp/pacemaker.c | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/mcp/pacemaker.c b/mcp/pacemaker.c +index dbe6220..6ec25be 100644 +--- a/mcp/pacemaker.c ++++ b/mcp/pacemaker.c +@@ -1280,12 +1280,12 @@ main(int argc, char **argv) + /* Restore the original facility so that mcp_read_config() does the right thing */ + set_daemon_option("logfacility", facility); + +- crm_debug("Checking for old instances of %s", CRM_SYSTEM_MCP); ++ crm_debug("Checking for existing Pacemaker instance"); + old_instance = crm_ipc_new(CRM_SYSTEM_MCP, 0); +- crm_ipc_connect(old_instance); ++ (void) crm_ipc_connect(old_instance); + + if (shutdown) { +- crm_debug("Terminating previous instance"); ++ crm_debug("Shutting down existing Pacemaker instance by request"); + while (crm_ipc_connected(old_instance)) { + xmlNode *cmd = + create_request(CRM_OP_QUIT, NULL, NULL, CRM_SYSTEM_MCP, CRM_SYSTEM_MCP, NULL); +@@ -1303,7 +1303,7 @@ main(int argc, char **argv) + } else if (crm_ipc_connected(old_instance)) { + crm_ipc_close(old_instance); + crm_ipc_destroy(old_instance); +- crm_err("Pacemaker is already active, aborting startup"); ++ crm_err("Aborting start-up because active Pacemaker instance found"); + crm_exit(DAEMON_RESPAWN_STOP); + } + +-- +1.8.3.1 + + +From e32a3350ade774c1f10e397c994bfeb96e459a73 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 12 Aug 2019 13:41:43 -0500 +Subject: [PATCH 60/96] Refactor: controller: avoid memcpy() for two characters + +makes static analysis happy (which complained about copying 2 characters of a +3-character literal) +--- + crmd/lrm.c | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/crmd/lrm.c b/crmd/lrm.c +index 776c02b..437840f 100644 +--- a/crmd/lrm.c ++++ b/crmd/lrm.c +@@ -2480,7 +2480,11 @@ unescape_newlines(const char *string) + ret = strdup(string); + pch = strstr(ret, escaped_newline); + while (pch != NULL) { +- memcpy(pch, "\n ", 2); ++ /* Replace newline escape pattern with actual newline (and a space so we ++ * don't have to shuffle the rest of the buffer) ++ */ ++ pch[0] = '\n'; ++ pch[1] = ' '; + pch = strstr(pch, escaped_newline); + } + +-- +1.8.3.1 + + +From e47691249062f46c7e2719994871717e5d42fb8f Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 12 Aug 2019 19:40:31 -0500 +Subject: [PATCH 61/96] Refactor: libcrmcommon: use constant for all uses of + XML parse options + +Mainly so we can comment an issue with it. +--- + lib/common/xml.c | 31 ++++++++++++++++++------------- + 1 file changed, 18 insertions(+), 13 deletions(-) + +diff --git a/lib/common/xml.c b/lib/common/xml.c +index ae4cc6a..26cd78d 100644 +--- a/lib/common/xml.c ++++ b/lib/common/xml.c +@@ -44,6 +44,17 @@ + #define XML_BUFFER_SIZE 4096 + #define XML_PARSER_DEBUG 0 + ++/* @TODO XML_PARSE_RECOVER allows some XML errors to be silently worked around ++ * by libxml2, which is potentially ambiguous and dangerous. We should drop it ++ * when we can break backward compatibility with configurations that might be ++ * relying on it (i.e. pacemaker 3.0.0). ++ * ++ * It might be a good idea to have a transitional period where we first try ++ * parsing without XML_PARSE_RECOVER, and if that fails, try parsing again with ++ * it, logging a warning if it succeeds. ++ */ ++#define PCMK__XML_PARSE_OPTS (XML_PARSE_NOBLANKS | XML_PARSE_RECOVER) ++ + typedef struct { + int found; + const char *string; +@@ -2154,14 +2165,10 @@ string2xml(const char *input) + ctxt = xmlNewParserCtxt(); + CRM_CHECK(ctxt != NULL, return NULL); + +- /* xmlCtxtUseOptions(ctxt, XML_PARSE_NOBLANKS|XML_PARSE_RECOVER); */ +- + xmlCtxtResetLastError(ctxt); + xmlSetGenericErrorFunc(ctxt, crm_xml_err); +- /* initGenericErrorDefaultFunc(crm_xml_err); */ +- output = +- xmlCtxtReadDoc(ctxt, (const xmlChar *)input, NULL, NULL, +- XML_PARSE_NOBLANKS | XML_PARSE_RECOVER); ++ output = xmlCtxtReadDoc(ctxt, (const xmlChar *) input, NULL, NULL, ++ PCMK__XML_PARSE_OPTS); + if (output) { + xml = xmlDocGetRootElement(output); + } +@@ -2328,17 +2335,13 @@ filename2xml(const char *filename) + gboolean uncompressed = TRUE; + xmlParserCtxtPtr ctxt = NULL; + xmlErrorPtr last_error = NULL; +- static int xml_options = XML_PARSE_NOBLANKS | XML_PARSE_RECOVER; + + /* create a parser context */ + ctxt = xmlNewParserCtxt(); + CRM_CHECK(ctxt != NULL, return NULL); + +- /* xmlCtxtUseOptions(ctxt, XML_PARSE_NOBLANKS|XML_PARSE_RECOVER); */ +- + xmlCtxtResetLastError(ctxt); + xmlSetGenericErrorFunc(ctxt, crm_xml_err); +- /* initGenericErrorDefaultFunc(crm_xml_err); */ + + if (filename) { + uncompressed = !crm_ends_with_ext(filename, ".bz2"); +@@ -2346,15 +2349,17 @@ filename2xml(const char *filename) + + if (filename == NULL) { + /* STDIN_FILENO == fileno(stdin) */ +- output = xmlCtxtReadFd(ctxt, STDIN_FILENO, "unknown.xml", NULL, xml_options); ++ output = xmlCtxtReadFd(ctxt, STDIN_FILENO, "unknown.xml", NULL, ++ PCMK__XML_PARSE_OPTS); + + } else if (uncompressed) { +- output = xmlCtxtReadFile(ctxt, filename, NULL, xml_options); ++ output = xmlCtxtReadFile(ctxt, filename, NULL, PCMK__XML_PARSE_OPTS); + + } else { + char *input = decompress_file(filename); + +- output = xmlCtxtReadDoc(ctxt, (const xmlChar *)input, NULL, NULL, xml_options); ++ output = xmlCtxtReadDoc(ctxt, (const xmlChar *) input, NULL, NULL, ++ PCMK__XML_PARSE_OPTS); + free(input); + } + +-- +1.8.3.1 + + +From ff79b7755d6debd12329632e5a14f8dd1e827e96 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 13 Aug 2019 16:32:47 -0500 +Subject: [PATCH 62/96] Refactor: tools: avoid use-of-NULL false positives in + stonith_admin + +to make static analysis happy +--- + fencing/admin.c | 46 ++++++++++++++++++++++++++++++---------------- + 1 file changed, 30 insertions(+), 16 deletions(-) + +diff --git a/fencing/admin.c b/fencing/admin.c +index 7da43e9..5fedb7b 100644 +--- a/fencing/admin.c ++++ b/fencing/admin.c +@@ -312,9 +312,15 @@ handle_level(stonith_t *st, char *target, int fence_level, + char *node = NULL; + char *pattern = NULL; + char *name = NULL; +- char *value = strchr(target, '='); ++ char *value = NULL; ++ ++ if (target == NULL) { ++ // Not really possible, but makes static analysis happy ++ return -EINVAL; ++ } + + /* Determine if targeting by attribute, node name pattern or node name */ ++ value = strchr(target, '='); + if (value != NULL) { + name = target; + *value++ = '\0'; +@@ -453,6 +459,28 @@ validate(stonith_t *st, const char *agent, const char *id, + return rc; + } + ++static void ++show_last_fenced(int as_nodeid, const char *target) ++{ ++ time_t when = 0; ++ ++ if (target == NULL) { ++ // Not really possible, but makes static analysis happy ++ return; ++ } ++ if (as_nodeid) { ++ uint32_t nodeid = atol(target); ++ when = stonith_api_time(nodeid, NULL, FALSE); ++ } else { ++ when = stonith_api_time(0, target, FALSE); ++ } ++ if(when) { ++ printf("Node %s last kicked at: %s\n", target, ctime(&when)); ++ } else { ++ printf("Node %s has never been kicked\n", target); ++ } ++} ++ + int + main(int argc, char **argv) + { +@@ -741,21 +769,7 @@ main(int argc, char **argv) + rc = mainloop_fencing(st, target, "on", timeout, tolerance); + break; + case 'h': +- { +- time_t when = 0; +- +- if(as_nodeid) { +- uint32_t nodeid = atol(target); +- when = stonith_api_time(nodeid, NULL, FALSE); +- } else { +- when = stonith_api_time(0, target, FALSE); +- } +- if(when) { +- printf("Node %s last kicked at: %s\n", target, ctime(&when)); +- } else { +- printf("Node %s has never been kicked\n", target); +- } +- } ++ show_last_fenced(as_nodeid, target); + break; + case 'H': + rc = handle_history(st, target, timeout, quiet, +-- +1.8.3.1 + + +From 9fe5ad7d746ed60e304fa5420f434fa7750289ca Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 13 Aug 2019 16:59:13 -0500 +Subject: [PATCH 63/96] Low: libcrmcommon: handle pcmk_strerror(INT_MIN) + +not realistic, but makes static analysis happy +--- + lib/common/logging.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/lib/common/logging.c b/lib/common/logging.c +index b8b0f6f..c392468 100644 +--- a/lib/common/logging.c ++++ b/lib/common/logging.c +@@ -1149,7 +1149,9 @@ pcmk_strerror(int rc) + + if (error == 0) { + return "OK"; +- } else if (error < PCMK_ERROR_OFFSET) { ++ ++ // Of course error > 0 ... unless someone passed INT_MIN as rc ++ } else if ((error > 0) && (error < PCMK_ERROR_OFFSET)) { + return strerror(error); + } + +-- +1.8.3.1 + + +From 8201550e12973ace36684156d14262f34386cfb7 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 15 Aug 2019 15:33:18 -0500 +Subject: [PATCH 64/96] Build: GNUmakefile: improve coverity targets + +Most importantly, add the ability to specify the coverity aggressiveness level, +and put outputs in the build directory. Otherwise mostly refactoring for +best practices. + +The public coverity instance changed its upload process, so we can't do a +simple curl anymore. We haven't been using it anyway, so just echo what needs +to be done for that case. +--- + GNUmakefile | 95 ++++++++++++++++++++++++++++++++++++++++++------------------- + 1 file changed, 65 insertions(+), 30 deletions(-) + +diff --git a/GNUmakefile b/GNUmakefile +index a084150..0822890 100644 +--- a/GNUmakefile ++++ b/GNUmakefile +@@ -122,6 +122,11 @@ rpmbuild-with = \ + init: + ./autogen.sh init + ++.PHONY: init-if-needed ++init-if-needed: ++ test -e configure || ./autogen.sh ++ test -e Makefile || ./configure ++ + export: + rm -f $(PACKAGE)-dirty.tar.* $(PACKAGE)-tip.tar.* $(PACKAGE)-HEAD.tar.* + if [ ! -f $(TARFILE) ]; then \ +@@ -265,37 +270,67 @@ rc: + dirty: + make TAG=dirty mock + +-COVERITY_DIR = $(shell pwd)/coverity-$(TAG) +-COVFILE = $(PACKAGE)-coverity-$(TAG).tgz +-COVHOST ?= scan5.coverity.com +-COVPASS ?= password + +-# Public coverity +-coverity: +- test -e configure || ./autogen.sh +- test -e Makefile || ./configure +- make core-clean +- rm -rf $(COVERITY_DIR) +- cov-build --dir $(COVERITY_DIR) make core +- tar czf $(COVFILE) --transform=s@.*$(TAG)@cov-int@ $(COVERITY_DIR) +- @echo "Uploading to public Coverity instance..." +- curl --form file=@$(COVFILE) --form project=$(PACKAGE) --form password=$(COVPASS) --form email=andrew@beekhof.net http://$(COVHOST)/cgi-bin/upload.py +- rm -rf $(COVFILE) $(COVERITY_DIR) +- +-coverity-corp: +- test -e configure || ./autogen.sh +- test -e Makefile || ./configure +- make core-clean +- rm -rf $(COVERITY_DIR) +- cov-build --dir $(COVERITY_DIR) make core +- @echo "Waiting for a corporate Coverity license..." +- cov-analyze --dir $(COVERITY_DIR) --wait-for-license +- cov-format-errors --dir $(COVERITY_DIR) --emacs-style > $(TAG).coverity +- cov-format-errors --dir $(COVERITY_DIR) +- rsync $(RSYNC_OPTS) "$(COVERITY_DIR)/c/output/errors/" "$(RSYNC_DEST)/coverity/$(PACKAGE)/$(TAG)" +- make core-clean +-# cov-commit-defects --host $(COVHOST) --dir $(COVERITY_DIR) --stream $(PACKAGE) --user auto --password $(COVPASS) +- rm -rf $(COVERITY_DIR) ++## Static analysis via coverity ++ ++# Aggressiveness (low, medium, or high) ++COVLEVEL ?= low ++ ++# Generated outputs ++COVERITY_DIR = $(builddir)/coverity-$(TAG) ++COVTAR = $(builddir)/$(PACKAGE)-coverity-$(TAG).tgz ++COVEMACS = $(builddir)/$(TAG).coverity ++COVHTML = $(COVERITY_DIR)/output/errors ++ ++# Coverity outputs are phony so they get rebuilt every invocation ++ ++.PHONY: $(COVERITY_DIR) ++$(COVERITY_DIR): init-if-needed core-clean coverity-clean ++ $(AM_V_GEN)cov-build --dir "$@" $(MAKE) $(AM_MAKEFLAGS) core ++ ++.PHONY: $(COVTAR) ++$(COVTAR): $(COVERITY_DIR) ++ $(AM_V_GEN)tar czf "$@" --transform="s@.*$(TAG)@cov-int@" "$<" ++ ++# emacs/html output assume $(COVERITY_DIR) has been built (don't want rebuild) ++ ++.PHONY: $(COVEMACS) ++$(COVEMACS): ++ $(AM_V_GEN)cov-format-errors --dir "$(COVERITY_DIR)" --emacs-style > "$@" ++ ++.PHONY: $(COVHTML) ++$(COVHTML): ++ $(AM_V_GEN)cov-format-errors --dir "$(COVERITY_DIR)" --html-output "$@" ++ ++# Public coverity instance ++.PHONY: coverity ++coverity: $(COVTAR) ++ @echo "Now go to https://scan.coverity.com/users/sign_in and upload:" ++ @echo " $(COVTAR)" ++ @echo "then make core-clean coverity-clean" ++ ++# Licensed coverity instance ++ ++.PHONY: coverity-analyze ++coverity-analyze: $(COVERITY_DIR) ++ @echo "" ++ @echo "Analyzing (waiting for coverity license if necessary) ..." ++ cov-analyze --dir "$<" --wait-for-license --security \ ++ --aggressiveness-level "$(COVLEVEL)" ++ ++.PHONY: coverity-corp ++coverity-corp: coverity-analyze $(COVEMACS) $(COVHTML) core-clean ++ @echo "Done. See:" ++ @echo " file://$(abs_builddir)/$(COVERITY_DIR)/output/errors/index.html" ++ @echo "When no longer needed, make coverity-clean" ++ ++# Remove all outputs regardless of tag ++.PHONY: coverity-clean ++coverity-clean: ++ -rm -rf "$(builddir)"/coverity-* \ ++ "$(builddir)"/$(PACKAGE)-coverity-*.tgz \ ++ "$(builddir)"/*.coverity ++ + + global: clean-generic + gtags -q +-- +1.8.3.1 + + +From 6659ba3cf69f95e0cd724b55a29c926010b68df8 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 15 Aug 2019 15:50:46 -0500 +Subject: [PATCH 66/96] Build: GNUmakefile: don't depend on prerequisite order + for coverity target + +--- + GNUmakefile | 30 +++++++++++++++++------------- + 1 file changed, 17 insertions(+), 13 deletions(-) + +diff --git a/GNUmakefile b/GNUmakefile +index 0822890..b2d5a28 100644 +--- a/GNUmakefile ++++ b/GNUmakefile +@@ -288,21 +288,12 @@ COVHTML = $(COVERITY_DIR)/output/errors + $(COVERITY_DIR): init-if-needed core-clean coverity-clean + $(AM_V_GEN)cov-build --dir "$@" $(MAKE) $(AM_MAKEFLAGS) core + ++# Public coverity instance ++ + .PHONY: $(COVTAR) + $(COVTAR): $(COVERITY_DIR) + $(AM_V_GEN)tar czf "$@" --transform="s@.*$(TAG)@cov-int@" "$<" + +-# emacs/html output assume $(COVERITY_DIR) has been built (don't want rebuild) +- +-.PHONY: $(COVEMACS) +-$(COVEMACS): +- $(AM_V_GEN)cov-format-errors --dir "$(COVERITY_DIR)" --emacs-style > "$@" +- +-.PHONY: $(COVHTML) +-$(COVHTML): +- $(AM_V_GEN)cov-format-errors --dir "$(COVERITY_DIR)" --html-output "$@" +- +-# Public coverity instance + .PHONY: coverity + coverity: $(COVTAR) + @echo "Now go to https://scan.coverity.com/users/sign_in and upload:" +@@ -310,6 +301,10 @@ coverity: $(COVTAR) + @echo "then make core-clean coverity-clean" + + # Licensed coverity instance ++# ++# The prerequisites are a little hacky; rather than actually required, some ++# of them are designed so that things execute in the proper order (which is ++# not the same as GNU make's order-only prerequisites). + + .PHONY: coverity-analyze + coverity-analyze: $(COVERITY_DIR) +@@ -318,10 +313,19 @@ coverity-analyze: $(COVERITY_DIR) + cov-analyze --dir "$<" --wait-for-license --security \ + --aggressiveness-level "$(COVLEVEL)" + ++.PHONY: $(COVEMACS) ++$(COVEMACS): coverity-analyze ++ $(AM_V_GEN)cov-format-errors --dir "$(COVERITY_DIR)" --emacs-style > "$@" ++ ++.PHONY: $(COVHTML) ++$(COVHTML): $(COVEMACS) ++ $(AM_V_GEN)cov-format-errors --dir "$(COVERITY_DIR)" --html-output "$@" ++ + .PHONY: coverity-corp +-coverity-corp: coverity-analyze $(COVEMACS) $(COVHTML) core-clean ++coverity-corp: $(COVHTML) ++ $(MAKE) $(AM_MAKEFLAGS) core-clean + @echo "Done. See:" +- @echo " file://$(abs_builddir)/$(COVERITY_DIR)/output/errors/index.html" ++ @echo " file://$(abs_builddir)/$(COVHTML)/index.html" + @echo "When no longer needed, make coverity-clean" + + # Remove all outputs regardless of tag +-- +1.8.3.1 + + +From 0158f59b11b3e14f86258265a491f81d83cc6fcf Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 15 Aug 2019 17:38:41 -0500 +Subject: [PATCH 67/96] Fix: all: restrict XML children loops to XML elements + where appropriate + +__xml_first_child() and __xml_next() are intended to be used in "for" loops +where all XML node types (elements, comments, etc.) are desired. + +__xml_first_child_element() and __xml_next_element() are intended when +only element children are desired. + +Previously, many element-only loops properly used __xml_next_element() but +started with __xml_first_child(). In most cases, this would (by lucky +circumstance) work without harm. However, there were cases where a comment as +the first child of an element would case problems (for example, +unpack_resources() would log a configuration error). + +Now, __xml_first_child_element() is always used in such cases. + +Additionally, there were some loops using __xml_first_child()/__xml_next() that +clearly were expecting only elements. These have been converted to +__xml_first_child_element()/__xml_next_element(). + +Many more cases exist where __xml_first_child()/__xml_next() is used with IPC +messages and patchsets. This commit does not convert those, though that would +probably be a good idea for the future. +--- + lib/common/acl.c | 12 ++--- + lib/common/xml.c | 7 +-- + lib/pengine/clone.c | 2 +- + lib/pengine/complex.c | 10 ++-- + lib/pengine/group.c | 6 ++- + lib/pengine/rules.c | 51 ++++++++++--------- + lib/pengine/rules_alerts.c | 4 +- + lib/pengine/unpack.c | 61 ++++++++++++++++------- + lib/pengine/utils.c | 14 ++++-- + pengine/allocate.c | 10 ++-- + pengine/constraints.c | 120 +++++++++++++++++++++++++++++++++------------ + pengine/native.c | 11 +++-- + tools/crm_mon.c | 19 ++++--- + tools/crm_resource_print.c | 4 +- + tools/crmadmin.c | 4 +- + tools/fake_transition.c | 4 +- + 16 files changed, 226 insertions(+), 113 deletions(-) + +diff --git a/lib/common/acl.c b/lib/common/acl.c +index 72dc707..b5c20bb 100644 +--- a/lib/common/acl.c ++++ b/lib/common/acl.c +@@ -131,8 +131,8 @@ __xml_acl_parse_entry(xmlNode *acl_top, xmlNode *acl_entry, GList *acls) + { + xmlNode *child = NULL; + +- for (child = __xml_first_child(acl_entry); child; +- child = __xml_next(child)) { ++ for (child = __xml_first_child_element(acl_entry); child; ++ child = __xml_next_element(child)) { + const char *tag = crm_element_name(child); + const char *kind = crm_element_value(child, XML_ACL_ATTR_KIND); + +@@ -151,8 +151,8 @@ __xml_acl_parse_entry(xmlNode *acl_top, xmlNode *acl_entry, GList *acls) + if (ref_role) { + xmlNode *role = NULL; + +- for (role = __xml_first_child(acl_top); role; +- role = __xml_next(role)) { ++ for (role = __xml_first_child_element(acl_top); role; ++ role = __xml_next_element(role)) { + if (!strcmp(XML_ACL_TAG_ROLE, (const char *) role->name)) { + const char *role_id = crm_element_value(role, + XML_ATTR_ID); +@@ -306,8 +306,8 @@ pcmk__unpack_acl(xmlNode *source, xmlNode *target, const char *user) + if (acls) { + xmlNode *child = NULL; + +- for (child = __xml_first_child(acls); child; +- child = __xml_next(child)) { ++ for (child = __xml_first_child_element(acls); child; ++ child = __xml_next_element(child)) { + const char *tag = crm_element_name(child); + + if (!strcmp(tag, XML_ACL_TAG_USER) +diff --git a/lib/common/xml.c b/lib/common/xml.c +index 26cd78d..2c4238d 100644 +--- a/lib/common/xml.c ++++ b/lib/common/xml.c +@@ -4242,7 +4242,8 @@ first_named_child(xmlNode * parent, const char *name) + { + xmlNode *match = NULL; + +- for (match = __xml_first_child(parent); match != NULL; match = __xml_next(match)) { ++ for (match = __xml_first_child_element(parent); match != NULL; ++ match = __xml_next_element(match)) { + /* + * name == NULL gives first child regardless of name; this is + * semantically incorrect in this function, but may be necessary +@@ -4265,14 +4266,14 @@ first_named_child(xmlNode * parent, const char *name) + xmlNode * + crm_next_same_xml(xmlNode *sibling) + { +- xmlNode *match = __xml_next(sibling); ++ xmlNode *match = __xml_next_element(sibling); + const char *name = crm_element_name(sibling); + + while (match != NULL) { + if (!strcmp(crm_element_name(match), name)) { + return match; + } +- match = __xml_next(match); ++ match = __xml_next_element(match); + } + return NULL; + } +diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c +index 88015a9..07a383c 100644 +--- a/lib/pengine/clone.c ++++ b/lib/pengine/clone.c +@@ -172,7 +172,7 @@ clone_unpack(resource_t * rsc, pe_working_set_t * data_set) + is_set(rsc->flags, pe_rsc_unique) ? "true" : "false"); + + // Clones may contain a single group or primitive +- for (a_child = __xml_first_child(xml_obj); a_child != NULL; ++ for (a_child = __xml_first_child_element(xml_obj); a_child != NULL; + a_child = __xml_next_element(a_child)) { + + if (crm_str_eq((const char *)a_child->name, XML_CIB_TAG_RESOURCE, TRUE) +diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c +index cdd409a..42492c9 100644 +--- a/lib/pengine/complex.c ++++ b/lib/pengine/complex.c +@@ -282,7 +282,7 @@ unpack_template(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * d + + template_ops = find_xml_node(new_xml, "operations", FALSE); + +- for (child_xml = __xml_first_child(xml_obj); child_xml != NULL; ++ for (child_xml = __xml_first_child_element(xml_obj); child_xml != NULL; + child_xml = __xml_next_element(child_xml)) { + xmlNode *new_child = NULL; + +@@ -298,13 +298,17 @@ unpack_template(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * d + GHashTable *rsc_ops_hash = + g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, NULL); + +- for (op = __xml_first_child(rsc_ops); op != NULL; op = __xml_next_element(op)) { ++ for (op = __xml_first_child_element(rsc_ops); op != NULL; ++ op = __xml_next_element(op)) { ++ + char *key = template_op_key(op); + + g_hash_table_insert(rsc_ops_hash, key, op); + } + +- for (op = __xml_first_child(template_ops); op != NULL; op = __xml_next_element(op)) { ++ for (op = __xml_first_child_element(template_ops); op != NULL; ++ op = __xml_next_element(op)) { ++ + char *key = template_op_key(op); + + if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) { +diff --git a/lib/pengine/group.c b/lib/pengine/group.c +index 258c6b5..72f066e 100644 +--- a/lib/pengine/group.c ++++ b/lib/pengine/group.c +@@ -1,5 +1,7 @@ + /* +- * Copyright 2004-2018 Andrew Beekhof ++ * Copyright 2004-2019 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. +@@ -46,7 +48,7 @@ group_unpack(resource_t * rsc, pe_working_set_t * data_set) + + clone_id = crm_element_value(rsc->xml, XML_RSC_ATTR_INCARNATION); + +- for (xml_native_rsc = __xml_first_child(xml_obj); xml_native_rsc != NULL; ++ for (xml_native_rsc = __xml_first_child_element(xml_obj); xml_native_rsc != NULL; + xml_native_rsc = __xml_next_element(xml_native_rsc)) { + if (crm_str_eq((const char *)xml_native_rsc->name, XML_CIB_TAG_RESOURCE, TRUE)) { + resource_t *new_rsc = NULL; +diff --git a/lib/pengine/rules.c b/lib/pengine/rules.c +index 2d948b9..523ed15 100644 +--- a/lib/pengine/rules.c ++++ b/lib/pengine/rules.c +@@ -1,19 +1,10 @@ +-/* +- * Copyright (C) 2004 Andrew Beekhof +- * +- * This library is free software; you can redistribute it and/or +- * modify it under the terms of the GNU Lesser General Public +- * License as published by the Free Software Foundation; either +- * version 2.1 of the License, or (at your option) any later version. +- * +- * This library is distributed in the hope that it will be useful, +- * but WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * Lesser General Public License for more details. +- * +- * You should have received a copy of the GNU Lesser General Public +- * License along with this library; if not, write to the Free Software +- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA ++/* ++ * Copyright 2004-2019 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. ++ * ++ * This source code is licensed under the GNU Lesser General Public License ++ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ + + #include +@@ -46,7 +37,9 @@ test_ruleset(xmlNode * ruleset, GHashTable * node_hash, crm_time_t * now) + gboolean ruleset_default = TRUE; + xmlNode *rule = NULL; + +- for (rule = __xml_first_child(ruleset); rule != NULL; rule = __xml_next_element(rule)) { ++ for (rule = __xml_first_child_element(ruleset); rule != NULL; ++ rule = __xml_next_element(rule)) { ++ + if (crm_str_eq((const char *)rule->name, XML_TAG_RULE, TRUE)) { + ruleset_default = FALSE; + if (test_rule(rule, node_hash, RSC_ROLE_UNKNOWN, now)) { +@@ -93,7 +86,9 @@ pe_test_rule_full(xmlNode * rule, GHashTable * node_hash, enum rsc_role_e role, + } + + crm_trace("Testing rule %s", ID(rule)); +- for (expr = __xml_first_child(rule); expr != NULL; expr = __xml_next_element(expr)) { ++ for (expr = __xml_first_child_element(rule); expr != NULL; ++ expr = __xml_next_element(expr)) { ++ + test = pe_test_expression_full(expr, node_hash, role, now, match_data); + empty = FALSE; + +@@ -722,7 +717,9 @@ populate_hash(xmlNode * nvpair_list, GHashTable * hash, gboolean overwrite, xmlN + list = list->children; + } + +- for (an_attr = __xml_first_child(list); an_attr != NULL; an_attr = __xml_next_element(an_attr)) { ++ for (an_attr = __xml_first_child_element(list); an_attr != NULL; ++ an_attr = __xml_next_element(an_attr)) { ++ + if (crm_str_eq((const char *)an_attr->name, XML_CIB_TAG_NVPAIR, TRUE)) { + xmlNode *ref_nvpair = expand_idref(an_attr, top); + +@@ -769,9 +766,13 @@ get_versioned_rule(xmlNode * attr_set) + xmlNode * rule = NULL; + xmlNode * expr = NULL; + +- for (rule = __xml_first_child(attr_set); rule != NULL; rule = __xml_next_element(rule)) { ++ for (rule = __xml_first_child_element(attr_set); rule != NULL; ++ rule = __xml_next_element(rule)) { ++ + if (crm_str_eq((const char *)rule->name, XML_TAG_RULE, TRUE)) { +- for (expr = __xml_first_child(rule); expr != NULL; expr = __xml_next_element(expr)) { ++ for (expr = __xml_first_child_element(rule); expr != NULL; ++ expr = __xml_next_element(expr)) { ++ + if (find_expression_type(expr) == version_expr) { + return rule; + } +@@ -801,7 +802,7 @@ add_versioned_attributes(xmlNode * attr_set, xmlNode * versioned_attrs) + return; + } + +- expr = __xml_first_child(rule); ++ expr = __xml_first_child_element(rule); + while (expr != NULL) { + if (find_expression_type(expr) != version_expr) { + xmlNode *node = expr; +@@ -879,7 +880,9 @@ make_pairs_and_populate_data(xmlNode * top, xmlNode * xml_obj, const char *set_n + } + + crm_trace("Checking for attributes"); +- for (attr_set = __xml_first_child(xml_obj); attr_set != NULL; attr_set = __xml_next_element(attr_set)) { ++ for (attr_set = __xml_first_child_element(xml_obj); attr_set != NULL; ++ attr_set = __xml_next_element(attr_set)) { ++ + /* Uncertain if set_name == NULL check is strictly necessary here */ + if (set_name == NULL || crm_str_eq((const char *)attr_set->name, set_name, TRUE)) { + pair = NULL; +@@ -1011,7 +1014,7 @@ pe_unpack_versioned_parameters(xmlNode *versioned_params, const char *ra_version + + if (versioned_params && ra_version) { + GHashTable *node_hash = crm_str_table_new(); +- xmlNode *attr_set = __xml_first_child(versioned_params); ++ xmlNode *attr_set = __xml_first_child_element(versioned_params); + + if (attr_set) { + g_hash_table_insert(node_hash, strdup(CRM_ATTR_RA_VERSION), +diff --git a/lib/pengine/rules_alerts.c b/lib/pengine/rules_alerts.c +index e2bab58..00be330 100644 +--- a/lib/pengine/rules_alerts.c ++++ b/lib/pengine/rules_alerts.c +@@ -127,8 +127,8 @@ unpack_alert_filter(xmlNode *basenode, crm_alert_entry_t *entry) + xmlNode *event_type = NULL; + uint32_t flags = crm_alert_none; + +- for (event_type = __xml_first_child(select); event_type != NULL; +- event_type = __xml_next(event_type)) { ++ for (event_type = __xml_first_child_element(select); event_type != NULL; ++ event_type = __xml_next_element(event_type)) { + + const char *tagname = crm_element_name(event_type); + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index cf725a1..4282a7c 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -430,7 +430,9 @@ remote_id_conflict(const char *remote_name, pe_working_set_t *data) + #else + if (data->name_check == NULL) { + data->name_check = g_hash_table_new(crm_str_hash, g_str_equal); +- for (xml_rsc = __xml_first_child(parent); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(parent); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + const char *id = ID(xml_rsc); + + /* avoiding heap allocation here because we know the duration of this hashtable allows us to */ +@@ -464,12 +466,14 @@ expand_remote_rsc_meta(xmlNode *xml_obj, xmlNode *parent, pe_working_set_t *data + const char *remote_allow_migrate=NULL; + const char *container_managed = NULL; + +- for (attr_set = __xml_first_child(xml_obj); attr_set != NULL; attr_set = __xml_next_element(attr_set)) { ++ for (attr_set = __xml_first_child_element(xml_obj); attr_set != NULL; ++ attr_set = __xml_next_element(attr_set)) { + if (safe_str_neq((const char *)attr_set->name, XML_TAG_META_SETS)) { + continue; + } + +- for (attr = __xml_first_child(attr_set); attr != NULL; attr = __xml_next_element(attr)) { ++ for (attr = __xml_first_child_element(attr_set); attr != NULL; ++ attr = __xml_next_element(attr)) { + const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE); + const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME); + +@@ -538,7 +542,9 @@ unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set) + const char *type = NULL; + const char *score = NULL; + +- for (xml_obj = __xml_first_child(xml_nodes); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) { ++ for (xml_obj = __xml_first_child_element(xml_nodes); xml_obj != NULL; ++ xml_obj = __xml_next_element(xml_obj)) { ++ + if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, TRUE)) { + new_node = NULL; + +@@ -620,7 +626,9 @@ unpack_remote_nodes(xmlNode * xml_resources, pe_working_set_t * data_set) + xmlNode *xml_obj = NULL; + + /* generate remote nodes from resource config before unpacking resources */ +- for (xml_obj = __xml_first_child(xml_resources); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) { ++ for (xml_obj = __xml_first_child_element(xml_resources); xml_obj != NULL; ++ xml_obj = __xml_next_element(xml_obj)) { ++ + const char *new_node_id = NULL; + + /* first check if this is a bare metal remote node. Bare metal remote nodes +@@ -659,7 +667,8 @@ unpack_remote_nodes(xmlNode * xml_resources, pe_working_set_t * data_set) + } else if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, TRUE)) { + xmlNode *xml_obj2 = NULL; + /* search through a group to see if any of the primitive contain a remote node. */ +- for (xml_obj2 = __xml_first_child(xml_obj); xml_obj2 != NULL; xml_obj2 = __xml_next_element(xml_obj2)) { ++ for (xml_obj2 = __xml_first_child_element(xml_obj); xml_obj2 != NULL; ++ xml_obj2 = __xml_next_element(xml_obj2)) { + + new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources, data_set); + +@@ -750,7 +759,9 @@ unpack_resources(xmlNode * xml_resources, pe_working_set_t * data_set) + g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, + destroy_tag); + +- for (xml_obj = __xml_first_child(xml_resources); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) { ++ for (xml_obj = __xml_first_child_element(xml_resources); xml_obj != NULL; ++ xml_obj = __xml_next_element(xml_obj)) { ++ + resource_t *new_rsc = NULL; + + if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE, TRUE)) { +@@ -808,7 +819,9 @@ unpack_tags(xmlNode * xml_tags, pe_working_set_t * data_set) + data_set->tags = + g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_tag); + +- for (xml_tag = __xml_first_child(xml_tags); xml_tag != NULL; xml_tag = __xml_next_element(xml_tag)) { ++ for (xml_tag = __xml_first_child_element(xml_tags); xml_tag != NULL; ++ xml_tag = __xml_next_element(xml_tag)) { ++ + xmlNode *xml_obj_ref = NULL; + const char *tag_id = ID(xml_tag); + +@@ -822,7 +835,9 @@ unpack_tags(xmlNode * xml_tags, pe_working_set_t * data_set) + continue; + } + +- for (xml_obj_ref = __xml_first_child(xml_tag); xml_obj_ref != NULL; xml_obj_ref = __xml_next_element(xml_obj_ref)) { ++ for (xml_obj_ref = __xml_first_child_element(xml_tag); xml_obj_ref != NULL; ++ xml_obj_ref = __xml_next_element(xml_obj_ref)) { ++ + const char *obj_ref = ID(xml_obj_ref); + + if (crm_str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, TRUE) == FALSE) { +@@ -916,7 +931,9 @@ unpack_tickets_state(xmlNode * xml_tickets, pe_working_set_t * data_set) + { + xmlNode *xml_obj = NULL; + +- for (xml_obj = __xml_first_child(xml_tickets); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) { ++ for (xml_obj = __xml_first_child_element(xml_tickets); xml_obj != NULL; ++ xml_obj = __xml_next_element(xml_obj)) { ++ + if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, TRUE) == FALSE) { + continue; + } +@@ -1075,7 +1092,9 @@ unpack_node_loop(xmlNode * status, bool fence, pe_working_set_t * data_set) + bool changed = false; + xmlNode *lrm_rsc = NULL; + +- for (xmlNode *state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) { ++ for (xmlNode *state = __xml_first_child_element(status); state != NULL; ++ state = __xml_next_element(state)) { ++ + const char *id = NULL; + const char *uname = NULL; + node_t *this_node = NULL; +@@ -1174,7 +1193,9 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set) + g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_ticket); + } + +- for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) { ++ for (state = __xml_first_child_element(status); state != NULL; ++ state = __xml_next_element(state)) { ++ + if (crm_str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, TRUE)) { + xmlNode *xml_tickets = state; + GHashTable *state_hash = NULL; +@@ -2285,7 +2306,8 @@ unpack_lrm_rsc_state(node_t * node, xmlNode * rsc_entry, pe_working_set_t * data + op_list = NULL; + sorted_op_list = NULL; + +- for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) { ++ for (rsc_op = __xml_first_child_element(rsc_entry); rsc_op != NULL; ++ rsc_op = __xml_next_element(rsc_op)) { + if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) { + op_list = g_list_prepend(op_list, rsc_op); + } +@@ -2354,8 +2376,8 @@ static void + handle_orphaned_container_fillers(xmlNode * lrm_rsc_list, pe_working_set_t * data_set) + { + xmlNode *rsc_entry = NULL; +- for (rsc_entry = __xml_first_child(lrm_rsc_list); rsc_entry != NULL; +- rsc_entry = __xml_next_element(rsc_entry)) { ++ for (rsc_entry = __xml_first_child_element(lrm_rsc_list); rsc_entry != NULL; ++ rsc_entry = __xml_next_element(rsc_entry)) { + + resource_t *rsc; + resource_t *container; +@@ -2400,7 +2422,7 @@ unpack_lrm_resources(node_t * node, xmlNode * lrm_rsc_list, pe_working_set_t * d + + crm_trace("Unpacking resources on %s", node->details->uname); + +- for (rsc_entry = __xml_first_child(lrm_rsc_list); rsc_entry != NULL; ++ for (rsc_entry = __xml_first_child_element(lrm_rsc_list); rsc_entry != NULL; + rsc_entry = __xml_next_element(rsc_entry)) { + + if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) { +@@ -3490,7 +3512,8 @@ extract_operations(const char *node, const char *rsc, xmlNode * rsc_entry, gbool + op_list = NULL; + sorted_op_list = NULL; + +- for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) { ++ for (rsc_op = __xml_first_child_element(rsc_entry); ++ rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) { + if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) { + crm_xml_add(rsc_op, "resource", rsc); + crm_xml_add(rsc_op, XML_ATTR_UNAME, node); +@@ -3548,7 +3571,7 @@ find_operations(const char *rsc, const char *node, gboolean active_filter, + + xmlNode *node_state = NULL; + +- for (node_state = __xml_first_child(status); node_state != NULL; ++ for (node_state = __xml_first_child_element(status); node_state != NULL; + node_state = __xml_next_element(node_state)) { + + if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) { +@@ -3580,7 +3603,7 @@ find_operations(const char *rsc, const char *node, gboolean active_filter, + tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE); + tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE); + +- for (lrm_rsc = __xml_first_child(tmp); lrm_rsc != NULL; ++ for (lrm_rsc = __xml_first_child_element(tmp); lrm_rsc != NULL; + lrm_rsc = __xml_next_element(lrm_rsc)) { + if (crm_str_eq((const char *)lrm_rsc->name, XML_LRM_TAG_RESOURCE, TRUE)) { + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 8eac2ce..67c1c3d 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -646,7 +646,7 @@ unpack_operation_on_fail(action_t * action) + + CRM_CHECK(action->rsc != NULL, return NULL); + +- for (operation = __xml_first_child(action->rsc->ops_xml); ++ for (operation = __xml_first_child_element(action->rsc->ops_xml); + operation && !value; operation = __xml_next_element(operation)) { + + if (!crm_str_eq((const char *)operation->name, "op", TRUE)) { +@@ -685,7 +685,7 @@ find_min_interval_mon(resource_t * rsc, gboolean include_disabled) + xmlNode *op = NULL; + xmlNode *operation = NULL; + +- for (operation = __xml_first_child(rsc->ops_xml); operation != NULL; ++ for (operation = __xml_first_child_element(rsc->ops_xml); operation != NULL; + operation = __xml_next_element(operation)) { + + if (crm_str_eq((const char *)operation->name, "op", TRUE)) { +@@ -873,8 +873,12 @@ unpack_versioned_meta(xmlNode *versioned_meta, xmlNode *xml_obj, unsigned long l + xmlNode *attrs = NULL; + xmlNode *attr = NULL; + +- for (attrs = __xml_first_child(versioned_meta); attrs != NULL; attrs = __xml_next_element(attrs)) { +- for (attr = __xml_first_child(attrs); attr != NULL; attr = __xml_next_element(attr)) { ++ for (attrs = __xml_first_child_element(versioned_meta); attrs != NULL; ++ attrs = __xml_next_element(attrs)) { ++ ++ for (attr = __xml_first_child_element(attrs); attr != NULL; ++ attr = __xml_next_element(attr)) { ++ + const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME); + const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE); + +@@ -1233,7 +1237,7 @@ find_rsc_op_entry_helper(resource_t * rsc, const char *key, gboolean include_dis + xmlNode *operation = NULL; + + retry: +- for (operation = __xml_first_child(rsc->ops_xml); operation != NULL; ++ for (operation = __xml_first_child_element(rsc->ops_xml); operation != NULL; + operation = __xml_next_element(operation)) { + if (crm_str_eq((const char *)operation->name, "op", TRUE)) { + name = crm_element_value(operation, "name"); +diff --git a/pengine/allocate.c b/pengine/allocate.c +index d600bbf..28f58b3 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -465,7 +465,9 @@ check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_worki + DeleteRsc(rsc, node, FALSE, data_set); + } + +- for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) { ++ for (rsc_op = __xml_first_child_element(rsc_entry); rsc_op != NULL; ++ rsc_op = __xml_next_element(rsc_op)) { ++ + if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) { + op_list = g_list_prepend(op_list, rsc_op); + } +@@ -596,7 +598,7 @@ check_actions(pe_working_set_t * data_set) + + xmlNode *node_state = NULL; + +- for (node_state = __xml_first_child(status); node_state != NULL; ++ for (node_state = __xml_first_child_element(status); node_state != NULL; + node_state = __xml_next_element(node_state)) { + if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) { + id = crm_element_value(node_state, XML_ATTR_ID); +@@ -619,8 +621,10 @@ check_actions(pe_working_set_t * data_set) + if (node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) { + xmlNode *rsc_entry = NULL; + +- for (rsc_entry = __xml_first_child(lrm_rscs); rsc_entry != NULL; ++ for (rsc_entry = __xml_first_child_element(lrm_rscs); ++ rsc_entry != NULL; + rsc_entry = __xml_next_element(rsc_entry)) { ++ + if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) { + + if (xml_has_children(rsc_entry)) { +diff --git a/pengine/constraints.c b/pengine/constraints.c +index 6bf5adf..776eee1 100644 +--- a/pengine/constraints.c ++++ b/pengine/constraints.c +@@ -1,5 +1,7 @@ + /* +- * Copyright 2004-2018 Andrew Beekhof ++ * Copyright 2004-2019 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. +@@ -53,7 +55,7 @@ unpack_constraints(xmlNode * xml_constraints, pe_working_set_t * data_set) + xmlNode *xml_obj = NULL; + xmlNode *lifetime = NULL; + +- for (xml_obj = __xml_first_child(xml_constraints); xml_obj != NULL; ++ for (xml_obj = __xml_first_child_element(xml_constraints); xml_obj != NULL; + xml_obj = __xml_next_element(xml_obj)) { + const char *id = crm_element_value(xml_obj, XML_ATTR_ID); + const char *tag = crm_element_name(xml_obj); +@@ -489,7 +491,9 @@ expand_tags_in_sets(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t + new_xml = copy_xml(xml_obj); + cons_id = ID(new_xml); + +- for (set = __xml_first_child(new_xml); set != NULL; set = __xml_next_element(set)) { ++ for (set = __xml_first_child_element(new_xml); set != NULL; ++ set = __xml_next_element(set)) { ++ + xmlNode *xml_rsc = NULL; + GListPtr tag_refs = NULL; + GListPtr gIter = NULL; +@@ -498,7 +502,9 @@ expand_tags_in_sets(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t + continue; + } + +- for (xml_rsc = __xml_first_child(set); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + resource_t *rsc = NULL; + tag_t *tag = NULL; + const char *id = ID(xml_rsc); +@@ -783,7 +789,7 @@ unpack_rsc_location(xmlNode * xml_obj, resource_t * rsc_lh, const char * role, + } else { + xmlNode *rule_xml = NULL; + +- for (rule_xml = __xml_first_child(xml_obj); rule_xml != NULL; ++ for (rule_xml = __xml_first_child_element(xml_obj); rule_xml != NULL; + rule_xml = __xml_next_element(rule_xml)) { + if (crm_str_eq((const char *)rule_xml->name, XML_TAG_RULE, TRUE)) { + empty = FALSE; +@@ -927,7 +933,9 @@ unpack_location_set(xmlNode * location, xmlNode * set, pe_working_set_t * data_s + role = crm_element_value(set, "role"); + local_score = crm_element_value(set, XML_RULE_ATTR_SCORE); + +- for (xml_rsc = __xml_first_child(set); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc)); + unpack_rsc_location(location, resource, role, local_score, data_set, NULL); +@@ -955,7 +963,9 @@ unpack_location(xmlNode * xml_obj, pe_working_set_t * data_set) + xml_obj = expanded_xml; + } + +- for (set = __xml_first_child(xml_obj); set != NULL; set = __xml_next_element(set)) { ++ for (set = __xml_first_child_element(xml_obj); set != NULL; ++ set = __xml_next_element(set)) { ++ + if (crm_str_eq((const char *)set->name, XML_CONS_TAG_RSC_SET, TRUE)) { + any_sets = TRUE; + set = expand_idref(set, data_set->input); +@@ -1628,7 +1638,9 @@ unpack_order_set(xmlNode * set, enum pe_order_kind parent_kind, resource_t ** rs + flags = get_asymmetrical_flags(local_kind); + } + +- for (xml_rsc = __xml_first_child(set); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + EXPAND_CONSTRAINT_IDREF(id, resource, ID(xml_rsc)); + resources = g_list_append(resources, resource); +@@ -1805,7 +1817,9 @@ order_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, enum pe_order_kin + free(task); + update_action_flags(unordered_action, pe_action_requires_any, __FUNCTION__, __LINE__); + +- for (xml_rsc = __xml_first_child(set1); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (!crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + continue; + } +@@ -1818,7 +1832,9 @@ order_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, enum pe_order_kin + NULL, NULL, unordered_action, + pe_order_one_or_more | pe_order_implies_then_printed, data_set); + } +- for (xml_rsc_2 = __xml_first_child(set2); xml_rsc_2 != NULL; xml_rsc_2 = __xml_next_element(xml_rsc_2)) { ++ for (xml_rsc_2 = __xml_first_child_element(set2); xml_rsc_2 != NULL; ++ xml_rsc_2 = __xml_next_element(xml_rsc_2)) { ++ + if (!crm_str_eq((const char *)xml_rsc_2->name, XML_TAG_RESOURCE_REF, TRUE)) { + continue; + } +@@ -1840,7 +1856,9 @@ order_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, enum pe_order_kin + /* get the last one */ + const char *rid = NULL; + +- for (xml_rsc = __xml_first_child(set1); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + rid = ID(xml_rsc); + } +@@ -1849,7 +1867,9 @@ order_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, enum pe_order_kin + + } else { + /* get the first one */ +- for (xml_rsc = __xml_first_child(set1); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); + break; +@@ -1861,7 +1881,9 @@ order_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, enum pe_order_kin + if (crm_is_true(sequential_2)) { + if (invert == FALSE) { + /* get the first one */ +- for (xml_rsc = __xml_first_child(set2); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set2); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc)); + break; +@@ -1872,7 +1894,9 @@ order_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, enum pe_order_kin + /* get the last one */ + const char *rid = NULL; + +- for (xml_rsc = __xml_first_child(set2); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set2); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + rid = ID(xml_rsc); + } +@@ -1885,7 +1909,9 @@ order_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, enum pe_order_kin + new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set); + + } else if (rsc_1 != NULL) { +- for (xml_rsc = __xml_first_child(set2); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set2); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc)); + new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set); +@@ -1895,7 +1921,9 @@ order_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, enum pe_order_kin + } else if (rsc_2 != NULL) { + xmlNode *xml_rsc = NULL; + +- for (xml_rsc = __xml_first_child(set1); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); + new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set); +@@ -1903,14 +1931,18 @@ order_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, enum pe_order_kin + } + + } else { +- for (xml_rsc = __xml_first_child(set1); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + xmlNode *xml_rsc_2 = NULL; + + EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); + +- for (xml_rsc_2 = __xml_first_child(set2); xml_rsc_2 != NULL; ++ for (xml_rsc_2 = __xml_first_child_element(set2); ++ xml_rsc_2 != NULL; + xml_rsc_2 = __xml_next_element(xml_rsc_2)) { ++ + if (crm_str_eq((const char *)xml_rsc_2->name, XML_TAG_RESOURCE_REF, TRUE)) { + EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2)); + new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set); +@@ -2078,7 +2110,9 @@ unpack_rsc_order(xmlNode * xml_obj, pe_working_set_t * data_set) + return FALSE; + } + +- for (set = __xml_first_child(xml_obj); set != NULL; set = __xml_next_element(set)) { ++ for (set = __xml_first_child_element(xml_obj); set != NULL; ++ set = __xml_next_element(set)) { ++ + if (crm_str_eq((const char *)set->name, XML_CONS_TAG_RSC_SET, TRUE)) { + any_sets = TRUE; + set = expand_idref(set, data_set->input); +@@ -2183,7 +2217,9 @@ unpack_colocation_set(xmlNode * set, int score, pe_working_set_t * data_set) + return TRUE; + + } else if (local_score >= 0 && safe_str_eq(ordering, "group")) { +- for (xml_rsc = __xml_first_child(set); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc)); + if (with != NULL) { +@@ -2197,7 +2233,9 @@ unpack_colocation_set(xmlNode * set, int score, pe_working_set_t * data_set) + } + } else if (local_score >= 0) { + resource_t *last = NULL; +- for (xml_rsc = __xml_first_child(set); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc)); + if (last != NULL) { +@@ -2216,14 +2254,18 @@ unpack_colocation_set(xmlNode * set, int score, pe_working_set_t * data_set) + * (i.e. that no one in the set can run with anyone else in the set) + */ + +- for (xml_rsc = __xml_first_child(set); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + xmlNode *xml_rsc_with = NULL; + + EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc)); + +- for (xml_rsc_with = __xml_first_child(set); xml_rsc_with != NULL; ++ for (xml_rsc_with = __xml_first_child_element(set); ++ xml_rsc_with != NULL; + xml_rsc_with = __xml_next_element(xml_rsc_with)) { ++ + if (crm_str_eq((const char *)xml_rsc_with->name, XML_TAG_RESOURCE_REF, TRUE)) { + if (safe_str_eq(resource->id, ID(xml_rsc_with))) { + break; +@@ -2258,7 +2300,9 @@ colocate_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, int score, + + if (sequential_1 == NULL || crm_is_true(sequential_1)) { + /* get the first one */ +- for (xml_rsc = __xml_first_child(set1); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); + break; +@@ -2270,7 +2314,9 @@ colocate_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, int score, + /* get the last one */ + const char *rid = NULL; + +- for (xml_rsc = __xml_first_child(set2); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set2); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + rid = ID(xml_rsc); + } +@@ -2282,7 +2328,9 @@ colocate_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, int score, + rsc_colocation_new(id, NULL, score, rsc_1, rsc_2, role_1, role_2, data_set); + + } else if (rsc_1 != NULL) { +- for (xml_rsc = __xml_first_child(set2); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set2); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc)); + rsc_colocation_new(id, NULL, score, rsc_1, rsc_2, role_1, role_2, data_set); +@@ -2290,7 +2338,9 @@ colocate_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, int score, + } + + } else if (rsc_2 != NULL) { +- for (xml_rsc = __xml_first_child(set1); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); + rsc_colocation_new(id, NULL, score, rsc_1, rsc_2, role_1, role_2, data_set); +@@ -2298,14 +2348,18 @@ colocate_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, int score, + } + + } else { +- for (xml_rsc = __xml_first_child(set1); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) { ++ for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL; ++ xml_rsc = __xml_next_element(xml_rsc)) { ++ + if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) { + xmlNode *xml_rsc_2 = NULL; + + EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); + +- for (xml_rsc_2 = __xml_first_child(set2); xml_rsc_2 != NULL; ++ for (xml_rsc_2 = __xml_first_child_element(set2); ++ xml_rsc_2 != NULL; + xml_rsc_2 = __xml_next_element(xml_rsc_2)) { ++ + if (crm_str_eq((const char *)xml_rsc_2->name, XML_TAG_RESOURCE_REF, TRUE)) { + EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2)); + rsc_colocation_new(id, NULL, score, rsc_1, rsc_2, role_1, role_2, data_set); +@@ -2537,7 +2591,9 @@ unpack_rsc_colocation(xmlNode * xml_obj, pe_working_set_t * data_set) + return FALSE; + } + +- for (set = __xml_first_child(xml_obj); set != NULL; set = __xml_next_element(set)) { ++ for (set = __xml_first_child_element(xml_obj); set != NULL; ++ set = __xml_next_element(set)) { ++ + if (crm_str_eq((const char *)set->name, XML_CONS_TAG_RSC_SET, TRUE)) { + any_sets = TRUE; + set = expand_idref(set, data_set->input); +@@ -2885,7 +2941,9 @@ unpack_rsc_ticket(xmlNode * xml_obj, pe_working_set_t * data_set) + return FALSE; + } + +- for (set = __xml_first_child(xml_obj); set != NULL; set = __xml_next_element(set)) { ++ for (set = __xml_first_child_element(xml_obj); set != NULL; ++ set = __xml_next_element(set)) { ++ + if (crm_str_eq((const char *)set->name, XML_CONS_TAG_RSC_SET, TRUE)) { + any_sets = TRUE; + set = expand_idref(set, data_set->input); +diff --git a/pengine/native.c b/pengine/native.c +index 747cb10..10f1264 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -606,8 +606,9 @@ is_op_dup(resource_t * rsc, const char *name, const char *interval) + xmlNode *operation = NULL; + + CRM_ASSERT(rsc); +- for (operation = __xml_first_child(rsc->ops_xml); operation != NULL; ++ for (operation = __xml_first_child_element(rsc->ops_xml); operation != NULL; + operation = __xml_next_element(operation)) { ++ + if (crm_str_eq((const char *)operation->name, "op", TRUE)) { + value = crm_element_value(operation, "name"); + if (safe_str_neq(value, name)) { +@@ -840,8 +841,10 @@ Recurring(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * + (node == NULL || node->details->maintenance == FALSE)) { + xmlNode *operation = NULL; + +- for (operation = __xml_first_child(rsc->ops_xml); operation != NULL; ++ for (operation = __xml_first_child_element(rsc->ops_xml); ++ operation != NULL; + operation = __xml_next_element(operation)) { ++ + if (crm_str_eq((const char *)operation->name, "op", TRUE)) { + RecurringOp(rsc, start, node, operation, data_set); + } +@@ -1064,8 +1067,10 @@ Recurring_Stopped(resource_t * rsc, action_t * start, node_t * node, pe_working_ + (node == NULL || node->details->maintenance == FALSE)) { + xmlNode *operation = NULL; + +- for (operation = __xml_first_child(rsc->ops_xml); operation != NULL; ++ for (operation = __xml_first_child_element(rsc->ops_xml); ++ operation != NULL; + operation = __xml_next_element(operation)) { ++ + if (crm_str_eq((const char *)operation->name, "op", TRUE)) { + RecurringOp_Stopped(rsc, start, node, operation, data_set); + } +diff --git a/tools/crm_mon.c b/tools/crm_mon.c +index 82e691e..ee9a3ef 100644 +--- a/tools/crm_mon.c ++++ b/tools/crm_mon.c +@@ -1685,7 +1685,8 @@ print_rsc_history(FILE *stream, pe_working_set_t *data_set, node_t *node, + } + + /* Create a list of this resource's operations */ +- for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next(rsc_op)) { ++ for (rsc_op = __xml_first_child_element(rsc_entry); rsc_op != NULL; ++ rsc_op = __xml_next_element(rsc_op)) { + if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) { + op_list = g_list_append(op_list, rsc_op); + } +@@ -1753,8 +1754,8 @@ print_node_history(FILE *stream, pe_working_set_t *data_set, + lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE); + + /* Print history of each of the node's resources */ +- for (rsc_entry = __xml_first_child(lrm_rsc); rsc_entry != NULL; +- rsc_entry = __xml_next(rsc_entry)) { ++ for (rsc_entry = __xml_first_child_element(lrm_rsc); rsc_entry != NULL; ++ rsc_entry = __xml_next_element(rsc_entry)) { + + if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) { + print_rsc_history(stream, data_set, node, rsc_entry, operations); +@@ -1974,8 +1975,8 @@ print_node_summary(FILE *stream, pe_working_set_t * data_set, gboolean operation + } + + /* Print each node in the CIB status */ +- for (node_state = __xml_first_child(cib_status); node_state != NULL; +- node_state = __xml_next(node_state)) { ++ for (node_state = __xml_first_child_element(cib_status); node_state != NULL; ++ node_state = __xml_next_element(node_state)) { + if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) { + print_node_history(stream, data_set, node_state, operations); + } +@@ -4590,7 +4591,9 @@ static void crm_diff_update_v2(const char *event, xmlNode * msg) + xmlNode *state = NULL; + xmlNode *status = first_named_child(match, XML_CIB_TAG_STATUS); + +- for (state = __xml_first_child(status); state != NULL; state = __xml_next(state)) { ++ for (state = __xml_first_child_element(status); state != NULL; ++ state = __xml_next_element(state)) { ++ + node = crm_element_value(state, XML_ATTR_UNAME); + if (node == NULL) { + node = ID(state); +@@ -4601,7 +4604,9 @@ static void crm_diff_update_v2(const char *event, xmlNode * msg) + } else if(strcmp(name, XML_CIB_TAG_STATUS) == 0) { + xmlNode *state = NULL; + +- for (state = __xml_first_child(match); state != NULL; state = __xml_next(state)) { ++ for (state = __xml_first_child_element(match); state != NULL; ++ state = __xml_next_element(state)) { ++ + node = crm_element_value(state, XML_ATTR_UNAME); + if (node == NULL) { + node = ID(state); +diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c +index c4f96cd..906ea75 100644 +--- a/tools/crm_resource_print.c ++++ b/tools/crm_resource_print.c +@@ -29,8 +29,8 @@ cli_resource_print_cts_constraints(pe_working_set_t * data_set) + xmlNode *lifetime = NULL; + xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input); + +- for (xml_obj = __xml_first_child(cib_constraints); xml_obj != NULL; +- xml_obj = __xml_next(xml_obj)) { ++ for (xml_obj = __xml_first_child_element(cib_constraints); xml_obj != NULL; ++ xml_obj = __xml_next_element(xml_obj)) { + const char *id = crm_element_value(xml_obj, XML_ATTR_ID); + + if (id == NULL) { +diff --git a/tools/crmadmin.c b/tools/crmadmin.c +index 920c262..c91807f 100644 +--- a/tools/crmadmin.c ++++ b/tools/crmadmin.c +@@ -546,7 +546,9 @@ do_find_node_list(xmlNode * xml_node) + xmlNode *node = NULL; + xmlNode *nodes = get_object_root(XML_CIB_TAG_NODES, xml_node); + +- for (node = __xml_first_child(nodes); node != NULL; node = __xml_next(node)) { ++ for (node = __xml_first_child_element(nodes); node != NULL; ++ node = __xml_next_element(node)) { ++ + if (crm_str_eq((const char *)node->name, XML_CIB_TAG_NODE, TRUE)) { + + if (BASH_EXPORT) { +diff --git a/tools/fake_transition.c b/tools/fake_transition.c +index 5741fed..3cdd1f1 100644 +--- a/tools/fake_transition.c ++++ b/tools/fake_transition.c +@@ -160,7 +160,9 @@ create_op(xmlNode * cib_resource, const char *task, int interval, int outcome) + op->t_rcchange = op->t_run; + + op->call_id = 0; +- for (xop = __xml_first_child(cib_resource); xop != NULL; xop = __xml_next(xop)) { ++ for (xop = __xml_first_child_element(cib_resource); xop != NULL; ++ xop = __xml_next_element(xop)) { ++ + int tmp = 0; + + crm_element_value_int(xop, XML_LRM_ATTR_CALLID, &tmp); +-- +1.8.3.1 + + +From b05f992627cf468c815370010b5f9e2e58d122d5 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 29 Jul 2019 11:52:51 -0500 +Subject: [PATCH 68/96] Build: include,doc: ensure non-installed files are + listed in makefiles + +This is especially important for headers, so they are distributed with +"make dist" for building. +--- + cts/Makefile.am | 2 ++ + include/Makefile.am | 20 ++++++-------------- + include/crm/Makefile.am | 2 ++ + include/crm/common/Makefile.am | 6 ++---- + 4 files changed, 12 insertions(+), 18 deletions(-) + +diff --git a/cts/Makefile.am b/cts/Makefile.am +index bf56215..f3f169c 100644 +--- a/cts/Makefile.am ++++ b/cts/Makefile.am +@@ -22,6 +22,8 @@ MAINTAINERCLEANFILES = Makefile.in + CLEANFILES = LSBDummy HBDummy + + EXTRA_DIST = $(cts_SCRIPTS) $(cts_DATA) ++noinst_SCRIPTS = cluster_test \ ++ OCFIPraTest.py + + ctsdir = $(datadir)/$(PACKAGE)/tests/cts + ctslibdir = $(pyexecdir)/cts +diff --git a/include/Makefile.am b/include/Makefile.am +index f7f5714..e6444ea 100644 +--- a/include/Makefile.am ++++ b/include/Makefile.am +@@ -1,24 +1,16 @@ + # + # Copyright (C) 2004-2009 Andrew Beekhof + # +-# This program is free software; you can redistribute it and/or +-# modify it under the terms of the GNU General Public License +-# as published by the Free Software Foundation; either version 2 +-# of the License, or (at your option) any later version. +-# +-# This program is distributed in the hope that it will be useful, +-# but WITHOUT ANY WARRANTY; without even the implied warranty of +-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-# GNU General Public License for more details. +-# +-# You should have received a copy of the GNU General Public License +-# along with this program; if not, write to the Free Software +-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. ++# This source code is licensed under the GNU General Public License version 2 ++# or later (GPLv2+) WITHOUT ANY WARRANTY. + # + + MAINTAINERCLEANFILES = Makefile.in config.h.in + +-noinst_HEADERS = portability.h config.h crm_internal.h ++noinst_HEADERS = config.h \ ++ crm_internal.h \ ++ doxygen.h \ ++ portability.h + pkginclude_HEADERS = crm_config.h + + SUBDIRS = crm +diff --git a/include/crm/Makefile.am b/include/crm/Makefile.am +index 951d483..1226537 100644 +--- a/include/crm/Makefile.am ++++ b/include/crm/Makefile.am +@@ -23,4 +23,6 @@ header_HEADERS = attrd.h cib.h cluster.h compatibility.h crm.h \ + error.h lrmd.h msg_xml.h services.h stonith-ng.h \ + transition.h + ++noinst_HEADERS = lrmd_alerts_internal.h ++ + SUBDIRS = common pengine cib fencing cluster +diff --git a/include/crm/common/Makefile.am b/include/crm/common/Makefile.am +index aacb6ff..7ed2360 100644 +--- a/include/crm/common/Makefile.am ++++ b/include/crm/common/Makefile.am +@@ -13,8 +13,6 @@ headerdir=$(pkgincludedir)/crm/common + + header_HEADERS = xml.h ipc.h util.h iso8601.h mainloop.h logging.h \ + nvpair.h +-noinst_HEADERS = ipcs.h internal.h remote_internal.h xml_internal.h \ ++noinst_HEADERS = cib_secrets.h ipcs.h internal.h alerts_internal.h \ ++ iso8601_internal.h remote_internal.h xml_internal.h \ + ipc_internal.h +-if BUILD_CIBSECRETS +-noinst_HEADERS += cib_secrets.h +-endif +-- +1.8.3.1 + + +From af96be2dd205fecee505ee9c86d21ed6123c466e Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 26 Jun 2019 16:44:30 -0500 +Subject: [PATCH 69/96] Build: libpe_status,libpe_rules: make sure pkg-config + files are built + +--- + lib/Makefile.am | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/lib/Makefile.am b/lib/Makefile.am +index d73bf2e..b00db48 100644 +--- a/lib/Makefile.am ++++ b/lib/Makefile.am +@@ -25,9 +25,9 @@ target_LIBS = $(LIBS:%=pacemaker-%.pc) + + target_PACKAGE = pacemaker.pc + +-all-local: $(target_LIBS) $(target_PACKAGE) ++all-local: $(target_LIBS) $(target_PACKAGE) pacemaker-pe_rules.pc pacemaker-pe_status.pc + +-install-exec-local: $(target_LIBS) $(target_PACKAGE) ++install-exec-local: $(target_LIBS) $(target_PACKAGE) pacemaker-pe_rules.pc pacemaker-pe_status.pc + $(INSTALL) -d $(DESTDIR)/$(libdir)/pkgconfig + $(INSTALL) -m 644 $(target_LIBS) $(target_PACKAGE) $(DESTDIR)/$(libdir)/pkgconfig + +-- +1.8.3.1 + + +From 941de1727bd07c2cc3f30f1f5508370896a8de53 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 22 Aug 2019 13:06:39 -0500 +Subject: [PATCH 70/96] Build: lib: reorganize makefile + +... for readability and simplicity. This takes care of a minor issue where not +all installed files were uninstalled. +--- + lib/Makefile.am | 19 +++++++++---------- + 1 file changed, 9 insertions(+), 10 deletions(-) + +diff --git a/lib/Makefile.am b/lib/Makefile.am +index b00db48..617008a 100644 +--- a/lib/Makefile.am ++++ b/lib/Makefile.am +@@ -17,26 +17,25 @@ + # + MAINTAINERCLEANFILES = Makefile.in + +-EXTRA_DIST = pacemaker.pc.in $(target_LIBS:%=%.in) +- + LIBS = cib lrmd service pengine fencing cluster + +-target_LIBS = $(LIBS:%=pacemaker-%.pc) ++PC_FILES = $(LIBS:%=pacemaker-%.pc) \ ++ pacemaker.pc + +-target_PACKAGE = pacemaker.pc ++EXTRA_DIST = $(PC_FILES:%=%.in) + +-all-local: $(target_LIBS) $(target_PACKAGE) pacemaker-pe_rules.pc pacemaker-pe_status.pc ++all-local: $(PC_FILES) + +-install-exec-local: $(target_LIBS) $(target_PACKAGE) pacemaker-pe_rules.pc pacemaker-pe_status.pc ++install-exec-local: $(PC_FILES) + $(INSTALL) -d $(DESTDIR)/$(libdir)/pkgconfig +- $(INSTALL) -m 644 $(target_LIBS) $(target_PACKAGE) $(DESTDIR)/$(libdir)/pkgconfig ++ $(INSTALL) -m 644 $(PC_FILES) $(DESTDIR)/$(libdir)/pkgconfig + + uninstall-local: +- cd $(DESTDIR)/$(libdir)/pkgconfig && rm -f $(target_LIBS) $(target_PACKAGE) +- rmdir $(DESTDIR)/$(libdir)/pkgconfig 2> /dev/null || : ++ -cd $(DESTDIR)/$(libdir)/pkgconfig && rm -f $(PC_FILES) ++ -rmdir $(DESTDIR)/$(libdir)/pkgconfig 2> /dev/null + + clean-local: +- rm -f *.pc ++ rm -f $(PC_FILES) + + ## Subdirectories... + SUBDIRS = gnu common pengine transition cib services fencing lrmd cluster +-- +1.8.3.1 + + +From 021dadf515c547a82dd00ad33dded91f9f8dac10 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 9 Jan 2018 18:27:09 -0600 +Subject: [PATCH 71/96] Build: www: update makefiles for website restructuring + +--- + GNUmakefile | 10 +++++----- + abi-check | 7 +++++-- + doc/Makefile.am | 6 +++--- + doc/Pacemaker_Explained/en-US/Ch-Intro.txt | 4 +++- + 4 files changed, 16 insertions(+), 11 deletions(-) + +diff --git a/GNUmakefile b/GNUmakefile +index b2d5a28..352903f 100644 +--- a/GNUmakefile ++++ b/GNUmakefile +@@ -341,17 +341,17 @@ global: clean-generic + + global-upload: global + htags -sanhIT +- rsync $(RSYNC_OPTS) HTML/ "$(RSYNC_DEST)/global/$(PACKAGE)/$(TAG)" ++ rsync $(RSYNC_OPTS) HTML/ "$(RSYNC_DEST)/$(PACKAGE)/global/$(TAG)/" + + %.8.html: %.8 + echo groff -mandoc `man -w ./$<` -T html > $@ + groff -mandoc `man -w ./$<` -T html > $@ +- rsync $(RSYNC_OPTS) "$@" "$(RSYNC_DEST)/man/$(PACKAGE)/" ++ rsync $(RSYNC_OPTS) "$@" "$(RSYNC_DEST)/$(PACKAGE)/man/" + + %.7.html: %.7 + echo groff -mandoc `man -w ./$<` -T html > $@ + groff -mandoc `man -w ./$<` -T html > $@ +- rsync $(RSYNC_OPTS) "$@" "$(RSYNC_DEST)/man/$(PACKAGE)/" ++ rsync $(RSYNC_OPTS) "$@" "$(RSYNC_DEST)/$(PACKAGE)/man/" + + manhtml-upload: all + find . -name "[a-z]*.[78]" -exec make \{\}.html \; +@@ -360,12 +360,12 @@ doxygen: Doxyfile + doxygen Doxyfile + + doxygen-upload: doxygen +- rsync $(RSYNC_OPTS) doc/api/html/ "$(RSYNC_DEST)/doxygen/$(PACKAGE)/$(TAG)" ++ rsync $(RSYNC_OPTS) doc/api/html/ "$(RSYNC_DEST)/$(PACKAGE)/doxygen/$(TAG)/" + + abi: + ./abi-check pacemaker $(LAST_RELEASE) $(TAG) + abi-www: +- ./abi-check -u pacemaker $(LAST_RELEASE) $(TAG) ++ export RSYNC_DEST=$(RSYNC_DEST); ./abi-check -u pacemaker $(LAST_RELEASE) $(TAG) + + www: manhtml-upload global-upload doxygen-upload + make RSYNC_DEST=$(RSYNC_DEST) -C doc www +diff --git a/abi-check b/abi-check +index d28192b..18002c8 100755 +--- a/abi-check ++++ b/abi-check +@@ -1,6 +1,9 @@ + #!/bin/bash +-UPLOAD=0 + ++# toplevel rsync destination for www targets (without trailing slash) ++: ${RSYNC_DEST:=root@www.clusterlabs.org:/var/www/html} ++ ++UPLOAD=0 + if [ $1 = "-u" ]; then + UPLOAD=1; shift + fi +@@ -98,6 +101,6 @@ if [ $# = 2 ]; then + -d2 abi_dumps/${PACKAGE}/${PACKAGE}_${V2}.abi.tar.gz + + if [ $UPLOAD = 1 -a -d compat_reports/pacemaker/${V1}_to_${V2} ]; then +- rsync -azxlSD --progress compat_reports/pacemaker/${V1}_to_${V2} root@www.clusterlabs.org:/var/www/html/abi/pacemaker/ ++ rsync -azxlSD --progress compat_reports/pacemaker/${V1}_to_${V2} ${RSYNC_DEST}/${PACKAGE}/abi/ + fi + fi +diff --git a/doc/Makefile.am b/doc/Makefile.am +index 8b04007..98ae680 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -29,7 +29,7 @@ docbook = Clusters_from_Scratch \ + doc_DATA = $(ascii) $(generated_docs) + + # toplevel rsync destination for www targets (without trailing slash) +-RSYNC_DEST ?= root@www.clusterlabs.org:/var/www/html/ ++RSYNC_DEST ?= root@www.clusterlabs.org:/var/www/html + + # recursive, preserve symlinks/permissions/times, verbose, compress, + # don't cross filesystems, sparse, show progress +@@ -294,7 +294,7 @@ pdf: + + www: clean-local $(generated_docs) $(ascii) + make www-cli +- rsync $(RSYNC_OPTS) $(generated_docs) $(ascii) $(asciiman) "$(RSYNC_DEST)/doc/" ++ rsync $(RSYNC_OPTS) $(generated_docs) $(ascii) $(asciiman) "$(RSYNC_DEST)/$(PACKAGE)/doc/" + + www-pcs: www-cli + +@@ -313,7 +313,7 @@ if BUILD_DOCBOOK + mv $$book/publish/$$lang/Pacemaker/$(PACKAGE_SERIES)-$(ASCIIDOC_CLI_TYPE)/epub/$$book/Pacemaker-1.1{-$(ASCIIDOC_CLI_TYPE),}-$$book-$$lang.epub; \ + mv $$book/publish/$$lang/Pacemaker/$(PACKAGE_SERIES)-$(ASCIIDOC_CLI_TYPE)/pdf/$$book/Pacemaker-1.1{-$(ASCIIDOC_CLI_TYPE),}-$$book-$$lang.pdf; \ + done; \ +- rsync $(RSYNC_OPTS) $$book/publish/* "$(RSYNC_DEST)/doc/"; \ ++ rsync $(RSYNC_OPTS) $$book/publish/* "$(RSYNC_DEST)/$(PACKAGE)/doc/"; \ + sed -i.sed 's@version:.*@version: $(PACKAGE_SERIES)@' $$book/publican.cfg; \ + done + endif +diff --git a/doc/Pacemaker_Explained/en-US/Ch-Intro.txt b/doc/Pacemaker_Explained/en-US/Ch-Intro.txt +index e610651..dad0635 100644 +--- a/doc/Pacemaker_Explained/en-US/Ch-Intro.txt ++++ b/doc/Pacemaker_Explained/en-US/Ch-Intro.txt +@@ -16,7 +16,9 @@ Additionally, this document is NOT a step-by-step how-to guide for + configuring a specific clustering scenario. + + Although such guides exist, +-footnote:[For example, see the http://www.clusterlabs.org/doc/[Clusters from Scratch] guide.] ++footnote:[ ++For example, see https://www.clusterlabs.org/pacemaker/doc/[Clusters from Scratch] ++] + the purpose of this document is to provide an understanding of the building + blocks that can be used to construct any type of Pacemaker cluster. + +-- +1.8.3.1 + + +From 3d7c882c22bcb4f68a549297c11288aacd54fad9 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 11 Jan 2018 12:45:33 -0600 +Subject: [PATCH 72/96] Build: www: get rid of ASCIIDOC_CLI_TYPE + +Only pcs has been supported for a long while, and it only ever applied to +"Clusters from Scratch" anyway, while being applied to all books. +--- + doc/Makefile.am | 28 ++++++++-------------------- + 1 file changed, 8 insertions(+), 20 deletions(-) + +diff --git a/doc/Makefile.am b/doc/Makefile.am +index 98ae680..a00b704 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -39,7 +39,6 @@ publican_docs = + generated_docs = + generated_mans = + +-ASCIIDOC_CLI_TYPE := pcs + + # What formats to build: pdf,html,html-single,html-desktop,epub + DOCBOOK_FORMATS := html-desktop +@@ -290,33 +289,22 @@ brand-rpm-install: brand-rpm-build + find publican-clusterlabs -name "*.noarch.rpm" -exec sudo rpm -Uvh --force \{\} \; + + pdf: +- make DOCBOOK_FORMATS="pdf" ASCIIDOC_CLI_TYPE=$(ASCIIDOC_CLI_TYPE) all-local ++ make DOCBOOK_FORMATS="pdf" all-local + + www: clean-local $(generated_docs) $(ascii) +- make www-cli +- rsync $(RSYNC_OPTS) $(generated_docs) $(ascii) $(asciiman) "$(RSYNC_DEST)/$(PACKAGE)/doc/" +- +-www-pcs: www-cli +- +-www-cli: +- for book in $(docbook); do \ +- sed -i.sed 's@brand:.*@brand: clusterlabs@' $$book/publican.cfg; \ +- sed -i.sed 's@version:.*@version: $(PACKAGE_SERIES)-$(ASCIIDOC_CLI_TYPE)@' $$book/publican.cfg; \ ++ for book in $(docbook); do \ ++ sed -i.sed 's@^brand:.*@brand: clusterlabs@' $$book/publican.cfg; \ + done +- make DOCBOOK_FORMATS="pdf,html,html-single,epub" DOCBOOK_LANGS="$(UPLOAD_LANGS)" ASCIIDOC_CLI_TYPE=$(ASCIIDOC_CLI_TYPE) all-local +- echo Uploading current $(PACKAGE_SERIES)-$(ASCIIDOC_CLI_TYPE) documentation set to clusterlabs.org ++ make DOCBOOK_FORMATS="pdf,html,html-single,epub" DOCBOOK_LANGS="$(UPLOAD_LANGS)" all-local ++ echo Uploading current $(PACKAGE_SERIES) documentation set to clusterlabs.org + if BUILD_DOCBOOK + for book in $(docbook); do \ + echo Uploading $$book...; \ +- echo "Generated on `date` from version: $(shell git log --pretty="format:%h %d" -n 1)" >> $$book/publish/build-$(PACKAGE_SERIES)-$(ASCIIDOC_CLI_TYPE).txt; \ +- for lang in `ls -1 $$book/publish | grep [a-z][a-z]-[A-Z][A-Z]`; do \ +- mv $$book/publish/$$lang/Pacemaker/$(PACKAGE_SERIES)-$(ASCIIDOC_CLI_TYPE)/epub/$$book/Pacemaker-1.1{-$(ASCIIDOC_CLI_TYPE),}-$$book-$$lang.epub; \ +- mv $$book/publish/$$lang/Pacemaker/$(PACKAGE_SERIES)-$(ASCIIDOC_CLI_TYPE)/pdf/$$book/Pacemaker-1.1{-$(ASCIIDOC_CLI_TYPE),}-$$book-$$lang.pdf; \ +- done; \ +- rsync $(RSYNC_OPTS) $$book/publish/* "$(RSYNC_DEST)/$(PACKAGE)/doc/"; \ +- sed -i.sed 's@version:.*@version: $(PACKAGE_SERIES)@' $$book/publican.cfg; \ ++ echo "Generated on `date` from version: $(shell git log --pretty="format:%h %d" -n 1)" >> $$book/publish/build-$(PACKAGE_SERIES).txt; \ ++ rsync $(RSYNC_OPTS) $$book/publish/* "$(RSYNC_DEST)/$(PACKAGE)/doc/"; \ + done + endif ++ rsync $(RSYNC_OPTS) $(generated_docs) $(ascii) "$(RSYNC_DEST)/$(PACKAGE)/doc/" + + clean-local: + -rm -f $(PNGS_GENERATED) +-- +1.8.3.1 + + +From c4b5a54893334bc11a7ceec31b780f664d315629 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= +Date: Tue, 7 Aug 2018 18:10:42 +0200 +Subject: [PATCH 73/96] Build: Makefile.common: use determined symbolic + reference for AsciiDoc + +Also fix a typo and drop attribute passing relevant to crm/pcs duality +in the former documentation, something finally ditched in 81b00e6da +that just missed this last reference behind. +--- + Makefile.common | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/Makefile.common b/Makefile.common +index 469417f..feb68bd 100644 +--- a/Makefile.common ++++ b/Makefile.common +@@ -52,11 +52,11 @@ endif + # Build docbook from asciidoc because XML is a PITA to edit + # + # Build each chapter as a book (since the numbering isn't right for +-# articles and only books can have appendicies) and then strip out the ++# articles and only books can have appendices) and then strip out the + # bits we don't want/need + # + %.xml: %.txt +- $(AM_V_ASCII)asciidoc -b docbook -a cli_name=$(ASCIIDOC_CLI_TYPE) -a $(ASCIIDOC_CLI_TYPE)=true -d book -o $@ $< ++ $(AM_V_ASCII)$(ASCIIDOC) -b docbook -d book -o $@ $< + $(AM_V_at)sed -i 's///' $@ + $(AM_V_at)sed -i 's/ //' $@ # Fix line endings + $(AM_V_at)sed -i 's/\ lang="en"//' $@ # Never specify a language in the chapters +-- +1.8.3.1 + + +From 242bd026c7b5e815c8b5b135341837bb33a9a827 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= +Date: Mon, 20 Aug 2018 20:29:27 +0200 +Subject: [PATCH 74/96] Build: Makefile.common: avoid using verbatim control + characters + +This is not a good practise, since it poses an imminent risk of +"deceptive display (rendering)" at various places, especially +tools that may be used for change reviews, incl. GitHub[1]. + +When at it, arrange for periodical CI to capture undesired +occurrences of such characters in the code base, and related +to that, obtain pristine LGPLv2.1 license text anew using + + curl https://www.gnu.org/licenses/lgpl-2.1.txt \ + | tr -d '\f' > licenses/LGPLv2.1 + +so that embedded "form feed" characters are got rid of (beside +that, also tabulators are changed into spaces, which is how this +authoritative server currently carries it). + +[1] https://github.com/isaacs/github/issues/1329 +--- + Makefile.common | 5 ++--- + licenses/LGPLv2.1 | 32 +++++++++++++++----------------- + 2 files changed, 17 insertions(+), 20 deletions(-) + +diff --git a/Makefile.common b/Makefile.common +index feb68bd..e50441b 100644 +--- a/Makefile.common ++++ b/Makefile.common +@@ -56,9 +56,8 @@ endif + # bits we don't want/need + # + %.xml: %.txt +- $(AM_V_ASCII)$(ASCIIDOC) -b docbook -d book -o $@ $< +- $(AM_V_at)sed -i 's///' $@ +- $(AM_V_at)sed -i 's/ //' $@ # Fix line endings ++ $(AM_V_ASCII)$(ASCIIDOC) -b docbook -d book -o $@-t $< ++ $(AM_V_at)tr -d '\036\r' <$@-t >$@; rm "$@-t"# Fix line endings + $(AM_V_at)sed -i 's/\ lang="en"//' $@ # Never specify a language in the chapters + $(AM_V_at)sed -i 's/simpara/para/g' $@ # publican doesn't correctly render footnotes with simpara + $(AM_V_at)sed -i 's/.*.*//g' $@ # Remove dangling tag +diff --git a/licenses/LGPLv2.1 b/licenses/LGPLv2.1 +index 602bfc9..e5ab03e 100644 +--- a/licenses/LGPLv2.1 ++++ b/licenses/LGPLv2.1 +@@ -1,5 +1,5 @@ +- GNU LESSER GENERAL PUBLIC LICENSE +- Version 2.1, February 1999 ++ GNU LESSER GENERAL PUBLIC LICENSE ++ Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +@@ -10,7 +10,7 @@ + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + +- Preamble ++ Preamble + + The licenses for most software are designed to take away your + freedom to share and change it. By contrast, the GNU General Public +@@ -55,7 +55,7 @@ modified by someone else and passed on, the recipients should know + that what they have is not the original version, so that the original + author's reputation will not be affected by problems that might be + introduced by others. +- ++ + Finally, software patents pose a constant threat to the existence of + any free program. We wish to make sure that a company cannot + effectively restrict the users of a free program by obtaining a +@@ -111,8 +111,8 @@ modification follow. Pay close attention to the difference between a + "work based on the library" and a "work that uses the library". The + former contains code derived from the library, whereas the latter must + be combined with the library in order to run. +- +- GNU LESSER GENERAL PUBLIC LICENSE ++ ++ GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +@@ -158,7 +158,7 @@ Library. + You may charge a fee for the physical act of transferring a copy, + and you may at your option offer warranty protection in exchange for a + fee. +- ++ + 2. You may modify your copy or copies of the Library or any portion + of it, thus forming a work based on the Library, and copy and + distribute such modifications or work under the terms of Section 1 +@@ -216,7 +216,7 @@ instead of to this License. (If a newer version than version 2 of the + ordinary GNU General Public License has appeared, then you can specify + that version instead if you wish.) Do not make any other change in + these notices. +- ++ + Once this change is made in a given copy, it is irreversible for + that copy, so the ordinary GNU General Public License applies to all + subsequent copies and derivative works made from that copy. +@@ -267,7 +267,7 @@ Library will still fall under Section 6.) + distribute the object code for the work under the terms of Section 6. + Any executables containing that work also fall under Section 6, + whether or not they are linked directly with the Library itself. +- ++ + 6. As an exception to the Sections above, you may also combine or + link a "work that uses the Library" with the Library to produce a + work containing portions of the Library, and distribute that work +@@ -329,7 +329,7 @@ restrictions of other proprietary libraries that do not normally + accompany the operating system. Such a contradiction means you cannot + use both them and the Library together in an executable that you + distribute. +- ++ + 7. You may place library facilities that are a work based on the + Library side-by-side in a single library together with other library + facilities not covered by this License, and distribute such a combined +@@ -370,7 +370,7 @@ subject to these terms and conditions. You may not impose any further + restrictions on the recipients' exercise of the rights granted herein. + You are not responsible for enforcing compliance by third parties with + this License. +- ++ + 11. If, as a consequence of a court judgment or allegation of patent + infringement or for any other reason (not limited to patent issues), + conditions are imposed on you (whether by court order, agreement or +@@ -422,7 +422,7 @@ conditions either of that version or of any later version published by + the Free Software Foundation. If the Library does not specify a + license version number, you may choose any version ever published by + the Free Software Foundation. +- ++ + 14. If you wish to incorporate parts of the Library into other free + programs whose distribution conditions are incompatible with these, + write to the author to ask for permission. For software which is +@@ -432,7 +432,7 @@ decision will be guided by the two goals of preserving the free status + of all derivatives of our free software and of promoting the sharing + and reuse of software generally. + +- NO WARRANTY ++ NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO + WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +@@ -455,8 +455,8 @@ FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF + SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH + DAMAGES. + +- END OF TERMS AND CONDITIONS +- ++ END OF TERMS AND CONDITIONS ++ + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +@@ -500,5 +500,3 @@ necessary. Here is a sample; alter the names: + Ty Coon, President of Vice + + That's all there is to it! +- +- +-- +1.8.3.1 + + +From 73fa92b8d95b2f16cf8257504b5a7e5c8b6c5099 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= +Date: Tue, 7 Aug 2018 21:45:51 +0200 +Subject: [PATCH 75/96] Build: add support for AsciiDoc's successor, + Asciidoctor + +Everything seems to be built just fine when Asciidoctor is present in +the build environment and AsciiDoc is not (as it takes the precedence), +except for Pacemake Explained that needs some more tweaking to be +usable with Asciidoctor: + +> Ch-Options.xml:125: validity error : Element emphasis is not declared in literal list of possible children +> Ch-Options.xml:419: validity error : Element xref is not declared in literal list of possible children +> Ch-Options.xml:716: validity error : Element xref is not declared in literal list of possible children +> Ch-Options.xml:824: validity error : Element emphasis is not declared in literal list of possible children +> Ch-Options.xml:860: validity error : Element emphasis is not declared in literal list of possible children +> Ch-Options.xml:896: validity error : Element emphasis is not declared in literal list of possible children +> Ch-Resources.xml:608: validity error : Element xref is not declared in literal list of possible children +> Ch-Constraints.xml:525: validity error : Element xref is not declared in literal list of possible children +> Ch-Advanced-Options.xml:662: validity error : Element xref is not declared in literal list of possible children +> Ch-Stonith.xml:153: validity error : Element emphasis is not declared in literal list of possible children +> Ch-Stonith.xml:153: validity error : Element emphasis is not declared in literal list of possible children + +Also remove superfluous "inverse grep" conditionalizing amounting to +a thinko how that's supposed to work. Another logical problem was with +"it's OK to generate the final product of the makefile's target early +and adjust it in-place in the steps of the recipe to follow" broken +assumption. +--- + INSTALL.md | 2 +- + Makefile.common | 31 ++++++++++++++++++++----------- + configure.ac | 7 ++++--- + doc/Makefile.am | 6 +++++- + 4 files changed, 30 insertions(+), 16 deletions(-) + +diff --git a/INSTALL.md b/INSTALL.md +index f02b589..8671ac2 100644 +--- a/INSTALL.md ++++ b/INSTALL.md +@@ -29,7 +29,7 @@ + * libesmtp-devel (crm_mon --mail-to option) + * lm_sensors-devel (crm_mon --snmp-traps option) + * net-snmp-devel (crm_mon --snmp-traps option) +-* asciidoc (documentation) ++* asciidoc or asciidoctor (documentation) + * help2man (documentation) + * publican (documentation) + * inkscape (documentation) +diff --git a/Makefile.common b/Makefile.common +index e50441b..0b06ec4 100644 +--- a/Makefile.common ++++ b/Makefile.common +@@ -55,17 +55,26 @@ endif + # articles and only books can have appendices) and then strip out the + # bits we don't want/need + # ++# XXX Sequence of tr/sed commands should be replaced with a single XSLT ++# + %.xml: %.txt +- $(AM_V_ASCII)$(ASCIIDOC) -b docbook -d book -o $@-t $< +- $(AM_V_at)tr -d '\036\r' <$@-t >$@; rm "$@-t"# Fix line endings +- $(AM_V_at)sed -i 's/\ lang="en"//' $@ # Never specify a language in the chapters +- $(AM_V_at)sed -i 's/simpara/para/g' $@ # publican doesn't correctly render footnotes with simpara +- $(AM_V_at)sed -i 's/.*.*//g' $@ # Remove dangling tag +- $(AM_V_at)sed -i 's/.*preface>//g' $@ # Remove preface elements +- $(AM_V_at)sed -i 's:::g' $@ # Remove empty title +- $(AM_V_at)sed -i 's/chapter/section/g' $@ # Chapters become sections, so that books can become chapters +- $(AM_V_at)sed -i 's/<.*bookinfo.*>//g' $@ # Strip out bookinfo, we don't need it +- -grep -qis "//' $@ # We just want the appendix tag +- -grep -vqis "/chapter>/g' $@ # Rename to chapter ++if IS_ASCIIDOC ++ $(AM_V_ASCII)$(ASCIIDOC_CONV) -b docbook -d book -o $@-tt $< ++else ++ $(AM_V_ASCII)$(ASCIIDOC_CONV) -b docbook45 -d book -o $@-tt $< ++endif ++ $(AM_V_at)tr -d '\036\r' <$@-tt >$@-t; rm -f $@-tt # Fix line endings ++ $(AM_V_at)sed -i 's/\ lang="en"//' $@-t # Never specify a language in the chapters ++ $(AM_V_at)sed -i 's/simpara/para/g' $@-t # publican doesn't correctly render footnotes with simpara ++ $(AM_V_at)sed -i 's/.*.*//g' $@-t # Remove dangling tag ++ $(AM_V_at)sed -i 's/.*preface>//g' $@-t # Remove preface elements ++ $(AM_V_at)sed -i 's:::g' $@-t # Remove empty title ++ $(AM_V_at)sed -i 's/chapter/section/g' $@-t # Chapters become sections, so that books can become chapters ++ $(AM_V_at)sed -i 's/<.*bookinfo.*>//g' $@-t # Strip out bookinfo, we don't need it ++ $(AM_V_at)! grep -q "//;tb;bf;:b;N;s/.*.*<\/title>.*//;tb;/<appendix/{:i;n;/<\/appendix/{p;d};bi};bb;:f;p;d' \ ++ $@-t # We just want the appendix tag (asciidoctor adds non-empty book-level title) ++ $(AM_V_at)sed -i 's/book>/chapter>/g' $@-t # Rename to chapter (won't trigger if previous sed did) ++ $(AM_V_GEN)mv $@-t $@ + + # echo Rebuilt $@ from $< +diff --git a/configure.ac b/configure.ac +index a7084e2..c58b556 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -605,7 +605,7 @@ AC_CHECK_PROGS(MAKE, gmake make) + AC_PATH_PROGS(HTML2TXT, lynx w3m) + AC_PATH_PROGS(HELP2MAN, help2man) + AC_PATH_PROGS(POD2MAN, pod2man, pod2man) +-AC_PATH_PROGS(ASCIIDOC, asciidoc) ++AC_PATH_PROGS([ASCIIDOC_CONV], [asciidoc asciidoctor]) + AC_PATH_PROGS(PUBLICAN, publican) + AC_PATH_PROGS(INKSCAPE, inkscape) + AC_PATH_PROGS(XSLTPROC, xsltproc) +@@ -667,8 +667,9 @@ if test x"${MANPAGE_XSLT}" != x""; then + PCMK_FEATURES="$PCMK_FEATURES agent-manpages" + fi + +-AM_CONDITIONAL(BUILD_ASCIIDOC, test x"${ASCIIDOC}" != x"") +-if test x"${ASCIIDOC}" != x""; then ++AM_CONDITIONAL([IS_ASCIIDOC], [echo "${ASCIIDOC_CONV}" | grep -Eq 'asciidoc$']) ++AM_CONDITIONAL([BUILD_ASCIIDOC], [test "x${ASCIIDOC_CONV}" != x]) ++if test "x${ASCIIDOC_CONV}" != x; then + PCMK_FEATURES="$PCMK_FEATURES ascii-docs" + fi + +diff --git a/doc/Makefile.am b/doc/Makefile.am +index a00b704..d59cdcd 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -103,7 +103,11 @@ endif + EXTRA_DIST = $(docbook:%=%.xml) + + %.html: %.txt +- $(AM_V_ASCII)$(ASCIIDOC) --unsafe --backend=xhtml11 $< ++if IS_ASCIIDOC ++ $(AM_V_ASCII)$(ASCIIDOC_CONV) --unsafe --backend=xhtml11 $< ++else ++ $(AM_V_ASCII)$(ASCIIDOC_CONV) --backend=html5 $< ++endif + + # publican-clusterlabs/xsl/{html,html-single,pdf}.xsl refer to URIs + # requiring Internet access, hence we shadow that with a XML catalog-based +-- +1.8.3.1 + + +From 2032f3e990c8fd39c8618dd15e9f8738290ec30f Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Mon, 29 Jan 2018 14:01:01 -0600 +Subject: [PATCH 76/96] Doc: clear detritus from previous failed builds when + building + +--- + doc/Makefile.am | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/doc/Makefile.am b/doc/Makefile.am +index d59cdcd..f1a3c63 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -152,7 +152,7 @@ endif + # With '%' the test for 'newness' fails + Clusters_from_Scratch.build: $(PNGS) $(wildcard Clusters_from_Scratch/en-US/*.xml) $(CFS_XML) $(CFS_SHARED_XML) $(PUBLICAN_INTREE_DEPS) + $(PCMK_V) @echo Building $(@:%.build=%) because of $? +- rm -rf $(@:%.build=%)/publish/* ++ rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + if PUBLICAN_INTREE_BRAND + $(AM_V_PUB)cd $(@:%.build=%) \ + && RPM_BUILD_DIR="" XML_CATALOG_FILES="$(CURDIR)/publican-catalog" \ +@@ -175,7 +175,7 @@ PD_XML=$(PD_TXT:%.txt=%.xml) + # With '%' the test for 'newness' fails + Pacemaker_Development.build: $(wildcard Pacemaker_Development/en-US/*.xml) $(PD_XML) $(PUBLICAN_INTREE_DEPS) + $(PCMK_V) @echo Building $(@:%.build=%) because of $? +- rm -rf $(@:%.build=%)/publish/* ++ rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + if PUBLICAN_INTREE_BRAND + $(AM_V_PUB)cd $(@:%.build=%) \ + && RPM_BUILD_DIR="" XML_CATALOG_FILES="$(CURDIR)/publican-catalog" \ +@@ -202,7 +202,7 @@ $(PE_XML): $(PE_SHARED_XML) + # With '%' the test for 'newness' fails + Pacemaker_Explained.build: $(PNGS) $(wildcard Pacemaker_Explained/en-US/*.xml) $(PE_XML) $(PE_SHARED_XML) $(PUBLICAN_INTREE_DEPS) + $(PCMK_V) @echo Building $(@:%.build=%) because of $? +- rm -rf $(@:%.build=%)/publish/* ++ rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + if PUBLICAN_INTREE_BRAND + $(AM_V_PUB)cd $(@:%.build=%) \ + && RPM_BUILD_DIR="" XML_CATALOG_FILES="$(CURDIR)/publican-catalog" \ +@@ -225,7 +225,7 @@ PR_XML=$(PR_TXT:%.txt=%.xml) + # With '%' the test for 'newness' fails + Pacemaker_Remote.build: $(PNGS) $(wildcard Pacemaker_Remote/en-US/*.xml) $(PR_XML) $(PUBLICAN_INTREE_DEPS) + $(PCMK_V) @echo Building $(@:%.build=%) because of $? +- rm -rf $(@:%.build=%)/publish/* ++ rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + if PUBLICAN_INTREE_BRAND + $(AM_V_PUB)cd $(@:%.build=%) \ + && RPM_BUILD_DIR="" XML_CATALOG_FILES="$(CURDIR)/publican-catalog" \ +-- +1.8.3.1 + + +From 0e072d3debfc2c03751ff69add211a20c9db01cb Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= <jpokorny@redhat.com> +Date: Thu, 9 May 2019 17:41:21 +0200 +Subject: [PATCH 77/96] Build: configure: let "make dist" tarballs be + ustar/posix, not v7 format + +This avoids problem with "file name is too long (max 99); not dumped" +-- we don't suffer from this currently, but eventually could (see also +the subsequent, related change in the set). Not also that some git +hosting sites (GitHub) will also offer tarballs (effectively our current +official, blessed and authoritative redistributables) using *only* even +newer format (pax), which may effectively cause portability issues(!). +--- + configure.ac | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/configure.ac b/configure.ac +index c58b556..e21eb03 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -59,7 +59,8 @@ AC_ARG_WITH(pkg-name, + [ PACKAGE_NAME="$withval" ]) + + dnl Older distros may need: AM_INIT_AUTOMAKE($PACKAGE_NAME, $PACKAGE_VERSION) +-AM_INIT_AUTOMAKE([foreign]) ++dnl tar-ustar: use (older) POSIX variant of generated tar rather than v7 ++AM_INIT_AUTOMAKE([foreign tar-ustar]) + AC_DEFINE_UNQUOTED(PACEMAKER_VERSION, "$PACKAGE_VERSION", Current pacemaker version) + + dnl Versioned attributes implementation is not yet production-ready +-- +1.8.3.1 + + +From 9ddc343d609e3082ec8baead451456a7b7c959de Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= <jpokorny@redhat.com> +Date: Thu, 9 May 2019 17:47:48 +0200 +Subject: [PATCH 78/96] Build: fix "make dist" in one go deficiency (and if, + it's brokenness) + +Problem was with EXTRA_DIST variable that got rightfully assigned +"$(docbook:%=%.xml)" in e99ccd1b0, but frightfully stayed unadjusted +in 7382e6241 -- actually it sort of worked, but only as a byproduct +of generating spurious empty <publication dir>.xml files, which +moreover required multiple iteration over "make dist" for it to +finish successfully -- in a limited sense of "successfully" since +it didn't include the publications' sources at all. + +All this is now rectified. Also note that preceding "Build: configure: +let "make dist" tarballs be posix/pax, not v7 format" commit prepared +the path for us, otherwise we'd be getting something like: + +pacemaker-<GITHASH>/doc/Pacemaker_Administration/en-US/Ch-Upgrading.txt: + file name is too long (max 99); not dumped +--- + doc/Makefile.am | 25 ++++++++++++++++++++----- + 1 file changed, 20 insertions(+), 5 deletions(-) + +diff --git a/doc/Makefile.am b/doc/Makefile.am +index f1a3c63..c981009 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -100,7 +100,12 @@ publican_docs += $(docbook) + endif + endif + +-EXTRA_DIST = $(docbook:%=%.xml) ++EXTRA_DIST = $(ascii) $(SHARED_TXT) $(PNGS_ORIGINAL) $(SVGS) ++EXTRA_DIST += $(CFS_TXT) $(CFS_XML_ONLY) ++EXTRA_DIST += $(PA_TXT) $(PA_XML_ONLY) ++EXTRA_DIST += $(PD_TXT) $(PD_XML_ONLY) ++EXTRA_DIST += $(PE_TXT) $(PE_XML_ONLY) ++EXTRA_DIST += $(PR_TXT) $(PR_XML_ONLY) + + %.html: %.txt + if IS_ASCIIDOC +@@ -132,6 +137,8 @@ publican-catalog: publican-catalog-fallback + && echo '</catalog>' + $(AM_V_GEN)mv $@-t $@ + ++COMMON_XML = Author_Group.xml Book_Info.xml Revision_History.xml ++ + SHARED_TXT=$(wildcard shared/en-US/*.txt) + SHARED_XML=$(SHARED_TXT:%.txt=%.xml) + +@@ -140,6 +147,8 @@ CFS_SHARED_TXT=$(addprefix shared/en-US/,pacemaker-intro.txt) + CFS_SHARED_XML=$(CFS_SHARED_TXT:%.txt=%.xml) + CFS_TXT=$(wildcard Clusters_from_Scratch/en-US/*.txt) + CFS_XML=$(CFS_TXT:%.txt=%.xml) ++CFS_XML_ONLY=$(addprefix Clusters_from_Scratch/en-US/,$(COMMON_XML) \ ++ Clusters_from_Scratch.ent Clusters_from_Scratch.xml Preface.xml) + + $(CFS_XML): $(CFS_SHARED_XML) + +@@ -150,7 +159,7 @@ endif + + # We have to hardcode the book name + # With '%' the test for 'newness' fails +-Clusters_from_Scratch.build: $(PNGS) $(wildcard Clusters_from_Scratch/en-US/*.xml) $(CFS_XML) $(CFS_SHARED_XML) $(PUBLICAN_INTREE_DEPS) ++Clusters_from_Scratch.build: $(PNGS) $(CFS_XML_ONLY) $(CFS_XML) $(CFS_SHARED_XML) $(PUBLICAN_INTREE_DEPS) + $(PCMK_V) @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + if PUBLICAN_INTREE_BRAND +@@ -170,10 +179,12 @@ endif + + PD_TXT=$(wildcard Pacemaker_Development/en-US/*.txt) + PD_XML=$(PD_TXT:%.txt=%.xml) ++PD_XML_ONLY=$(addprefix Pacemaker_Development/en-US/,$(COMMON_XML) \ ++ Pacemaker_Development.ent Pacemaker_Development.xml) + + # We have to hardcode the book name + # With '%' the test for 'newness' fails +-Pacemaker_Development.build: $(wildcard Pacemaker_Development/en-US/*.xml) $(PD_XML) $(PUBLICAN_INTREE_DEPS) ++Pacemaker_Development.build: $(PD_XML_ONLY) $(PD_XML) $(PUBLICAN_INTREE_DEPS) + $(PCMK_V) @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + if PUBLICAN_INTREE_BRAND +@@ -195,12 +206,14 @@ PE_SHARED_TXT=$(addprefix shared/en-US/,pacemaker-intro.txt) + PE_SHARED_XML=$(PE_SHARED_TXT:%.txt=%.xml) + PE_TXT=$(wildcard Pacemaker_Explained/en-US/*.txt) + PE_XML=$(PE_TXT:%.txt=%.xml) ++PE_XML_ONLY=$(addprefix Pacemaker_Explained/en-US/,$(COMMON_XML) \ ++ Pacemaker_Explained.ent Pacemaker_Explained.xml Preface.xml) + + $(PE_XML): $(PE_SHARED_XML) + + # We have to hardcode the book name + # With '%' the test for 'newness' fails +-Pacemaker_Explained.build: $(PNGS) $(wildcard Pacemaker_Explained/en-US/*.xml) $(PE_XML) $(PE_SHARED_XML) $(PUBLICAN_INTREE_DEPS) ++Pacemaker_Explained.build: $(PNGS) $(PE_XML_ONLY) $(PE_XML) $(PE_SHARED_XML) $(PUBLICAN_INTREE_DEPS) + $(PCMK_V) @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + if PUBLICAN_INTREE_BRAND +@@ -220,10 +233,12 @@ endif + + PR_TXT=$(wildcard Pacemaker_Remote/en-US/*.txt) + PR_XML=$(PR_TXT:%.txt=%.xml) ++PR_XML_ONLY=$(addprefix Pacemaker_Remote/en-US/,$(COMMON_XML) \ ++ Pacemaker_Remote.ent Pacemaker_Remote.xml) + + # We have to hardcode the book name + # With '%' the test for 'newness' fails +-Pacemaker_Remote.build: $(PNGS) $(wildcard Pacemaker_Remote/en-US/*.xml) $(PR_XML) $(PUBLICAN_INTREE_DEPS) ++Pacemaker_Remote.build: $(PNGS) $(PR_XML_ONLY) $(PR_XML) $(PUBLICAN_INTREE_DEPS) + $(PCMK_V) @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + if PUBLICAN_INTREE_BRAND +-- +1.8.3.1 + + +From 54b8f258f864d63596b71f38db49491873bddbdf Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= <jpokorny@redhat.com> +Date: Tue, 5 Jun 2018 17:10:28 +0200 +Subject: [PATCH 79/96] Build: xml: *.rng: ship needed non-generated (& only + such) RNG schemas + +Previously, "make dist" would omit those(!), and over-approximation +of such files would get, undesirably, selected (since 0a7a4b4d7 +incl. cibtr-2.rng) in the respective source variable. +Also fix some whitespace issues. +--- + xml/Makefile.am | 9 +++++---- + 1 file changed, 5 insertions(+), 4 deletions(-) + +diff --git a/xml/Makefile.am b/xml/Makefile.am +index ffd09e3..1e5649f 100644 +--- a/xml/Makefile.am ++++ b/xml/Makefile.am +@@ -58,14 +58,15 @@ RNG_version_pairs_last = $(wordlist \ + + RNG_generated = pacemaker.rng $(foreach base,$(RNG_versions),pacemaker-$(base).rng) versions.rng + +-RNG_cfg_base = options nodes resources constraints fencing acls tags alerts +-RNG_base = cib $(RNG_cfg_base) status score rule nvset +-RNG_files = $(foreach base,$(RNG_base),$(wildcard $(base)*.rng)) ++RNG_cfg_base = options nodes resources constraints fencing acls tags alerts ++RNG_base = cib $(RNG_cfg_base) status score rule nvset ++RNG_files = $(foreach base,$(RNG_base),$(wildcard $(base).rng $(base)-*.rng)) + + # List of non-Pacemaker RNGs + RNG_extra = crm_mon.rng + +-RNG_DATA = $(RNG_files) $(RNG_generated) $(RNG_extra) ++dist_RNG_DATA = $(RNG_files) $(RNG_extra) ++nodist_RNG_DATA = $(RNG_generated) + + EXTRA_DIST = best-match.sh + +-- +1.8.3.1 + + +From 898accf5febfac5ddc4304ad4a0f92b8fc866964 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Mon, 5 Aug 2019 10:48:18 -0500 +Subject: [PATCH 80/96] Build: makefiles: make sure all files that should be + distributed are + +... so the result of "make dist" has everything needed to build, +and everything of interest to end users +--- + Makefile.am | 19 +++++++++++--- + Makefile.common | 2 +- + cts/Makefile.am | 56 ++++++++++++++++++++-------------------- + cts/benchmark/Makefile.am | 25 ++++-------------- + doc/Makefile.am | 9 ++++--- + extra/Makefile.am | 25 ++++++------------ + extra/alerts/Makefile.am | 23 ++++++----------- + extra/logrotate/Makefile.am | 21 +++++---------- + extra/resources/Makefile.am | 62 ++++++++++++++++++--------------------------- + fencing/Makefile.am | 4 ++- + lrmd/Makefile.am | 2 +- + mcp/Makefile.am | 4 ++- + pengine/Makefile.am | 6 ++--- + tools/Makefile.am | 11 ++++---- + xml/Makefile.am | 14 ++++++---- + 15 files changed, 127 insertions(+), 156 deletions(-) + +diff --git a/Makefile.am b/Makefile.am +index 874f6ed..3080445 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -18,7 +18,15 @@ + # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + # + +-EXTRA_DIST = autogen.sh ConfigureMe README.in m4/gnulib-cache.m4 ++EXTRA_DIST = CONTRIBUTING.md \ ++ GNUmakefile \ ++ INSTALL.md \ ++ README.markdown \ ++ autogen.sh \ ++ pacemaker.spec.in \ ++ rpmlintrc \ ++ m4/gnulib-cache.m4 \ ++ m4/gnulib-tool.m4 + + MAINTAINERCLEANFILES = Makefile.in aclocal.m4 configure DRF/config-h.in \ + DRF/stamp-h.in libtool.m4 ltdl.m4 +@@ -33,14 +41,17 @@ doc_DATA = README.markdown COPYING + ACLOCAL_AMFLAGS = -I m4 + + licensedir = $(docdir)/licenses/ +-license_DATA = $(wildcard licenses/*) ++dist_license_DATA = $(wildcard licenses/*) + + # Test components + SUBDIRS += cts + + testdir = $(datadir)/$(PACKAGE)/tests/ +-test_SCRIPTS = coverage.sh BasicSanity.sh +-test_DATA = valgrind-pcmk.suppressions ++test_SCRIPTS = coverage.sh ++dist_test_SCRIPTS = BasicSanity.sh ++dist_test_DATA = valgrind-pcmk.suppressions ++ ++EXTRA_SCRIPTS = abi-check bumplibs.sh + + # Scratch file for ad-hoc testing + noinst_PROGRAMS = scratch +diff --git a/Makefile.common b/Makefile.common +index 0b06ec4..a4842b7 100644 +--- a/Makefile.common ++++ b/Makefile.common +@@ -33,7 +33,7 @@ AM_CPPFLAGS = -I$(top_builddir)/include -I$(top_srcdir)/include \ + -I$(top_builddir)/libltdl -I$(top_srcdir)/libltdl + + if BUILD_HELP +-man8_MANS = $(sbin_PROGRAMS:%=%.8) $(sbin_SCRIPTS:%=%.8) ++man8_MANS = $(sbin_PROGRAMS:%=%.8) $(sbin_SCRIPTS:%=%.8) $(dist_sbin_SCRIPTS:%=%.8) + endif + + %.8: % $(MAN8DEPS) +diff --git a/cts/Makefile.am b/cts/Makefile.am +index f3f169c..c0c5707 100644 +--- a/cts/Makefile.am ++++ b/cts/Makefile.am +@@ -21,37 +21,37 @@ MAINTAINERCLEANFILES = Makefile.in + + CLEANFILES = LSBDummy HBDummy + +-EXTRA_DIST = $(cts_SCRIPTS) $(cts_DATA) +-noinst_SCRIPTS = cluster_test \ ++EXTRA_SCRIPTS = cluster_test \ + OCFIPraTest.py + +-ctsdir = $(datadir)/$(PACKAGE)/tests/cts +-ctslibdir = $(pyexecdir)/cts ++testdir = $(datadir)/$(PACKAGE)/tests + +-ctslib_PYTHON = __init__.py \ +- CTSvars.py \ +- CM_lha.py \ +- CM_ais.py \ +- CTS.py \ +- CTSaudits.py \ +- CTStests.py \ +- CTSscenarios.py \ +- CIB.py \ +- cib_xml.py \ +- environment.py \ +- logging.py \ +- patterns.py \ +- remote.py \ +- watcher.py ++ctslibdir = $(pyexecdir)/cts ++ctslib_PYTHON = __init__.py \ ++ CIB.py \ ++ cib_xml.py \ ++ CM_lha.py \ ++ CM_ais.py \ ++ CTS.py \ ++ CTSaudits.py \ ++ CTSscenarios.py \ ++ CTStests.py \ ++ environment.py \ ++ logging.py \ ++ patterns.py \ ++ remote.py \ ++ watcher.py ++nodist_ctslib_PYTHON = CTSvars.py + +-cts_DATA = README.md cts.supp pacemaker-cts-dummyd.service +- +-cts_SCRIPTS = cts \ +- CTSlab.py \ +- lxc_autogen.sh \ +- LSBDummy \ +- HBDummy \ +- pacemaker-cts-dummyd \ +- $(top_srcdir)/fencing/fence_dummy ++ctsdir = $(testdir)/cts ++cts_DATA = pacemaker-cts-dummyd.service ++dist_cts_DATA = README.md cts.supp ++dist_cts_SCRIPTS = cts \ ++ CTSlab.py \ ++ $(top_srcdir)/fencing/fence_dummy ++cts_SCRIPTS = HBDummy \ ++ LSBDummy \ ++ lxc_autogen.sh \ ++ pacemaker-cts-dummyd + + SUBDIRS = benchmark +diff --git a/cts/benchmark/Makefile.am b/cts/benchmark/Makefile.am +index 8a50ac7..1fd6171 100644 +--- a/cts/benchmark/Makefile.am ++++ b/cts/benchmark/Makefile.am +@@ -3,26 +3,11 @@ + # + # Copyright (C) 2001 Michael Moerz + # +-# This program is free software; you can redistribute it and/or +-# modify it under the terms of the GNU General Public License +-# as published by the Free Software Foundation; either version 2 +-# of the License, or (at your option) any later version. +-# +-# This program is distributed in the hope that it will be useful, +-# but WITHOUT ANY WARRANTY; without even the implied warranty of +-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-# GNU General Public License for more details. +-# +-# You should have received a copy of the GNU General Public License +-# along with this program; if not, write to the Free Software +-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. ++# This source code is licensed under the GNU General Public License version 2 ++# or later (GPLv2+) WITHOUT ANY WARRANTY. + # + MAINTAINERCLEANFILES = Makefile.in + +-EXTRA_DIST = $(bench_SCRIPTS) $(bench_DATA) +- +-benchdir = $(datadir)/$(PACKAGE)/tests/cts/benchmark +- +-bench_DATA = README.benchmark control +- +-bench_SCRIPTS = clubench ++benchdir = $(datadir)/$(PACKAGE)/tests/cts/benchmark ++dist_bench_DATA = README.benchmark control ++bench_SCRIPTS = clubench +diff --git a/doc/Makefile.am b/doc/Makefile.am +index c981009..a929acb 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -56,11 +56,13 @@ UPLOAD_LANGS = en-US + + # Scheduler transition graphs + # @TODO Add original XML, and generate DOTs via crm_simulate +-DOTS = $(wildcard shared/en-US/images/*.dot) ++DOTS = $(wildcard Clusters_from_Scratch/en-US/images/*.dot) \ ++ $(wildcard Pacemaker_Explained/en-US/images/*.dot) + + # Vector sources for images + # @TODO Generate transition SVGs from DOTs via dot +-SVGS = $(wildcard shared/en-US/images/pcmk-*.svg) \ ++SVGS = $(wildcard Clusters_from_Scratch/en-US/images/pcmk-*.svg) \ ++ $(wildcard Pacemaker_Explained/en-US/images/pcmk-*.svg) \ + $(DOTS:%.dot=%.svg) + + # Final images +@@ -100,12 +102,13 @@ publican_docs += $(docbook) + endif + endif + +-EXTRA_DIST = $(ascii) $(SHARED_TXT) $(PNGS_ORIGINAL) $(SVGS) ++EXTRA_DIST = $(ascii) $(SHARED_TXT) $(PNGS_ORIGINAL) $(DOTS) $(SVGS) + EXTRA_DIST += $(CFS_TXT) $(CFS_XML_ONLY) + EXTRA_DIST += $(PA_TXT) $(PA_XML_ONLY) + EXTRA_DIST += $(PD_TXT) $(PD_XML_ONLY) + EXTRA_DIST += $(PE_TXT) $(PE_XML_ONLY) + EXTRA_DIST += $(PR_TXT) $(PR_XML_ONLY) ++EXTRA_DIST += pcs-crmsh-quick-ref.md + + %.html: %.txt + if IS_ASCIIDOC +diff --git a/extra/Makefile.am b/extra/Makefile.am +index d742ae2..fb23caf 100644 +--- a/extra/Makefile.am ++++ b/extra/Makefile.am +@@ -1,24 +1,15 @@ + # +-# Copyright (C) 2004-2009 Andrew Beekhof ++# Copyright 2004-2019 the Pacemaker project contributors + # +-# This program is free software; you can redistribute it and/or +-# modify it under the terms of the GNU General Public License +-# as published by the Free Software Foundation; either version 2 +-# of the License, or (at your option) any later version. +-# +-# This program is distributed in the hope that it will be useful, +-# but WITHOUT ANY WARRANTY; without even the implied warranty of +-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-# GNU General Public License for more details. +-# +-# You should have received a copy of the GNU General Public License +-# along with this program; if not, write to the Free Software +-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. ++# The version control history for this file may have further details. ++# ++# This source code is licensed under the GNU General Public License version 2 ++# or later (GPLv2+) WITHOUT ANY WARRANTY. + # + + MAINTAINERCLEANFILES = Makefile.in + +-SUBDIRS = alerts resources logrotate ++SUBDIRS = alerts resources logrotate + +-mibdir = $(datadir)/snmp/mibs +-mib_DATA = PCMK-MIB.txt ++mibdir = $(datadir)/snmp/mibs ++dist_mib_DATA = PCMK-MIB.txt +diff --git a/extra/alerts/Makefile.am b/extra/alerts/Makefile.am +index 2cd3bd6..e798ae9 100644 +--- a/extra/alerts/Makefile.am ++++ b/extra/alerts/Makefile.am +@@ -1,22 +1,15 @@ + # +-# Copyright (C) 2016 Ken Gaillot <kgaillot@redhat.com> ++# Copyright 2016-2019 the Pacemaker project contributors + # +-# This program is free software; you can redistribute it and/or +-# modify it under the terms of the GNU General Public License +-# as published by the Free Software Foundation; either version 2 +-# of the License, or (at your option) any later version. ++# The version control history for this file may have further details. + # +-# This program is distributed in the hope that it will be useful, +-# but WITHOUT ANY WARRANTY; without even the implied warranty of +-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-# GNU General Public License for more details. +-# +-# You should have received a copy of the GNU General Public License +-# along with this program; if not, write to the Free Software +-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. ++# This source code is licensed under the GNU General Public License version 2 ++# or later (GPLv2+) WITHOUT ANY WARRANTY. + # + + MAINTAINERCLEANFILES = Makefile.in + +-samplesdir = $(datadir)/$(PACKAGE)/alerts/ +-samples_DATA = alert_file.sh.sample alert_smtp.sh.sample alert_snmp.sh.sample ++samplesdir = $(datadir)/$(PACKAGE)/alerts/ ++dist_samples_DATA = alert_file.sh.sample \ ++ alert_smtp.sh.sample \ ++ alert_snmp.sh.sample +diff --git a/extra/logrotate/Makefile.am b/extra/logrotate/Makefile.am +index 55c669c..cafd0d5 100644 +--- a/extra/logrotate/Makefile.am ++++ b/extra/logrotate/Makefile.am +@@ -1,22 +1,13 @@ + # +-# Copyright (C) 2014 Gao,Yan <ygao@suse.com> ++# Copyright 2014-2019 the Pacemaker project contributors + # +-# This program is free software; you can redistribute it and/or +-# modify it under the terms of the GNU General Public License +-# as published by the Free Software Foundation; either version 2 +-# of the License, or (at your option) any later version. ++# The version control history for this file may have further details. + # +-# This program is distributed in the hope that it will be useful, +-# but WITHOUT ANY WARRANTY; without even the implied warranty of +-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-# GNU General Public License for more details. +-# +-# You should have received a copy of the GNU General Public License +-# along with this program; if not, write to the Free Software +-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. ++# This source code is licensed under the GNU General Public License version 2 ++# or later (GPLv2+) WITHOUT ANY WARRANTY. + # + + MAINTAINERCLEANFILES = Makefile.in + +-logrotatedir = $(sysconfdir)/logrotate.d +-logrotate_DATA = pacemaker ++logrotatedir = $(sysconfdir)/logrotate.d ++logrotate_DATA = pacemaker +diff --git a/extra/resources/Makefile.am b/extra/resources/Makefile.am +index c84dfdf..e4b54cc 100644 +--- a/extra/resources/Makefile.am ++++ b/extra/resources/Makefile.am +@@ -1,22 +1,12 @@ +-# Makefile.am for OCF RAs + # +-# Author: Andrew Beekhof +-# Copyright (C) 2008 Andrew Beekhof ++# Copyright 2008-2019 the Pacemaker project contributors + # +-# This program is free software; you can redistribute it and/or +-# modify it under the terms of the GNU General Public License +-# as published by the Free Software Foundation; either version 2 +-# of the License, or (at your option) any later version. +-# +-# This program is distributed in the hope that it will be useful, +-# but WITHOUT ANY WARRANTY; without even the implied warranty of +-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-# GNU General Public License for more details. +-# +-# You should have received a copy of the GNU General Public License +-# along with this program; if not, write to the Free Software +-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. ++# The version control history for this file may have further details. + # ++# This source code is licensed under the GNU General Public License version 2 ++# or later (GPLv2+) WITHOUT ANY WARRANTY. ++# ++ + include $(top_srcdir)/Makefile.common + + EXTRA_DIST = $(ocf_SCRIPTS) +@@ -24,28 +14,27 @@ EXTRA_DIST = $(ocf_SCRIPTS) + + isolationtechdir = @OCF_RA_DIR@/.isolation + +-ocfdir = @OCF_RA_DIR@/pacemaker +- +-ocf_SCRIPTS = ClusterMon \ +- controld \ +- Dummy \ +- HealthCPU \ +- HealthSMART \ +- ifspeed \ +- o2cb \ +- ping \ +- pingd \ +- Stateful \ +- SysInfo \ +- SystemHealth \ +- attribute \ +- remote +- +-isolationtech_SCRIPTS = docker-wrapper ++ocfdir = @OCF_RA_DIR@/pacemaker ++dist_ocf_SCRIPTS = attribute \ ++ ClusterMon \ ++ controld \ ++ Dummy \ ++ HealthCPU \ ++ HealthSMART \ ++ ifspeed \ ++ o2cb \ ++ ping \ ++ pingd \ ++ remote \ ++ Stateful \ ++ SysInfo \ ++ SystemHealth ++ ++dist_isolationtech_SCRIPTS = docker-wrapper + + if BUILD_XML_HELP + +-man7_MANS = $(ocf_SCRIPTS:%=ocf_pacemaker_%.7) ++man7_MANS = $(ocf_SCRIPTS:%=ocf_pacemaker_%.7) $(dist_ocf_SCRIPTS:%=ocf_pacemaker_%.7) + DBOOK_OPTS = --stringparam command.prefix ocf_pacemaker_ --stringparam variable.prefix OCF_RESKEY_ --param man.vol 7 + + ocf_pacemaker_%.xml: % +@@ -54,5 +43,4 @@ ocf_pacemaker_%.xml: % + endif + + clean-generic: +- rm -f $(man7_MANS) $(ocf_SCRIPTS:%=%.xml) *~ +- ++ rm -f $(man7_MANS) $(ocf_SCRIPTS:%=%.xml) $(dist_ocf_SCRIPTS:%=%.xml) *~ +diff --git a/fencing/Makefile.am b/fencing/Makefile.am +index cb7b551..6cda8ef 100644 +--- a/fencing/Makefile.am ++++ b/fencing/Makefile.am +@@ -25,7 +25,7 @@ halibdir = $(CRM_DAEMON_DIR) + halib_PROGRAMS = stonithd stonith-test + + sbin_PROGRAMS = stonith_admin +-sbin_SCRIPTS = fence_legacy fence_pcmk ++dist_sbin_SCRIPTS = fence_legacy fence_pcmk + + noinst_HEADERS = internal.h standalone_config.h + +@@ -33,6 +33,8 @@ if BUILD_XML_HELP + man7_MANS = stonithd.7 + endif + ++EXTRA_DIST = README.md ++ + stonith_test_SOURCES = test.c + + stonith_test_LDADD = $(top_builddir)/lib/common/libcrmcommon.la \ +diff --git a/lrmd/Makefile.am b/lrmd/Makefile.am +index 33611cb..7aa5414 100644 +--- a/lrmd/Makefile.am ++++ b/lrmd/Makefile.am +@@ -26,7 +26,7 @@ testdir = $(datadir)/$(PACKAGE)/tests/lrmd + test_SCRIPTS = regression.py + + initdir = $(INITDIR) +-init_SCRIPTS = pacemaker_remote ++dist_init_SCRIPTS = pacemaker_remote + sbin_PROGRAMS = pacemaker_remoted + + if BUILD_SYSTEMD +diff --git a/mcp/Makefile.am b/mcp/Makefile.am +index 074d251..5e1147d 100644 +--- a/mcp/Makefile.am ++++ b/mcp/Makefile.am +@@ -20,13 +20,15 @@ include $(top_srcdir)/Makefile.common + if BUILD_CS_SUPPORT + + initdir = $(INITDIR) +-init_SCRIPTS = pacemaker ++dist_init_SCRIPTS = pacemaker + sbin_PROGRAMS = pacemakerd + + if BUILD_SYSTEMD + systemdunit_DATA = pacemaker.service + endif + ++EXTRA_DIST = pacemaker.sysconfig ++ + ## SOURCES + + noinst_HEADERS = pacemaker.h +diff --git a/pengine/Makefile.am b/pengine/Makefile.am +index fdac3e3..c121ab5 100644 +--- a/pengine/Makefile.am ++++ b/pengine/Makefile.am +@@ -24,11 +24,11 @@ halibdir = $(CRM_DAEMON_DIR) + PE_TESTS = $(wildcard test10/*.scores) + + testdir = $(datadir)/$(PACKAGE)/tests/pengine +-test_SCRIPTS = regression.sh +-test_DATA = regression.core.sh ++dist_test_SCRIPTS = regression.sh ++dist_test_DATA = regression.core.sh + + test10dir = $(datadir)/$(PACKAGE)/tests/pengine/test10 +-test10_DATA = $(PE_TESTS) $(PE_TESTS:%.scores=%.xml) $(PE_TESTS:%.scores=%.exp) $(PE_TESTS:%.scores=%.dot) $(PE_TESTS:%.scores=%.summary) $(wildcard test10/*.stderr) ++dist_test10_DATA = $(PE_TESTS) $(PE_TESTS:%.scores=%.xml) $(PE_TESTS:%.scores=%.exp) $(PE_TESTS:%.scores=%.dot) $(PE_TESTS:%.scores=%.summary) $(wildcard test10/*.stderr) + + beekhof: + echo $(shell ls -1 test10/*.xml) +diff --git a/tools/Makefile.am b/tools/Makefile.am +index d8c3215..6960548 100644 +--- a/tools/Makefile.am ++++ b/tools/Makefile.am +@@ -31,18 +31,20 @@ noinst_HEADERS = crm_resource.h fake_transition.h + pcmkdir = $(datadir)/$(PACKAGE) + pcmk_DATA = report.common report.collector + +-sbin_SCRIPTS = crm_report crm_standby crm_master crm_failcount ++sbin_SCRIPTS = crm_report + if BUILD_CIBSECRETS + sbin_SCRIPTS += cibsecret + endif +-EXTRA_DIST = $(sbin_SCRIPTS) ++dist_sbin_SCRIPTS = crm_standby crm_master crm_failcount ++ ++EXTRA_DIST = crm_mon.sysconfig + + sbin_PROGRAMS = crm_simulate crmadmin cibadmin crm_node crm_attribute crm_resource crm_verify \ + crm_shadow attrd_updater crm_diff crm_mon iso8601 crm_ticket crm_error + + testdir = $(datadir)/$(PACKAGE)/tests/cli +-test_SCRIPTS = regression.sh +-test_DATA = regression.dates.exp \ ++dist_test_SCRIPTS = regression.sh ++dist_test_DATA = regression.dates.exp \ + regression.tools.exp \ + regression.acls.exp \ + regression.validity.exp \ +@@ -102,7 +104,6 @@ crm_mon_LDADD = $(top_builddir)/lib/pengine/libpe_status.la \ + $(top_builddir)/pengine/libpengine.la \ + $(COMMONLIBS) $(SNMPLIBS) $(ESMTPLIBS) + +-# Arguments could be made that this should live in crm/pengine + crm_verify_SOURCES = crm_verify.c + crm_verify_LDADD = $(top_builddir)/lib/pengine/libpe_status.la \ + $(top_builddir)/pengine/libpengine.la \ +diff --git a/xml/Makefile.am b/xml/Makefile.am +index 1e5649f..c801842 100644 +--- a/xml/Makefile.am ++++ b/xml/Makefile.am +@@ -18,12 +18,11 @@ + MAINTAINERCLEANFILES = Makefile.in + + dtddir = $(CRM_DTD_DIRECTORY) +-dtd_DATA = crm.dtd crm-transitional.dtd ++dist_dtd_DATA = crm.dtd crm-transitional.dtd + + xsltdir = $(dtddir) +-xslt_DATA = $(top_srcdir)/xml/upgrade06.xsl $(top_srcdir)/xml/upgrade-*.xsl +- +-noinst_DATA = context-of.xsl ++dist_xslt_DATA = $(top_srcdir)/xml/upgrade06.xsl \ ++ $(top_srcdir)/xml/upgrade-*.xsl + + RNGdir = $(dtddir) + +@@ -68,7 +67,12 @@ RNG_extra = crm_mon.rng + dist_RNG_DATA = $(RNG_files) $(RNG_extra) + nodist_RNG_DATA = $(RNG_generated) + +-EXTRA_DIST = best-match.sh ++EXTRA_DIST = Readme.md \ ++ best-match.sh \ ++ context-of.xsl \ ++ ocf-meta2man.xsl \ ++ regression.core.sh \ ++ regression.sh + + versions: + echo "Max: $(RNG_max)" +-- +1.8.3.1 + + +From 50825612a2d3922fbeabef390303ca2ac596e846 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Mon, 5 Aug 2019 17:38:42 -0500 +Subject: [PATCH 81/96] Build: GNUmakefile: allow all/clean to work without git + +e.g. in a distribution rather than a checkout +--- + GNUmakefile | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/GNUmakefile b/GNUmakefile +index 352903f..d790865 100644 +--- a/GNUmakefile ++++ b/GNUmakefile +@@ -45,12 +45,12 @@ ARCH ?= $(shell test -e /etc/fedora-release && rpm --eval %{_arch}) + MOCK_CFG ?= $(shell test -e /etc/fedora-release && echo fedora-$(F)-$(ARCH)) + DISTRO ?= $(shell test -e /etc/SuSE-release && echo suse; echo fedora) + COMMIT ?= HEAD +-TAG ?= $(shell T=$$(git describe --all '$(COMMIT)' | sed -n 's|tags/\(.*\)|\1|p'); \ ++TAG ?= $(shell T=$$(git describe --all '$(COMMIT)' 2>/dev/null | sed -n 's|tags/\(.*\)|\1|p'); \ + test -n "$${T}" && echo "$${T}" \ +- || git log --pretty=format:%H -n 1 '$(COMMIT)') ++ || git log --pretty=format:%H -n 1 '$(COMMIT)' 2>/dev/null || echo DIST) + lparen = ( + rparen = ) +-SHORTTAG ?= $(shell case $(TAG) in Pacemaker-*$(rparen) echo '$(TAG)' | cut -c11-;; \ ++SHORTTAG ?= $(shell case $(TAG) in Pacemaker-*|DIST$(rparen) echo '$(TAG)' | cut -c11-;; \ + *$(rparen) git log --pretty=format:%h -n 1 '$(TAG)';; esac) + SHORTTAG_ABBREV = $(shell printf %s '$(SHORTTAG)' | wc -c) + WITH ?= --without doc +-- +1.8.3.1 + + +From d545f94c95dbf9c205b458aee6bdff49fa9c3dd9 Mon Sep 17 00:00:00 2001 +From: Chris Lumens <clumens@redhat.com> +Date: Thu, 28 Feb 2019 10:28:35 -0500 +Subject: [PATCH 82/96] Feature: xml: Add a schema for API results. + +This describes the layout of all the XML that will be emitted by the +formatted output patches. That output can be validated against the +schema with xmllint. + +[small portion of commit backported to 1.1] +--- + xml/Makefile.am | 31 ++++++++++++++++++------------- + 1 file changed, 18 insertions(+), 13 deletions(-) + +diff --git a/xml/Makefile.am b/xml/Makefile.am +index c801842..09d3503 100644 +--- a/xml/Makefile.am ++++ b/xml/Makefile.am +@@ -30,31 +30,36 @@ RNGdir = $(dtddir) + + # Sorted list of available numeric RNG versions, + # extracted from filenames like NAME-MAJOR[.MINOR][.MINOR-MINOR].rng +-RNG_numeric_versions = $(shell ls -1 $(top_srcdir)/xml/*.rng \ ++numeric_versions = $(shell ls -1 $(1) \ + | sed -n -e 's/^.*-\([0-9][0-9.]*\).rng$$/\1/p' \ + | sort -u -t. -k 1,1n -k 2,2n -k 3,3n) + +-# The highest numeric version +-RNG_max ?= $(lastword $(RNG_numeric_versions)) +- +-# A sorted list of all RNG versions (numeric and "next") +-RNG_versions = next $(RNG_numeric_versions) +-RNG_version_pairs = $(join \ +- ${RNG_numeric_versions},$(addprefix \ ++version_pairs = $(join \ ++ $(1),$(addprefix \ + -,$(wordlist \ +- 2,$(words ${RNG_numeric_versions}),${RNG_numeric_versions} \ ++ 2,$(words $(1)),$(1) \ + ) next \ + ) \ + ) +-RNG_version_pairs_cnt = $(words ${RNG_version_pairs}) +-RNG_version_pairs_last = $(wordlist \ ++ ++version_pairs_last = $(wordlist \ + $(words \ + $(wordlist \ +- 2,${RNG_version_pairs_cnt},${RNG_version_pairs} \ ++ 2,$(1),$(2) \ + ) \ +- ),${RNG_version_pairs_cnt},${RNG_version_pairs} \ ++ ),$(1),$(2) \ + ) + ++RNG_numeric_versions = $(call numeric_versions,${RNG_files}) ++ ++# The highest numeric version ++RNG_max ?= $(lastword $(RNG_numeric_versions)) ++ ++RNG_versions = next $(RNG_numeric_versions) ++RNG_version_pairs = $(call version_pairs,${RNG_numeric_versions}) ++RNG_version_pairs_cnt = $(words ${RNG_version_pairs}) ++RNG_version_pairs_last = $(call version_pairs_last,${RNG_version_pairs_cnt},${RNG_version_pairs}) ++ + RNG_generated = pacemaker.rng $(foreach base,$(RNG_versions),pacemaker-$(base).rng) versions.rng + + RNG_cfg_base = options nodes resources constraints fencing acls tags alerts +-- +1.8.3.1 + + +From fd3587f67b405c08473a636554f4822e72149495 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 2 Aug 2019 14:41:25 -0500 +Subject: [PATCH 83/96] Build: xml: remove broken and unneeded "make sync" + target + +--- + xml/Makefile.am | 4 ---- + 1 file changed, 4 deletions(-) + +diff --git a/xml/Makefile.am b/xml/Makefile.am +index 09d3503..5734979 100644 +--- a/xml/Makefile.am ++++ b/xml/Makefile.am +@@ -178,8 +178,4 @@ fulldiff: best-match.sh + @echo "# Comparing all changes across all the subsequent increments" + $(call version_diff,${RNG_version_pairs}) + +-sync: +- git rm -f $(wildcard *-next.rng) +- make pacemaker-next.rng +- + CLEANFILES = $(RNG_generated) +-- +1.8.3.1 + + +From 2e8384cef56365c187c3b80719cd49f72d66b8e4 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Tue, 6 Aug 2019 14:52:20 -0500 +Subject: [PATCH 84/96] Build: xml: rearrange and comment Makefile for + readability + +Similarly, rename some variables, and rename the versions target to +cib-versions. +--- + xml/Makefile.am | 147 +++++++++++++++++++++++++++++--------------------------- + 1 file changed, 75 insertions(+), 72 deletions(-) + +diff --git a/xml/Makefile.am b/xml/Makefile.am +index 5734979..8ff805b 100644 +--- a/xml/Makefile.am ++++ b/xml/Makefile.am +@@ -1,35 +1,22 @@ + # + # Copyright (C) 2004 Andrew Beekhof + # +-# This program is free software; you can redistribute it and/or +-# modify it under the terms of the GNU General Public License +-# as published by the Free Software Foundation; either version 2 +-# of the License, or (at your option) any later version. +-# +-# This program is distributed in the hope that it will be useful, +-# but WITHOUT ANY WARRANTY; without even the implied warranty of +-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-# GNU General Public License for more details. +-# +-# You should have received a copy of the GNU General Public License +-# along with this program; if not, write to the Free Software +-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. ++# This source code is licensed under the GNU General Public License version 2 ++# or later (GPLv2+) WITHOUT ANY WARRANTY. + # + MAINTAINERCLEANFILES = Makefile.in + +-dtddir = $(CRM_DTD_DIRECTORY) +-dist_dtd_DATA = crm.dtd crm-transitional.dtd +- +-xsltdir = $(dtddir) +-dist_xslt_DATA = $(top_srcdir)/xml/upgrade06.xsl \ +- $(top_srcdir)/xml/upgrade-*.xsl +- +-RNGdir = $(dtddir) ++# Pacemaker 1.1 has 2 schemas: the CIB schema, and a schema for ++# crm_mon --as-xml. ++# ++# See Readme.md for details on updating CIB schema files + +-# See Readme.md for details on updating schema files ++# The CIB and crm_mon schemas are installed directly in CRM_DTD_DIRECTORY. ++CIBdir = $(CRM_DTD_DIRECTORY) ++MONdir = $(CRM_DTD_DIRECTORY) + +-# Sorted list of available numeric RNG versions, +-# extracted from filenames like NAME-MAJOR[.MINOR][.MINOR-MINOR].rng ++# Extract a sorted list of available numeric schema versions ++# from filenames like NAME-MAJOR[.MINOR][.MINOR-MINOR].rng + numeric_versions = $(shell ls -1 $(1) \ + | sed -n -e 's/^.*-\([0-9][0-9.]*\).rng$$/\1/p' \ + | sort -u -t. -k 1,1n -k 2,2n -k 3,3n) +@@ -50,27 +37,40 @@ version_pairs_last = $(wordlist \ + ),$(1),$(2) \ + ) + +-RNG_numeric_versions = $(call numeric_versions,${RNG_files}) ++# Names of CIB schemas that form the choices for cib/configuration content ++CIB_cfg_base = options nodes resources constraints fencing acls tags alerts ++ ++# Names of all schemas (including top level and those included by others) ++CIB_base = cib $(CIB_cfg_base) status score rule nvset ++ ++# All static schema files ++CIB_files = $(foreach base,$(CIB_base),$(wildcard $(base).rng $(base)-*.rng)) ++MON_files = crm_mon.rng ++ ++# Sorted lists of all numeric schema versions ++CIB_numeric_versions = $(call numeric_versions,${CIB_files}) + +-# The highest numeric version +-RNG_max ?= $(lastword $(RNG_numeric_versions)) ++# The highest numeric schema version ++CIB_max ?= $(lastword $(CIB_numeric_versions)) + +-RNG_versions = next $(RNG_numeric_versions) +-RNG_version_pairs = $(call version_pairs,${RNG_numeric_versions}) +-RNG_version_pairs_cnt = $(words ${RNG_version_pairs}) +-RNG_version_pairs_last = $(call version_pairs_last,${RNG_version_pairs_cnt},${RNG_version_pairs}) ++# Sorted lists of all schema versions (including "next") ++CIB_versions = next $(CIB_numeric_versions) + +-RNG_generated = pacemaker.rng $(foreach base,$(RNG_versions),pacemaker-$(base).rng) versions.rng ++# Dynamically generated schema files ++CIB_generated = pacemaker.rng $(foreach base,$(CIB_versions),pacemaker-$(base).rng) versions.rng + +-RNG_cfg_base = options nodes resources constraints fencing acls tags alerts +-RNG_base = cib $(RNG_cfg_base) status score rule nvset +-RNG_files = $(foreach base,$(RNG_base),$(wildcard $(base).rng $(base)-*.rng)) ++CIB_version_pairs = $(call version_pairs,${CIB_numeric_versions}) ++CIB_version_pairs_cnt = $(words ${CIB_version_pairs}) ++CIB_version_pairs_last = $(call version_pairs_last,${CIB_version_pairs_cnt},${CIB_version_pairs}) + +-# List of non-Pacemaker RNGs +-RNG_extra = crm_mon.rng ++dist_CIB_DATA = $(CIB_files) \ ++ upgrade06.xsl \ ++ upgrade-1.3.xsl \ ++ crm.dtd \ ++ crm-transitional.dtd ++dist_MON_DATA = $(MON_files) + +-dist_RNG_DATA = $(RNG_files) $(RNG_extra) +-nodist_RNG_DATA = $(RNG_generated) ++nodist_CIB_DATA = $(CIB_generated) + + EXTRA_DIST = Readme.md \ + best-match.sh \ +@@ -79,10 +79,38 @@ EXTRA_DIST = Readme.md \ + regression.core.sh \ + regression.sh + +-versions: +- echo "Max: $(RNG_max)" +- echo "Available: $(RNG_versions)" ++cib-versions: ++ @echo "Max: $(CIB_max)" ++ @echo "Available: $(CIB_versions)" + ++ ++# Dynamically generated top-level CIB schema ++pacemaker.rng: pacemaker-$(CIB_max).rng ++ echo " RNG $@" ++ cp $(top_builddir)/xml/$< $@ ++ ++pacemaker-%.rng: $(CIB_files) best-match.sh Makefile.am ++ echo " RNG $@" ++ echo '<?xml version="1.0" encoding="UTF-8"?>' > $@ ++ echo '<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">' >> $@ ++ echo ' <start>' >> $@ ++ echo ' <element name="cib">' >> $@ ++ $(srcdir)/best-match.sh cib $(*) $(@) " " ++ echo ' <element name="configuration">' >> $@ ++ echo ' <interleave>' >> $@ ++ for rng in $(CIB_cfg_base); do $(srcdir)/best-match.sh $$rng $(*) $(@) " " || :; done ++ echo ' </interleave>' >> $@ ++ echo ' </element>' >> $@ ++ echo ' <optional>' >> $@ ++ echo ' <element name="status">' >> $@ ++ $(srcdir)/best-match.sh status $(*) $(@) " " ++ echo ' </element>' >> $@ ++ echo ' </optional>' >> $@ ++ echo ' </element>' >> $@ ++ echo ' </start>' >> $@ ++ echo '</grammar>' >> $@ ++ ++# Dynamically generated CIB schema listing all pacemaker versions + versions.rng: Makefile.am + echo " RNG $@" + echo '<?xml version="1.0" encoding="UTF-8"?>' > $@ +@@ -97,7 +125,7 @@ versions.rng: Makefile.am + echo ' <value>transitional-0.6</value>' >> $@ + echo ' <value>pacemaker-0.7</value>' >> $@ + echo ' <value>pacemaker-1.1</value>' >> $@ +- for rng in $(RNG_versions); do echo " <value>pacemaker-$$rng</value>" >> $@; done ++ for rng in $(CIB_versions); do echo " <value>pacemaker-$$rng</value>" >> $@; done + echo ' </choice>' >> $@ + echo ' </attribute>' >> $@ + echo ' </optional>' >> $@ +@@ -108,31 +136,6 @@ versions.rng: Makefile.am + echo ' </start>' >> $@ + echo '</grammar>' >> $@ + +-pacemaker.rng: pacemaker-$(RNG_max).rng +- echo " RNG $@" +- cp $(top_builddir)/xml/$< $@ +- +-pacemaker-%.rng: $(RNG_files) best-match.sh Makefile.am +- echo " RNG $@" +- echo '<?xml version="1.0" encoding="UTF-8"?>' > $@ +- echo '<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">' >> $@ +- echo ' <start>' >> $@ +- echo ' <element name="cib">' >> $@ +- $(top_srcdir)/xml/best-match.sh cib $(*) $(@) " " +- echo ' <element name="configuration">' >> $@ +- echo ' <interleave>' >> $@ +- for rng in $(RNG_cfg_base); do $(top_srcdir)/xml/best-match.sh $$rng $(*) $(@) " " || :; done +- echo ' </interleave>' >> $@ +- echo ' </element>' >> $@ +- echo ' <optional>' >> $@ +- echo ' <element name="status">' >> $@ +- $(top_srcdir)/xml/best-match.sh status $(*) $(@) " " +- echo ' </element>' >> $@ +- echo ' </optional>' >> $@ +- echo ' </element>' >> $@ +- echo ' </start>' >> $@ +- echo '</grammar>' >> $@ +- + # diff fails with ec=2 if no predecessor is found; + # this uses '=' GNU extension to sed, if that's not available, + # one can use: hline=`echo "$${p}" | grep -Fn "$${hunk}" | cut -d: -f1`; +@@ -171,11 +174,11 @@ version_diff = \ + done + + diff: best-match.sh +- @echo "# Comparing changes in + since $(RNG_max)" +- $(call version_diff,${RNG_version_pairs_last}) ++ @echo "# Comparing changes in + since $(CIB_max)" ++ $(call version_diff,${CIB_version_pairs_last}) + + fulldiff: best-match.sh + @echo "# Comparing all changes across all the subsequent increments" +- $(call version_diff,${RNG_version_pairs}) ++ $(call version_diff,${CIB_version_pairs}) + +-CLEANFILES = $(RNG_generated) ++CLEANFILES = $(CIB_generated) +-- +1.8.3.1 + + +From da2dd7c8aef6b19fe7b45a0df6ec2d042a3d0049 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Mon, 29 Jul 2019 18:33:29 -0500 +Subject: [PATCH 85/96] Build: xml: make schema files work with VPATH builds + +a.k.a pristine builds, where the source and build directories are different +--- + xml/Makefile.am | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/xml/Makefile.am b/xml/Makefile.am +index 8ff805b..972750f 100644 +--- a/xml/Makefile.am ++++ b/xml/Makefile.am +@@ -44,7 +44,7 @@ CIB_cfg_base = options nodes resources constraints fencing acls tags alerts + CIB_base = cib $(CIB_cfg_base) status score rule nvset + + # All static schema files +-CIB_files = $(foreach base,$(CIB_base),$(wildcard $(base).rng $(base)-*.rng)) ++CIB_files = $(foreach base,$(CIB_base),$(wildcard $(srcdir)/$(base).rng $(srcdir)/$(base)-*.rng)) + MON_files = crm_mon.rng + + # Sorted lists of all numeric schema versions +-- +1.8.3.1 + + +From a08ebb7599992a783dadb7cf74f83e127c29f8cc Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Tue, 30 Jul 2019 12:27:06 -0500 +Subject: [PATCH 86/96] Build: doc: make HTML documents compatible with VPATH + builds + +--- + doc/Makefile.am | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/doc/Makefile.am b/doc/Makefile.am +index a929acb..6c2a3c7 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -110,13 +110,15 @@ EXTRA_DIST += $(PE_TXT) $(PE_XML_ONLY) + EXTRA_DIST += $(PR_TXT) $(PR_XML_ONLY) + EXTRA_DIST += pcs-crmsh-quick-ref.md + +-%.html: %.txt + if IS_ASCIIDOC +- $(AM_V_ASCII)$(ASCIIDOC_CONV) --unsafe --backend=xhtml11 $< ++ASCIIDOC_HTML_ARGS = --unsafe --backend=xhtml11 + else +- $(AM_V_ASCII)$(ASCIIDOC_CONV) --backend=html5 $< ++ASCIIDOC_HTML_ARGS = --backend=html5 + endif + ++%.html: %.txt ++ $(AM_V_ASCII)$(ASCIIDOC_CONV) $(ASCIIDOC_HTML_ARGS) --out-file=$@ $< ++ + # publican-clusterlabs/xsl/{html,html-single,pdf}.xsl refer to URIs + # requiring Internet access, hence we shadow that with a XML catalog-based + # redirect to local files brought with Publican installation; +-- +1.8.3.1 + + +From 736051ab087c97372891591b0aa907b5d7c1f2dc Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 1 Aug 2019 10:44:26 -0500 +Subject: [PATCH 87/96] Build: makefiles: simplify silent rules usage + +make output is terser with --enable-silent-rules at configure time or make V=0 +at make run-time. This simplifies our handling of such silent rules, mainly by +using AM_V_at and AM_V_GEN appropriately. + +This gets rid of AM_V_IMG (which we never defined) and AM_V_ASCII and AM_V_XSL +(which weren't particularly useful), and adds AM_V_SCHEMA (to replace the +half-hearted attempt at RNG handling). Our PCMK_quiet now just silences stdout, +not stderr. +--- + Makefile.common | 52 ++++++++++++++++++++-------------- + doc/Makefile.am | 18 ++++++------ + xml/Makefile.am | 88 ++++++++++++++++++++++++++++----------------------------- + 3 files changed, 83 insertions(+), 75 deletions(-) + +diff --git a/Makefile.common b/Makefile.common +index a4842b7..386d59d 100644 +--- a/Makefile.common ++++ b/Makefile.common +@@ -1,27 +1,37 @@ +-# Not all current distros support AM_V_P ++# ++# Copyright 2014-2019 the Pacemaker project contributors ++# ++# The version control history for this file may have further details. ++# ++# This source code is licensed under the GNU General Public License version 2 ++# or later (GPLv2+) WITHOUT ANY WARRANTY. ++# ++ ++# ++# Some variables to help with silent rules + # https://www.gnu.org/software/automake/manual/html_node/Automake-silent_002drules-Option.html ++# ++# We require a minimum automake version of 1.11, which includes AM_V_GEN and ++# AM_V_at, but AM_V_P is not available until 1.13. + + V ?= $(AM_DEFAULT_VERBOSITY) + +-PCMK_V = $(pcmk__v_$(V)) +-pcmk__v_0 = : +-pcmk__v_1 = +- ++# When a make command is prefixed with one of the AM_V_* macros, it may also be ++# desirable to suffix the command with this, to silence stdout. + PCMK_quiet = $(pcmk_quiet_$(V)) +-pcmk_quiet_0 = >/dev/null 2>&1 ++pcmk_quiet_0 = >/dev/null + pcmk_quiet_1 = + +-AM_V_XSL = $(am__v_XSL_$(V)) +-am__v_XSL_0 = @echo " XSL " $@; +-am__v_XSL_1 = +- ++# AM_V_GEN is intended to be used in custom pattern rules, and replaces echoing ++# the command used with a more concise line with "GEN" and the name of the file ++# being generated. Our AM_V_* macros are similar but more descriptive. + AM_V_MAN = $(am__v_MAN_$(V)) +-am__v_MAN_0 = @echo " MAN " $@; ++am__v_MAN_0 = @echo " MAN $@"; + am__v_MAN_1 = + +-AM_V_ASCII = $(am__v_ASCII_$(V)) +-am__v_ASCII_0 = @echo " ASCII " $@; +-am__v_ASCII_1 = ++AM_V_SCHEMA = $(am__v_SCHEMA_$(V)) ++am__v_SCHEMA_0 = @echo " SCHEMA $@"; ++am__v_SCHEMA_1 = + + AM_V_PUB = $(am__v_PUB_$(V)) + am__v_PUB_0 = @echo " PUB $@: $(DOCBOOK_FORMATS)"; +@@ -37,18 +47,18 @@ man8_MANS = $(sbin_PROGRAMS:%=%.8) $(sbin_SCRIPTS:%=%.8) $(dist_sbin_SCRIPTS:%= + endif + + %.8: % $(MAN8DEPS) +- chmod a+x $(abs_builddir)/$< ++ $(AM_V_at)chmod a+x $(abs_builddir)/$< + $(AM_V_MAN)PATH=$(abs_builddir):$$PATH $(HELP2MAN) --output $@ --no-info --section 8 --name "Part of the Pacemaker cluster resource manager" $(abs_builddir)/$< + + %.xml: % +- $(AM_V_GEN)$(abs_builddir)/$< metadata > $@ ++ $(AM_V_at)$(abs_builddir)/$< metadata > $@ + + %.dbook: %.xml +- $(AM_V_XSL)$(XSLTPROC) --nonet --novalid --stringparam man.name $* $(DBOOK_OPTS) $(top_srcdir)/xml/ocf-meta2man.xsl $(abs_builddir)/$< > $(abs_builddir)/$@ ++ $(AM_V_at)$(XSLTPROC) --nonet --novalid --stringparam man.name $* $(DBOOK_OPTS) $(top_srcdir)/xml/ocf-meta2man.xsl $(abs_builddir)/$< > $(abs_builddir)/$@ + + %.7: %.dbook +- $(AM_V_XSL)$(XSLTPROC) $(MANPAGE_XSLT) $(abs_builddir)/$< $(PCMK_quiet) +- ++ $(AM_V_MAN)$(XSLTPROC) $(MANPAGE_XSLT) $(abs_builddir)/$< $(PCMK_quiet) ++# + # Build docbook from asciidoc because XML is a PITA to edit + # + # Build each chapter as a book (since the numbering isn't right for +@@ -59,9 +69,9 @@ endif + # + %.xml: %.txt + if IS_ASCIIDOC +- $(AM_V_ASCII)$(ASCIIDOC_CONV) -b docbook -d book -o $@-tt $< ++ $(AM_V_GEN)$(ASCIIDOC_CONV) -b docbook -d book -o $@-tt $< + else +- $(AM_V_ASCII)$(ASCIIDOC_CONV) -b docbook45 -d book -o $@-tt $< ++ $(AM_V_GEN)$(ASCIIDOC_CONV) -b docbook45 -d book -o $@-tt $< + endif + $(AM_V_at)tr -d '\036\r' <$@-tt >$@-t; rm -f $@-tt # Fix line endings + $(AM_V_at)sed -i 's/\ lang="en"//' $@-t # Never specify a language in the chapters +diff --git a/doc/Makefile.am b/doc/Makefile.am +index 6c2a3c7..a01423d 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -86,13 +86,13 @@ PNGS = $(PNGS_ORIGINAL) $(PNGS_GENERATED) + graphics: $(PNGS) + + %.png: %.svg +- $(AM_V_IMG)$(INKSCAPE) --file=$< --export-dpi=90 -C --export-png=$@ ++ $(AM_V_GEN)$(INKSCAPE) --file=$< --export-dpi=90 -C --export-png=$@ $(PCMK_quiet) + + %-small.png: %.svg +- $(AM_V_IMG)$(INKSCAPE) --file=$< --export-dpi=45 -C --export-png=$@ ++ $(AM_V_GEN)$(INKSCAPE) --file=$< --export-dpi=45 -C --export-png=$@ $(PCMK_quiet) + + %-large.png: %.svg +- $(AM_V_IMG)$(INKSCAPE) --file=$< --export-dpi=180 -C --export-png=$@ ++ $(AM_V_GEN)$(INKSCAPE) --file=$< --export-dpi=180 -C --export-png=$@ $(PCMK_quiet) + + if BUILD_ASCIIDOC + generated_docs += $(ascii:%.txt=%.html) +@@ -117,7 +117,7 @@ ASCIIDOC_HTML_ARGS = --backend=html5 + endif + + %.html: %.txt +- $(AM_V_ASCII)$(ASCIIDOC_CONV) $(ASCIIDOC_HTML_ARGS) --out-file=$@ $< ++ $(AM_V_GEN)$(ASCIIDOC_CONV) $(ASCIIDOC_HTML_ARGS) --out-file=$@ $< $(PCMK_quiet) + + # publican-clusterlabs/xsl/{html,html-single,pdf}.xsl refer to URIs + # requiring Internet access, hence we shadow that with a XML catalog-based +@@ -164,8 +164,8 @@ endif + + # We have to hardcode the book name + # With '%' the test for 'newness' fails +-Clusters_from_Scratch.build: $(PNGS) $(CFS_XML_ONLY) $(CFS_XML) $(CFS_SHARED_XML) $(PUBLICAN_INTREE_DEPS) +- $(PCMK_V) @echo Building $(@:%.build=%) because of $? ++Clusters_from_Scratch.build: $(PNGS) $(CFS_XML_ONLY) $(CFS_XML) $(CFS_SHARED_XML) $(PUBLICAN_INTREE_DEPS) ++ @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + if PUBLICAN_INTREE_BRAND + $(AM_V_PUB)cd $(@:%.build=%) \ +@@ -190,7 +190,7 @@ PD_XML_ONLY=$(addprefix Pacemaker_Development/en-US/,$(COMMON_XML) \ + # We have to hardcode the book name + # With '%' the test for 'newness' fails + Pacemaker_Development.build: $(PD_XML_ONLY) $(PD_XML) $(PUBLICAN_INTREE_DEPS) +- $(PCMK_V) @echo Building $(@:%.build=%) because of $? ++ @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + if PUBLICAN_INTREE_BRAND + $(AM_V_PUB)cd $(@:%.build=%) \ +@@ -219,7 +219,7 @@ $(PE_XML): $(PE_SHARED_XML) + # We have to hardcode the book name + # With '%' the test for 'newness' fails + Pacemaker_Explained.build: $(PNGS) $(PE_XML_ONLY) $(PE_XML) $(PE_SHARED_XML) $(PUBLICAN_INTREE_DEPS) +- $(PCMK_V) @echo Building $(@:%.build=%) because of $? ++ @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + if PUBLICAN_INTREE_BRAND + $(AM_V_PUB)cd $(@:%.build=%) \ +@@ -244,7 +244,7 @@ PR_XML_ONLY=$(addprefix Pacemaker_Remote/en-US/,$(COMMON_XML) \ + # We have to hardcode the book name + # With '%' the test for 'newness' fails + Pacemaker_Remote.build: $(PNGS) $(PR_XML_ONLY) $(PR_XML) $(PUBLICAN_INTREE_DEPS) +- $(PCMK_V) @echo Building $(@:%.build=%) because of $? ++ @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + if PUBLICAN_INTREE_BRAND + $(AM_V_PUB)cd $(@:%.build=%) \ +diff --git a/xml/Makefile.am b/xml/Makefile.am +index 972750f..af53a6d 100644 +--- a/xml/Makefile.am ++++ b/xml/Makefile.am +@@ -4,7 +4,8 @@ + # This source code is licensed under the GNU General Public License version 2 + # or later (GPLv2+) WITHOUT ANY WARRANTY. + # +-MAINTAINERCLEANFILES = Makefile.in ++ ++include $(top_srcdir)/Makefile.common + + # Pacemaker 1.1 has 2 schemas: the CIB schema, and a schema for + # crm_mon --as-xml. +@@ -86,55 +87,52 @@ cib-versions: + + # Dynamically generated top-level CIB schema + pacemaker.rng: pacemaker-$(CIB_max).rng +- echo " RNG $@" +- cp $(top_builddir)/xml/$< $@ ++ $(AM_V_SCHEMA)cp $(top_builddir)/xml/$< $@ + + pacemaker-%.rng: $(CIB_files) best-match.sh Makefile.am +- echo " RNG $@" +- echo '<?xml version="1.0" encoding="UTF-8"?>' > $@ +- echo '<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">' >> $@ +- echo ' <start>' >> $@ +- echo ' <element name="cib">' >> $@ +- $(srcdir)/best-match.sh cib $(*) $(@) " " +- echo ' <element name="configuration">' >> $@ +- echo ' <interleave>' >> $@ +- for rng in $(CIB_cfg_base); do $(srcdir)/best-match.sh $$rng $(*) $(@) " " || :; done +- echo ' </interleave>' >> $@ +- echo ' </element>' >> $@ +- echo ' <optional>' >> $@ +- echo ' <element name="status">' >> $@ +- $(srcdir)/best-match.sh status $(*) $(@) " " +- echo ' </element>' >> $@ +- echo ' </optional>' >> $@ +- echo ' </element>' >> $@ +- echo ' </start>' >> $@ +- echo '</grammar>' >> $@ ++ $(AM_V_at)echo '<?xml version="1.0" encoding="UTF-8"?>' > $@ ++ $(AM_V_at)echo '<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">' >> $@ ++ $(AM_V_at)echo ' <start>' >> $@ ++ $(AM_V_at)echo ' <element name="cib">' >> $@ ++ $(AM_V_at)$(srcdir)/best-match.sh cib $(*) $(@) " " ++ $(AM_V_at)echo ' <element name="configuration">' >> $@ ++ $(AM_V_at)echo ' <interleave>' >> $@ ++ $(AM_V_at)for rng in $(CIB_cfg_base); do $(srcdir)/best-match.sh $$rng $(*) $(@) " " || :; done ++ $(AM_V_at)echo ' </interleave>' >> $@ ++ $(AM_V_at)echo ' </element>' >> $@ ++ $(AM_V_at)echo ' <optional>' >> $@ ++ $(AM_V_at)echo ' <element name="status">' >> $@ ++ $(AM_V_at)$(srcdir)/best-match.sh status $(*) $(@) " " ++ $(AM_V_at)echo ' </element>' >> $@ ++ $(AM_V_at)echo ' </optional>' >> $@ ++ $(AM_V_at)echo ' </element>' >> $@ ++ $(AM_V_at)echo ' </start>' >> $@ ++ $(AM_V_SCHEMA)echo '</grammar>' >> $@ + + # Dynamically generated CIB schema listing all pacemaker versions + versions.rng: Makefile.am +- echo " RNG $@" +- echo '<?xml version="1.0" encoding="UTF-8"?>' > $@ +- echo '<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">' >> $@ +- echo ' <start>' >> $@ +- echo ' <interleave>' >> $@ +- echo ' <optional>' >> $@ +- echo ' <attribute name="validate-with">' >> $@ +- echo ' <choice>' >> $@ +- echo ' <value>none</value>' >> $@ +- echo ' <value>pacemaker-0.6</value>' >> $@ +- echo ' <value>transitional-0.6</value>' >> $@ +- echo ' <value>pacemaker-0.7</value>' >> $@ +- echo ' <value>pacemaker-1.1</value>' >> $@ +- for rng in $(CIB_versions); do echo " <value>pacemaker-$$rng</value>" >> $@; done +- echo ' </choice>' >> $@ +- echo ' </attribute>' >> $@ +- echo ' </optional>' >> $@ +- echo ' <attribute name="admin_epoch"><data type="nonNegativeInteger"/></attribute>' >> $@ +- echo ' <attribute name="epoch"><data type="nonNegativeInteger"/></attribute>' >> $@ +- echo ' <attribute name="num_updates"><data type="nonNegativeInteger"/></attribute>' >> $@ +- echo ' </interleave>' >> $@ +- echo ' </start>' >> $@ +- echo '</grammar>' >> $@ ++ $(AM_V_at)echo '<?xml version="1.0" encoding="UTF-8"?>' > $@ ++ $(AM_V_at)echo '<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">' >> $@ ++ $(AM_V_at)echo ' <start>' >> $@ ++ $(AM_V_at)echo ' <interleave>' >> $@ ++ $(AM_V_at)echo ' <optional>' >> $@ ++ $(AM_V_at)echo ' <attribute name="validate-with">' >> $@ ++ $(AM_V_at)echo ' <choice>' >> $@ ++ $(AM_V_at)echo ' <value>none</value>' >> $@ ++ $(AM_V_at)echo ' <value>pacemaker-0.6</value>' >> $@ ++ $(AM_V_at)echo ' <value>transitional-0.6</value>' >> $@ ++ $(AM_V_at)echo ' <value>pacemaker-0.7</value>' >> $@ ++ $(AM_V_at)echo ' <value>pacemaker-1.1</value>' >> $@ ++ $(AM_V_at)for rng in $(CIB_versions); do echo " <value>pacemaker-$$rng</value>" >> $@; done ++ $(AM_V_at)echo ' </choice>' >> $@ ++ $(AM_V_at)echo ' </attribute>' >> $@ ++ $(AM_V_at)echo ' </optional>' >> $@ ++ $(AM_V_at)echo ' <attribute name="admin_epoch"><data type="nonNegativeInteger"/></attribute>' >> $@ ++ $(AM_V_at)echo ' <attribute name="epoch"><data type="nonNegativeInteger"/></attribute>' >> $@ ++ $(AM_V_at)echo ' <attribute name="num_updates"><data type="nonNegativeInteger"/></attribute>' >> $@ ++ $(AM_V_at)echo ' </interleave>' >> $@ ++ $(AM_V_at)echo ' </start>' >> $@ ++ $(AM_V_SCHEMA)echo '</grammar>' >> $@ + + # diff fails with ec=2 if no predecessor is found; + # this uses '=' GNU extension to sed, if that's not available, +-- +1.8.3.1 + + +From d801c43b35b457170604f2e9d0c16c81cf2c0f98 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 1 Aug 2019 11:09:59 -0500 +Subject: [PATCH 88/96] Build: makefiles: don't echo echo + +echo should usually be used as @echo in makefiles; there's no point in seeing: + + echo blah + blah +--- + GNUmakefile | 16 ++++++++-------- + Makefile.am | 15 ++++++++++++--- + doc/Makefile.am | 14 +++++++------- + 3 files changed, 27 insertions(+), 18 deletions(-) + +diff --git a/GNUmakefile b/GNUmakefile +index d790865..3dd1055 100644 +--- a/GNUmakefile ++++ b/GNUmakefile +@@ -207,7 +207,7 @@ srpm-%: export $(PACKAGE)-%.spec + $(call rpmbuild-with,$(WITH),-bs --define "dist .$*" $(RPM_OPTS),$(PACKAGE).spec) + + chroot: mock-$(MOCK_CFG) mock-install-$(MOCK_CFG) mock-sh-$(MOCK_CFG) +- echo "Done" ++ @echo "Done" + + mock-next: + make F=$(shell expr 1 + $(F)) mock +@@ -216,19 +216,19 @@ mock-rawhide: + make F=rawhide mock + + mock-install-%: +- echo "Installing packages" ++ @echo "Installing packages" + mock --root=$* $(MOCK_OPTIONS) --install $(RPM_ROOT)/mock/*.rpm vi sudo valgrind lcov gdb fence-agents psmisc + + mock-install: mock-install-$(MOCK_CFG) +- echo "Done" ++ @echo "Done" + + mock-sh: mock-sh-$(MOCK_CFG) +- echo "Done" ++ @echo "Done" + + mock-sh-%: +- echo "Connecting" ++ @echo "Connecting" + mock --root=$* $(MOCK_OPTIONS) --shell +- echo "Done" ++ @echo "Done" + + # eg. WITH="--with cman" make rpm + mock-%: +@@ -238,10 +238,10 @@ mock-%: + mock --root=$* --no-cleanup-after --rebuild $(WITH) $(MOCK_OPTIONS) $(RPM_ROOT)/*.src.rpm + + srpm: srpm-$(DISTRO) +- echo "Done" ++ @echo "Done" + + mock: mock-$(MOCK_CFG) +- echo "Done" ++ @echo "Done" + + rpm-dep: $(PACKAGE)-$(DISTRO).spec + if [ x != x`which yum-builddep 2>/dev/null` ]; then \ +diff --git a/Makefile.am b/Makefile.am +index 3080445..b47f488 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -63,15 +63,24 @@ scratch.c: + + core: + @echo "Building only core components: $(CORE)" +- list='$(CORE)'; for subdir in $$list; do echo "Building $$subdir"; $(MAKE) -C $$subdir all || exit 1; done ++ @for subdir in $(CORE); do \ ++ echo "Building $$subdir"; \ ++ $(MAKE) -C $$subdir all || exit 1; \ ++ done + + core-install: + @echo "Installing only core components: $(CORE)" +- list='$(CORE)'; for subdir in $$list; do echo "Installing $$subdir"; $(MAKE) -C $$subdir install || exit 1; done ++ @for subdir in $(CORE); do \ ++ echo "Installing $$subdir"; \ ++ $(MAKE) -C $$subdir install || exit 1; \ ++ done + + core-clean: + @echo "Cleaning only core components: $(CORE)" +- list='$(CORE)'; for subdir in $$list; do echo "Cleaning $$subdir"; $(MAKE) -C $$subdir clean || exit 1; done ++ @for subdir in $(CORE); do \ ++ echo "Cleaning $$subdir"; \ ++ $(MAKE) -C $$subdir clean || exit 1; \ ++ done + + install-exec-local: + $(INSTALL) -d $(DESTDIR)/$(LCRSODIR) +diff --git a/doc/Makefile.am b/doc/Makefile.am +index a01423d..8389054 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -262,14 +262,14 @@ endif + + # Update the translation template + pot: +- for book in $(docbook); do \ ++ @for book in $(docbook); do \ + echo "Updating translation templates in: $$book"; \ + ( cd $$book && RPM_BUILD_DIR="" $(PUBLICAN) update_pot ); \ + done + + # Update the actual translations + po: pot +- for book in $(docbook); do \ ++ @for book in $(docbook); do \ + echo "Updating translations in: $$book"; \ + ( cd $$book && RPM_BUILD_DIR="" $(PUBLICAN) update_po --langs=all );\ + done +@@ -300,7 +300,7 @@ brand-build: $(BRAND_DEPS) + cd publican-clusterlabs && publican build --formats=xml --langs=all --publish + + brand: brand-build +- echo "Installing..." ++ @echo "Installing branded content..." + cd publican-clusterlabs && sudo publican install_brand --path=$(datadir)/publican/Common_Content + + brand-rpm-clean: +@@ -315,14 +315,14 @@ brand-rpm-install: brand-rpm-build + pdf: + make DOCBOOK_FORMATS="pdf" all-local + +-www: clean-local $(generated_docs) $(ascii) ++www: clean-local $(doc_DATA) + for book in $(docbook); do \ + sed -i.sed 's@^brand:.*@brand: clusterlabs@' $$book/publican.cfg; \ + done +- make DOCBOOK_FORMATS="pdf,html,html-single,epub" DOCBOOK_LANGS="$(UPLOAD_LANGS)" all-local +- echo Uploading current $(PACKAGE_SERIES) documentation set to clusterlabs.org ++ $(MAKE) DOCBOOK_FORMATS="pdf,html,html-single,epub" DOCBOOK_LANGS="$(UPLOAD_LANGS)" all-local ++ @echo Uploading current $(PACKAGE_SERIES) documentation set to clusterlabs.org + if BUILD_DOCBOOK +- for book in $(docbook); do \ ++ @for book in $(docbook); do \ + echo Uploading $$book...; \ + echo "Generated on `date` from version: $(shell git log --pretty="format:%h %d" -n 1)" >> $$book/publish/build-$(PACKAGE_SERIES).txt; \ + rsync $(RSYNC_OPTS) $$book/publish/* "$(RSYNC_DEST)/$(PACKAGE)/doc/"; \ +-- +1.8.3.1 + + +From 1ac3f3ec7702e5aaff3c77e75da817370602eab7 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 1 Aug 2019 11:11:25 -0500 +Subject: [PATCH 89/96] Build: doc: reorganize and comment makefile + +for simplicity and readability +--- + doc/Makefile.am | 208 +++++++++++++++++++++++++------------------------------- + 1 file changed, 94 insertions(+), 114 deletions(-) + +diff --git a/doc/Makefile.am b/doc/Makefile.am +index 8389054..b1c9e06 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -19,14 +19,29 @@ + # + include $(top_srcdir)/Makefile.common + +-helpdir = $(datadir)/$(PACKAGE) ++# Deprecated plaintext documents (also dynamically converted to HTML) ++ascii = acls.txt \ ++ crm_fencing.txt ++generated_docs = ++if BUILD_ASCIIDOC ++generated_docs += $(ascii:%.txt=%.html) ++endif ++ ++# Current Publican/docbook-based documentation ++docbook = Clusters_from_Scratch \ ++ Pacemaker_Development \ ++ Pacemaker_Explained \ ++ Pacemaker_Remote ++docbook_build = $(docbook:%=%.build) ++ ++doc_DATA = $(ascii) $(generated_docs) + +-ascii = crm_fencing.txt acls.txt +-docbook = Clusters_from_Scratch \ +- Pacemaker_Development \ +- Pacemaker_Explained \ +- Pacemaker_Remote +-doc_DATA = $(ascii) $(generated_docs) ++EXTRA_DIST = $(ascii) $(SHARED_TXT) $(PNGS_ORIGINAL) $(DOTS) $(SVGS) ++EXTRA_DIST += $(CFS_TXT) $(CFS_XML_ONLY) ++EXTRA_DIST += $(PD_TXT) $(PD_XML_ONLY) ++EXTRA_DIST += $(PE_TXT) $(PE_XML_ONLY) ++EXTRA_DIST += $(PR_TXT) $(PR_XML_ONLY) ++EXTRA_DIST += pcs-crmsh-quick-ref.md + + # toplevel rsync destination for www targets (without trailing slash) + RSYNC_DEST ?= root@www.clusterlabs.org:/var/www/html +@@ -35,21 +50,13 @@ RSYNC_DEST ?= root@www.clusterlabs.org:/var/www/html + # don't cross filesystems, sparse, show progress + RSYNC_OPTS = -rlptvzxS --progress + +-publican_docs = +-generated_docs = +-generated_mans = +- +- +-# What formats to build: pdf,html,html-single,html-desktop,epub ++# What formats to build by default: pdf,html,html-single,html-desktop,epub + DOCBOOK_FORMATS := html-desktop + +-# What languages to build ++# What languages to build and upload to website by default ++# (currently only en-US because translations are out of date) + DOCBOOK_LANGS := en-US + +-# What languages to build for uploading to website +-# (currently only en-US because translations aren't up-to-date) +-UPLOAD_LANGS = en-US +- + # @TODO We could simplify this (and .gitignore) by establishing a convention + # that original image source begins with an uppercase letter and generated + # files with lowercase. +@@ -65,7 +72,7 @@ SVGS = $(wildcard Clusters_from_Scratch/en-US/images/pcmk-*.svg) \ + $(wildcard Pacemaker_Explained/en-US/images/pcmk-*.svg) \ + $(DOTS:%.dot=%.svg) + +-# Final images ++# Final images (some originally in PNG, others generated from SVG) + PNGS_ORIGINAL = Pacemaker_Remote/en-US/images/pcmk-ha-cluster-stack.png \ + Pacemaker_Remote/en-US/images/pcmk-ha-remote-stack.png \ + Clusters_from_Scratch/en-US/images/Console.png \ +@@ -94,22 +101,6 @@ graphics: $(PNGS) + %-large.png: %.svg + $(AM_V_GEN)$(INKSCAPE) --file=$< --export-dpi=180 -C --export-png=$@ $(PCMK_quiet) + +-if BUILD_ASCIIDOC +-generated_docs += $(ascii:%.txt=%.html) +- +-if BUILD_DOCBOOK +-publican_docs += $(docbook) +-endif +-endif +- +-EXTRA_DIST = $(ascii) $(SHARED_TXT) $(PNGS_ORIGINAL) $(DOTS) $(SVGS) +-EXTRA_DIST += $(CFS_TXT) $(CFS_XML_ONLY) +-EXTRA_DIST += $(PA_TXT) $(PA_XML_ONLY) +-EXTRA_DIST += $(PD_TXT) $(PD_XML_ONLY) +-EXTRA_DIST += $(PE_TXT) $(PE_XML_ONLY) +-EXTRA_DIST += $(PR_TXT) $(PR_XML_ONLY) +-EXTRA_DIST += pcs-crmsh-quick-ref.md +- + if IS_ASCIIDOC + ASCIIDOC_HTML_ARGS = --unsafe --backend=xhtml11 + else +@@ -147,119 +138,107 @@ COMMON_XML = Author_Group.xml Book_Info.xml Revision_History.xml + SHARED_TXT=$(wildcard shared/en-US/*.txt) + SHARED_XML=$(SHARED_TXT:%.txt=%.xml) + ++if PUBLICAN_INTREE_BRAND ++PUBLICAN_INTREE_DEPS = publican-catalog ++PUBLICAN_INTREE_ENV = XML_CATALOG_FILES="$(CURDIR)/publican-catalog" ++PUBLICAN_INTREE_OPT = --brand_dir=../publican-clusterlabs ++else ++PUBLICAN_INTREE_DEPS = ++PUBLICAN_INTREE_ENV = ++PUBLICAN_INTREE_OPT = ++endif + +-CFS_SHARED_TXT=$(addprefix shared/en-US/,pacemaker-intro.txt) +-CFS_SHARED_XML=$(CFS_SHARED_TXT:%.txt=%.xml) +-CFS_TXT=$(wildcard Clusters_from_Scratch/en-US/*.txt) +-CFS_XML=$(CFS_TXT:%.txt=%.xml) +-CFS_XML_ONLY=$(addprefix Clusters_from_Scratch/en-US/,$(COMMON_XML) \ +- Clusters_from_Scratch.ent Clusters_from_Scratch.xml Preface.xml) + +-$(CFS_XML): $(CFS_SHARED_XML) ++# Clusters From Scratch + +-PUBLICAN_INTREE_DEPS = +-if PUBLICAN_INTREE_BRAND +-PUBLICAN_INTREE_DEPS += publican-catalog +-endif ++CFS_SHARED_TXT = $(addprefix shared/en-US/,pacemaker-intro.txt) ++CFS_SHARED_XML = $(CFS_SHARED_TXT:%.txt=%.xml) ++CFS_TXT = $(wildcard Clusters_from_Scratch/en-US/*.txt) ++CFS_XML_GEN = $(CFS_TXT:%.txt=%.xml) ++CFS_XML_ONLY = $(addprefix Clusters_from_Scratch/en-US/,$(COMMON_XML) \ ++ Clusters_from_Scratch.ent \ ++ Clusters_from_Scratch.xml \ ++ Preface.xml) ++CFS_DEPS = $(PNGS) $(CFS_SHARED_XML) $(CFS_XML_ONLY) $(CFS_XML_GEN) + + # We have to hardcode the book name + # With '%' the test for 'newness' fails +-Clusters_from_Scratch.build: $(PNGS) $(CFS_XML_ONLY) $(CFS_XML) $(CFS_SHARED_XML) $(PUBLICAN_INTREE_DEPS) ++Clusters_from_Scratch.build: $(CFS_DEPS) $(PUBLICAN_INTREE_DEPS) + @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp +-if PUBLICAN_INTREE_BRAND +- $(AM_V_PUB)cd $(@:%.build=%) \ +- && RPM_BUILD_DIR="" XML_CATALOG_FILES="$(CURDIR)/publican-catalog" \ +- $(PUBLICAN) build --publish --langs=$(DOCBOOK_LANGS) --formats=$(DOCBOOK_FORMATS) --brand_dir=../publican-clusterlabs \ +- $(PCMK_quiet) +-else +- $(AM_V_PUB)cd $(@:%.build=%) \ +- && RPM_BUILD_DIR="" \ +- $(PUBLICAN) build --publish --langs=$(DOCBOOK_LANGS) --formats=$(DOCBOOK_FORMATS) \ +- $(PCMK_quiet) +-endif ++ $(AM_V_PUB)cd $(@:%.build=%) && RPM_BUILD_DIR="" $(PUBLICAN_INTREE_ENV) \ ++ $(PUBLICAN) build --publish --langs="$(DOCBOOK_LANGS)" \ ++ --formats="$(DOCBOOK_FORMATS)" $(PUBLICAN_INTREE_OPT) $(PCMK_quiet) + rm -rf $(@:%.build=%)/tmp + touch $@ + + +-PD_TXT=$(wildcard Pacemaker_Development/en-US/*.txt) +-PD_XML=$(PD_TXT:%.txt=%.xml) +-PD_XML_ONLY=$(addprefix Pacemaker_Development/en-US/,$(COMMON_XML) \ +- Pacemaker_Development.ent Pacemaker_Development.xml) ++# Pacemaker Development ++ ++PD_TXT = $(wildcard Pacemaker_Development/en-US/*.txt) ++PD_XML_GEN = $(PD_TXT:%.txt=%.xml) ++PD_XML_ONLY = $(addprefix Pacemaker_Development/en-US/,$(COMMON_XML) \ ++ Pacemaker_Development.ent \ ++ Pacemaker_Development.xml) ++PD_DEPS = $(PD_XML_ONLY) $(PD_XML_GEN) + + # We have to hardcode the book name + # With '%' the test for 'newness' fails +-Pacemaker_Development.build: $(PD_XML_ONLY) $(PD_XML) $(PUBLICAN_INTREE_DEPS) ++Pacemaker_Development.build: $(PD_DEPS) $(PUBLICAN_INTREE_DEPS) + @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp +-if PUBLICAN_INTREE_BRAND +- $(AM_V_PUB)cd $(@:%.build=%) \ +- && RPM_BUILD_DIR="" XML_CATALOG_FILES="$(CURDIR)/publican-catalog" \ +- $(PUBLICAN) build --publish --langs=$(DOCBOOK_LANGS) --formats=$(DOCBOOK_FORMATS) --brand_dir=../publican-clusterlabs \ +- $(PCMK_quiet) +-else +- $(AM_V_PUB)cd $(@:%.build=%) \ +- && RPM_BUILD_DIR="" \ +- $(PUBLICAN) build --publish --langs=$(DOCBOOK_LANGS) --formats=$(DOCBOOK_FORMATS) \ +- $(PCMK_quiet) +-endif ++ $(AM_V_PUB)cd $(@:%.build=%) && RPM_BUILD_DIR="" $(PUBLICAN_INTREE_ENV) \ ++ $(PUBLICAN) build --publish --langs="$(DOCBOOK_LANGS)" \ ++ --formats="$(DOCBOOK_FORMATS)" $(PUBLICAN_INTREE_OPT) $(PCMK_quiet) + rm -rf $(@:%.build=%)/tmp + touch $@ + + +-PE_SHARED_TXT=$(addprefix shared/en-US/,pacemaker-intro.txt) +-PE_SHARED_XML=$(PE_SHARED_TXT:%.txt=%.xml) +-PE_TXT=$(wildcard Pacemaker_Explained/en-US/*.txt) +-PE_XML=$(PE_TXT:%.txt=%.xml) +-PE_XML_ONLY=$(addprefix Pacemaker_Explained/en-US/,$(COMMON_XML) \ +- Pacemaker_Explained.ent Pacemaker_Explained.xml Preface.xml) ++# Pacemaker Explained + +-$(PE_XML): $(PE_SHARED_XML) ++PE_SHARED_TXT = $(addprefix shared/en-US/,pacemaker-intro.txt) ++PE_SHARED_XML = $(PE_SHARED_TXT:%.txt=%.xml) ++PE_TXT = $(wildcard Pacemaker_Explained/en-US/*.txt) ++PE_XML_GEN = $(PE_TXT:%.txt=%.xml) ++PE_XML_ONLY = $(addprefix Pacemaker_Explained/en-US/,$(COMMON_XML) \ ++ Pacemaker_Explained.ent \ ++ Pacemaker_Explained.xml \ ++ Preface.xml) ++PE_DEPS = $(PNGS) $(PE_SHARED_XML) $(PE_XML_ONLY) $(PE_XML_GEN) + + # We have to hardcode the book name + # With '%' the test for 'newness' fails +-Pacemaker_Explained.build: $(PNGS) $(PE_XML_ONLY) $(PE_XML) $(PE_SHARED_XML) $(PUBLICAN_INTREE_DEPS) ++Pacemaker_Explained.build: $(PE_DEPS) $(PUBLICAN_INTREE_DEPS) + @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp +-if PUBLICAN_INTREE_BRAND +- $(AM_V_PUB)cd $(@:%.build=%) \ +- && RPM_BUILD_DIR="" XML_CATALOG_FILES="$(CURDIR)/publican-catalog" \ +- $(PUBLICAN) build --publish --langs=$(DOCBOOK_LANGS) --formats=$(DOCBOOK_FORMATS) --brand_dir=../publican-clusterlabs \ +- $(PCMK_quiet) +-else +- $(AM_V_PUB)cd $(@:%.build=%) \ +- && RPM_BUILD_DIR="" \ +- $(PUBLICAN) build --publish --langs=$(DOCBOOK_LANGS) --formats=$(DOCBOOK_FORMATS) \ +- $(PCMK_quiet) +-endif ++ $(AM_V_PUB)cd $(@:%.build=%) && RPM_BUILD_DIR="" $(PUBLICAN_INTREE_ENV) \ ++ $(PUBLICAN) build --publish --langs="$(DOCBOOK_LANGS)" \ ++ --formats="$(DOCBOOK_FORMATS)" $(PUBLICAN_INTREE_OPT) $(PCMK_quiet) + rm -rf $(@:%.build=%)/tmp + touch $@ + + +-PR_TXT=$(wildcard Pacemaker_Remote/en-US/*.txt) +-PR_XML=$(PR_TXT:%.txt=%.xml) +-PR_XML_ONLY=$(addprefix Pacemaker_Remote/en-US/,$(COMMON_XML) \ +- Pacemaker_Remote.ent Pacemaker_Remote.xml) ++# Pacemaker Remote ++ ++PR_TXT = $(wildcard Pacemaker_Remote/en-US/*.txt) ++PR_XML_GEN = $(PR_TXT:%.txt=%.xml) ++PR_XML_ONLY = $(addprefix Pacemaker_Remote/en-US/,$(COMMON_XML) \ ++ Pacemaker_Remote.ent \ ++ Pacemaker_Remote.xml) ++PR_DEPS = $(PR_XML_ONLY) $(PR_XML_GEN) + + # We have to hardcode the book name + # With '%' the test for 'newness' fails +-Pacemaker_Remote.build: $(PNGS) $(PR_XML_ONLY) $(PR_XML) $(PUBLICAN_INTREE_DEPS) ++Pacemaker_Remote.build: $(PNGS) $(PR_DEPS) $(PUBLICAN_INTREE_DEPS) + @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp +-if PUBLICAN_INTREE_BRAND +- $(AM_V_PUB)cd $(@:%.build=%) \ +- && RPM_BUILD_DIR="" XML_CATALOG_FILES="$(CURDIR)/publican-catalog" \ +- $(PUBLICAN) build --publish --langs=$(DOCBOOK_LANGS) --formats=$(DOCBOOK_FORMATS) --brand_dir=../publican-clusterlabs \ +- $(PCMK_quiet) +-else +- $(AM_V_PUB)cd $(@:%.build=%) \ +- && RPM_BUILD_DIR="" \ +- $(PUBLICAN) build --publish --langs=$(DOCBOOK_LANGS) --formats=$(DOCBOOK_FORMATS) \ +- $(PCMK_quiet) +-endif ++ $(AM_V_PUB)cd $(@:%.build=%) && RPM_BUILD_DIR="" $(PUBLICAN_INTREE_ENV) \ ++ $(PUBLICAN) build --publish --langs="$(DOCBOOK_LANGS)" \ ++ --formats="$(DOCBOOK_FORMATS)" $(PUBLICAN_INTREE_OPT) $(PCMK_quiet) + rm -rf $(@:%.build=%)/tmp + touch $@ + ++ + # Update the translation template + pot: + @for book in $(docbook); do \ +@@ -275,8 +254,6 @@ po: pot + done + + if BUILD_DOCBOOK +-docbook_build = $(docbook:%=%.build) +- + all-local: $(docbook_build) */publican.cfg + + install-data-local: all-local +@@ -316,10 +293,12 @@ pdf: + make DOCBOOK_FORMATS="pdf" all-local + + www: clean-local $(doc_DATA) ++if BUILD_DOCBOOK + for book in $(docbook); do \ + sed -i.sed 's@^brand:.*@brand: clusterlabs@' $$book/publican.cfg; \ + done +- $(MAKE) DOCBOOK_FORMATS="pdf,html,html-single,epub" DOCBOOK_LANGS="$(UPLOAD_LANGS)" all-local ++endif ++ $(MAKE) DOCBOOK_FORMATS="pdf,html,html-single,epub" DOCBOOK_LANGS="$(DOCBOOK_LANGS)" all-local + @echo Uploading current $(PACKAGE_SERIES) documentation set to clusterlabs.org + if BUILD_DOCBOOK + @for book in $(docbook); do \ +@@ -328,11 +307,12 @@ if BUILD_DOCBOOK + rsync $(RSYNC_OPTS) $$book/publish/* "$(RSYNC_DEST)/$(PACKAGE)/doc/"; \ + done + endif +- rsync $(RSYNC_OPTS) $(generated_docs) $(ascii) "$(RSYNC_DEST)/$(PACKAGE)/doc/" ++ rsync $(RSYNC_OPTS) $(doc_DATA) "$(RSYNC_DEST)/$(PACKAGE)/doc/" ++ + + clean-local: + -rm -f $(PNGS_GENERATED) + -rm -rf $(generated_docs) $(generated_mans) $(docbook_build) +- -rm -rf $(SHARED_XML) $(CFS_XML) $(PE_XML) $(PR_XML) ++ -rm -rf $(SHARED_XML) $(CFS_XML_GEN) $(PD_XML_GEN) $(PE_XML_GEN) $(PR_XML_GEN) + -rm -rf publican-catalog-fallback publican-catalog + for book in $(docbook); do rm -rf $$book/tmp $$book/publish; done +-- +1.8.3.1 + + +From 00c927f263760bc3163f45fabc2744842f128d3f Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 1 Aug 2019 14:29:41 -0500 +Subject: [PATCH 90/96] Build: doc: properly clean all generated files + +--- + doc/Makefile.am | 25 +++++++++++++++++-------- + 1 file changed, 17 insertions(+), 8 deletions(-) + +diff --git a/doc/Makefile.am b/doc/Makefile.am +index b1c9e06..dfc6732 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -281,7 +281,7 @@ brand: brand-build + cd publican-clusterlabs && sudo publican install_brand --path=$(datadir)/publican/Common_Content + + brand-rpm-clean: +- find publican-clusterlabs -name "*.noarch.rpm" -exec rm -f \{\} \; ++ -find publican-clusterlabs -name "*.noarch.rpm" -exec rm -f \{\} \; + + brand-rpm-build: brand-rpm-clean brand-build + cd publican-clusterlabs && $(PUBLICAN) package --binary +@@ -309,10 +309,19 @@ if BUILD_DOCBOOK + endif + rsync $(RSYNC_OPTS) $(doc_DATA) "$(RSYNC_DEST)/$(PACKAGE)/doc/" + +- +-clean-local: +- -rm -f $(PNGS_GENERATED) +- -rm -rf $(generated_docs) $(generated_mans) $(docbook_build) +- -rm -rf $(SHARED_XML) $(CFS_XML_GEN) $(PD_XML_GEN) $(PE_XML_GEN) $(PR_XML_GEN) +- -rm -rf publican-catalog-fallback publican-catalog +- for book in $(docbook); do rm -rf $$book/tmp $$book/publish; done ++ALL_GEN = $(generated_docs) \ ++ $(docbook_build) \ ++ $(PNGS_GENERATED) \ ++ $(SHARED_XML) \ ++ $(CFS_XML_GEN) \ ++ $(PD_XML_GEN) \ ++ $(PE_XML_GEN) \ ++ $(PR_XML_GEN) \ ++ publican-catalog-fallback \ ++ publican-catalog ++ ++clean-local: brand-rpm-clean ++ -rm -f $(ALL_GEN) ++ -for book in $(docbook); do \ ++ rm -rf $$book/tmp $$book/publish; \ ++ done +-- +1.8.3.1 + + +From b022f4fe58424fe9149aebeef45f85c2abd6fcf9 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 1 Aug 2019 15:56:20 -0500 +Subject: [PATCH 91/96] Build: clean up Makefile.common + +Reorganize and comment for readability +--- + Makefile.common | 43 ++++++++++++++++++++++++++++++++++++++----- + tools/Makefile.am | 5 ++++- + 2 files changed, 42 insertions(+), 6 deletions(-) + +diff --git a/Makefile.common b/Makefile.common +index 386d59d..f7661c9 100644 +--- a/Makefile.common ++++ b/Makefile.common +@@ -42,22 +42,57 @@ MAINTAINERCLEANFILES = Makefile.in + AM_CPPFLAGS = -I$(top_builddir)/include -I$(top_srcdir)/include \ + -I$(top_builddir)/libltdl -I$(top_srcdir)/libltdl + ++# ++# Man page builders ++# ++# We have three types of man pages: ++# - man pages for the tools ++# - man pages for OCF agents ++# - man pages for cluster properties used by daemons ++# ++# "BUILD_HELP" actually means "help2man is available", so it only controls the ++# tool man pages, which are generated by help2man. The other man pages are ++# generated via XSL transforms. ++# ++ + if BUILD_HELP + man8_MANS = $(sbin_PROGRAMS:%=%.8) $(sbin_SCRIPTS:%=%.8) $(dist_sbin_SCRIPTS:%=%.8) +-endif + ++HELP2MAN_ARGS = -N --section 8 --name "Part of the Pacemaker cluster resource manager" ++ ++# Some of our tools' help are just shell script invocations of another tool's ++# help. Putting the real tool in MAN8DEPS helps detect when the wrapped help ++# needs updating. ++# ++# If a ".inc" file exists, the tool has been converted to use glib for ++# argument parsing, otherwise it still uses the libcrmcommon functions. ++# ++# @TODO Drop MAN8DEPS once we've converted all tools to libpacemaker API calls ++# and all wrappers to C code. + %.8: % $(MAN8DEPS) + $(AM_V_at)chmod a+x $(abs_builddir)/$< +- $(AM_V_MAN)PATH=$(abs_builddir):$$PATH $(HELP2MAN) --output $@ --no-info --section 8 --name "Part of the Pacemaker cluster resource manager" $(abs_builddir)/$< ++ $(AM_V_MAN)PATH=$(abs_builddir):$$PATH $(HELP2MAN) --output $@ \ ++ $(HELP2MAN_ARGS) $(abs_builddir)/$< ++endif + ++# Save raw XML meta-data from daemon executables, for later conversion into man ++# pages. (Note that more specific rules may override this for creating other ++# types of XML files.) + %.xml: % + $(AM_V_at)$(abs_builddir)/$< metadata > $@ + ++# Process the raw daemon and OCF agent meta-data output using our ++# meta-data-to-docbook-XML tranform. + %.dbook: %.xml +- $(AM_V_at)$(XSLTPROC) --nonet --novalid --stringparam man.name $* $(DBOOK_OPTS) $(top_srcdir)/xml/ocf-meta2man.xsl $(abs_builddir)/$< > $(abs_builddir)/$@ ++ $(AM_V_at)$(XSLTPROC) --nonet --novalid --stringparam man.name $* \ ++ $(DBOOK_OPTS) $(top_srcdir)/xml/ocf-meta2man.xsl \ ++ $(abs_builddir)/$< > $(abs_builddir)/$@ + ++# Generate the actual man page for an OCF resource agent from the intermediate ++# docbook XML. + %.7: %.dbook + $(AM_V_MAN)$(XSLTPROC) $(MANPAGE_XSLT) $(abs_builddir)/$< $(PCMK_quiet) ++ + # + # Build docbook from asciidoc because XML is a PITA to edit + # +@@ -86,5 +121,3 @@ endif + $@-t # We just want the appendix tag (asciidoctor adds non-empty book-level title) + $(AM_V_at)sed -i 's/book>/chapter>/g' $@-t # Rename to chapter (won't trigger if previous sed did) + $(AM_V_GEN)mv $@-t $@ +- +-# echo Rebuilt $@ from $< +diff --git a/tools/Makefile.am b/tools/Makefile.am +index 6960548..e403849 100644 +--- a/tools/Makefile.am ++++ b/tools/Makefile.am +@@ -63,7 +63,10 @@ endif + + ## SOURCES + +-MAN8DEPS = crm_attribute crm_node ++# A few tools are just thin wrappers around crm_attribute. ++# This makes their help get updated when crm_attribute changes ++# (see Makefile.common). ++MAN8DEPS = crm_attribute + + crmadmin_SOURCES = crmadmin.c + crmadmin_LDADD = $(top_builddir)/lib/pengine/libpe_status.la \ +-- +1.8.3.1 + + +From 49e892641dce5870407dcd3bd5322e04893461da Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 1 Aug 2019 16:50:21 -0500 +Subject: [PATCH 92/96] Build: doc: move text-to-DocBook rule from common to + doc + +... since it's only used there, and was a bit confusing next to the +meta-data-to-DocBook rule in common. +--- + Makefile.common | 29 ----------------------------- + doc/Makefile.am | 26 ++++++++++++++++++++++++++ + 2 files changed, 26 insertions(+), 29 deletions(-) + +diff --git a/Makefile.common b/Makefile.common +index f7661c9..2731922 100644 +--- a/Makefile.common ++++ b/Makefile.common +@@ -92,32 +92,3 @@ endif + # docbook XML. + %.7: %.dbook + $(AM_V_MAN)$(XSLTPROC) $(MANPAGE_XSLT) $(abs_builddir)/$< $(PCMK_quiet) +- +-# +-# Build docbook from asciidoc because XML is a PITA to edit +-# +-# Build each chapter as a book (since the numbering isn't right for +-# articles and only books can have appendices) and then strip out the +-# bits we don't want/need +-# +-# XXX Sequence of tr/sed commands should be replaced with a single XSLT +-# +-%.xml: %.txt +-if IS_ASCIIDOC +- $(AM_V_GEN)$(ASCIIDOC_CONV) -b docbook -d book -o $@-tt $< +-else +- $(AM_V_GEN)$(ASCIIDOC_CONV) -b docbook45 -d book -o $@-tt $< +-endif +- $(AM_V_at)tr -d '\036\r' <$@-tt >$@-t; rm -f $@-tt # Fix line endings +- $(AM_V_at)sed -i 's/\ lang="en"//' $@-t # Never specify a language in the chapters +- $(AM_V_at)sed -i 's/simpara/para/g' $@-t # publican doesn't correctly render footnotes with simpara +- $(AM_V_at)sed -i 's/.*<date>.*//g' $@-t # Remove dangling tag +- $(AM_V_at)sed -i 's/.*preface>//g' $@-t # Remove preface elements +- $(AM_V_at)sed -i 's:<title>::g' $@-t # Remove empty title +- $(AM_V_at)sed -i 's/chapter/section/g' $@-t # Chapters become sections, so that books can become chapters +- $(AM_V_at)sed -i 's/<.*bookinfo.*>//g' $@-t # Strip out bookinfo, we don't need it +- $(AM_V_at)! grep -q "//;tb;bf;:b;N;s/.*.*<\/title>.*//;tb;/<appendix/{:i;n;/<\/appendix/{p;d};bi};bb;:f;p;d' \ +- $@-t # We just want the appendix tag (asciidoctor adds non-empty book-level title) +- $(AM_V_at)sed -i 's/book>/chapter>/g' $@-t # Rename to chapter (won't trigger if previous sed did) +- $(AM_V_GEN)mv $@-t $@ +diff --git a/doc/Makefile.am b/doc/Makefile.am +index dfc6732..5ff350c 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -103,13 +103,39 @@ graphics: $(PNGS) + + if IS_ASCIIDOC + ASCIIDOC_HTML_ARGS = --unsafe --backend=xhtml11 ++ASCIIDOC_DBOOK_ARGS = -b docbook -d book + else + ASCIIDOC_HTML_ARGS = --backend=html5 ++ASCIIDOC_DBOOK_ARGS = -b docbook45 -d book + endif + + %.html: %.txt + $(AM_V_GEN)$(ASCIIDOC_CONV) $(ASCIIDOC_HTML_ARGS) --out-file=$@ $< $(PCMK_quiet) + ++# ++# Generate DocBook XML from asciidoc text. ++# ++# Build each chapter as a book (since the numbering isn't right for ++# articles and only books can have appendices) and then strip out the ++# bits we don't want or need. ++# ++# XXX Sequence of tr/sed commands should be replaced with a single XSLT ++# ++%.xml: %.txt ++ $(AM_V_at)$(ASCIIDOC_CONV) $(ASCIIDOC_DBOOK_ARGS) -o - $< | tr -d '\036\r' >$@-t # Convert, fix line endings ++ $(AM_V_at)sed -i 's/\ lang="en"//' $@-t # Never specify a language in the chapters ++ $(AM_V_at)sed -i 's/simpara/para/g' $@-t # publican doesn't correctly render footnotes with simpara ++ $(AM_V_at)sed -i 's/.*<date>.*//g' $@-t # Remove dangling tag ++ $(AM_V_at)sed -i 's/.*preface>//g' $@-t # Remove preface elements ++ $(AM_V_at)sed -i 's:<title>::g' $@-t # Remove empty title ++ $(AM_V_at)sed -i 's/chapter/section/g' $@-t # Chapters become sections, so that books can become chapters ++ $(AM_V_at)sed -i 's/<.*bookinfo.*>//g' $@-t # Strip out bookinfo, we don't need it ++ $(AM_V_at)! grep -q "//;tb;bf;:b;N;s/.*.*<\/title>.*//;tb;/<appendix/{:i;n;/<\/appendix/{p;d};bi};bb;:f;p;d' \ ++ $@-t # We just want the appendix tag (asciidoctor adds non-empty book-level title) ++ $(AM_V_at)sed -i 's/book>/chapter>/g' $@-t # Rename to chapter (won't trigger if previous sed did) ++ $(AM_V_GEN)mv $@-t $@ ++ + # publican-clusterlabs/xsl/{html,html-single,pdf}.xsl refer to URIs + # requiring Internet access, hence we shadow that with a XML catalog-based + # redirect to local files brought with Publican installation; +-- +1.8.3.1 + + +From 2aa5764c84c7fac7c06ccebccee7295872ec8b40 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 1 Aug 2019 17:47:53 -0500 +Subject: [PATCH 93/96] Build: doc: skip publican documentation with "make + distcheck" + +I got publican partly working with VPATH builds, but ran into an issue +that wasn't worth spending more time on. +--- + Makefile.am | 9 +++++++++ + doc/Makefile.am | 44 ++++++++++++++++++++++++++------------------ + 2 files changed, 35 insertions(+), 18 deletions(-) + +diff --git a/Makefile.am b/Makefile.am +index b47f488..5db35c2 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -31,6 +31,15 @@ EXTRA_DIST = CONTRIBUTING.md \ + MAINTAINERCLEANFILES = Makefile.in aclocal.m4 configure DRF/config-h.in \ + DRF/stamp-h.in libtool.m4 ltdl.m4 + ++# Disable building Publican documentation when doing "make distcheck", because ++# some of our book sources are in the source directory, while others are ++# dynamically generated in the build directory, and publican can't handle that. ++# ++# @TODO To support VPATH builds for Publican, we'd probably have to create ++# a separate subtree of the build directory to use as Publican's source ++# directory, and copy the static sources into it. ++AM_DISTCHECK_CONFIGURE_FLAGS = --with-brand="" ++ + CORE = replace include lib mcp attrd pengine cib crmd fencing lrmd tools xml + SUBDIRS = $(CORE) extra doc + +diff --git a/doc/Makefile.am b/doc/Makefile.am +index 5ff350c..3d4be7f 100644 +--- a/doc/Makefile.am ++++ b/doc/Makefile.am +@@ -122,6 +122,7 @@ endif + # XXX Sequence of tr/sed commands should be replaced with a single XSLT + # + %.xml: %.txt ++ $(AM_V_at)$(MKDIR_P) $(shell dirname $@) # might not exist in VPATH build + $(AM_V_at)$(ASCIIDOC_CONV) $(ASCIIDOC_DBOOK_ARGS) -o - $< | tr -d '\036\r' >$@-t # Convert, fix line endings + $(AM_V_at)sed -i 's/\ lang="en"//' $@-t # Never specify a language in the chapters + $(AM_V_at)sed -i 's/simpara/para/g' $@-t # publican doesn't correctly render footnotes with simpara +@@ -167,7 +168,7 @@ SHARED_XML=$(SHARED_TXT:%.txt=%.xml) + if PUBLICAN_INTREE_BRAND + PUBLICAN_INTREE_DEPS = publican-catalog + PUBLICAN_INTREE_ENV = XML_CATALOG_FILES="$(CURDIR)/publican-catalog" +-PUBLICAN_INTREE_OPT = --brand_dir=../publican-clusterlabs ++PUBLICAN_INTREE_OPT = --brand_dir="$(top_srcdir)/publican-clusterlabs" + else + PUBLICAN_INTREE_DEPS = + PUBLICAN_INTREE_ENV = +@@ -181,7 +182,7 @@ CFS_SHARED_TXT = $(addprefix shared/en-US/,pacemaker-intro.txt) + CFS_SHARED_XML = $(CFS_SHARED_TXT:%.txt=%.xml) + CFS_TXT = $(wildcard Clusters_from_Scratch/en-US/*.txt) + CFS_XML_GEN = $(CFS_TXT:%.txt=%.xml) +-CFS_XML_ONLY = $(addprefix Clusters_from_Scratch/en-US/,$(COMMON_XML) \ ++CFS_XML_ONLY = $(addprefix $(srcdir)/Clusters_from_Scratch/en-US/,$(COMMON_XML) \ + Clusters_from_Scratch.ent \ + Clusters_from_Scratch.xml \ + Preface.xml) +@@ -193,8 +194,9 @@ Clusters_from_Scratch.build: $(CFS_DEPS) $(PUBLICAN_INTREE_DEPS) + @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + $(AM_V_PUB)cd $(@:%.build=%) && RPM_BUILD_DIR="" $(PUBLICAN_INTREE_ENV) \ +- $(PUBLICAN) build --publish --langs="$(DOCBOOK_LANGS)" \ +- --formats="$(DOCBOOK_FORMATS)" $(PUBLICAN_INTREE_OPT) $(PCMK_quiet) ++ $(PUBLICAN) build --src_dir="$(srcdir)" --publish \ ++ --langs="$(DOCBOOK_LANGS)" --formats="$(DOCBOOK_FORMATS)" \ ++ $(PUBLICAN_INTREE_OPT) $(PCMK_quiet) + rm -rf $(@:%.build=%)/tmp + touch $@ + +@@ -203,7 +205,7 @@ Clusters_from_Scratch.build: $(CFS_DEPS) $(PUBLICAN_INTREE_DEPS) + + PD_TXT = $(wildcard Pacemaker_Development/en-US/*.txt) + PD_XML_GEN = $(PD_TXT:%.txt=%.xml) +-PD_XML_ONLY = $(addprefix Pacemaker_Development/en-US/,$(COMMON_XML) \ ++PD_XML_ONLY = $(addprefix $(srcdir)/Pacemaker_Development/en-US/,$(COMMON_XML) \ + Pacemaker_Development.ent \ + Pacemaker_Development.xml) + PD_DEPS = $(PD_XML_ONLY) $(PD_XML_GEN) +@@ -226,7 +228,7 @@ PE_SHARED_TXT = $(addprefix shared/en-US/,pacemaker-intro.txt) + PE_SHARED_XML = $(PE_SHARED_TXT:%.txt=%.xml) + PE_TXT = $(wildcard Pacemaker_Explained/en-US/*.txt) + PE_XML_GEN = $(PE_TXT:%.txt=%.xml) +-PE_XML_ONLY = $(addprefix Pacemaker_Explained/en-US/,$(COMMON_XML) \ ++PE_XML_ONLY = $(addprefix $(srcdir)/Pacemaker_Explained/en-US/,$(COMMON_XML) \ + Pacemaker_Explained.ent \ + Pacemaker_Explained.xml \ + Preface.xml) +@@ -238,8 +240,9 @@ Pacemaker_Explained.build: $(PE_DEPS) $(PUBLICAN_INTREE_DEPS) + @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + $(AM_V_PUB)cd $(@:%.build=%) && RPM_BUILD_DIR="" $(PUBLICAN_INTREE_ENV) \ +- $(PUBLICAN) build --publish --langs="$(DOCBOOK_LANGS)" \ +- --formats="$(DOCBOOK_FORMATS)" $(PUBLICAN_INTREE_OPT) $(PCMK_quiet) ++ $(PUBLICAN) build --src_dir="$(srcdir)" --publish \ ++ --langs="$(DOCBOOK_LANGS)" --formats="$(DOCBOOK_FORMATS)" \ ++ $(PUBLICAN_INTREE_OPT) $(PCMK_quiet) + rm -rf $(@:%.build=%)/tmp + touch $@ + +@@ -248,7 +251,7 @@ Pacemaker_Explained.build: $(PE_DEPS) $(PUBLICAN_INTREE_DEPS) + + PR_TXT = $(wildcard Pacemaker_Remote/en-US/*.txt) + PR_XML_GEN = $(PR_TXT:%.txt=%.xml) +-PR_XML_ONLY = $(addprefix Pacemaker_Remote/en-US/,$(COMMON_XML) \ ++PR_XML_ONLY = $(addprefix $(srcdir)/Pacemaker_Remote/en-US/,$(COMMON_XML) \ + Pacemaker_Remote.ent \ + Pacemaker_Remote.xml) + PR_DEPS = $(PR_XML_ONLY) $(PR_XML_GEN) +@@ -259,24 +262,28 @@ Pacemaker_Remote.build: $(PNGS) $(PR_DEPS) $(PUBLICAN_INTREE_DEPS) + @echo Building $(@:%.build=%) because of $? + rm -rf $(@:%.build=%)/publish/* $(@:%.build=%)/tmp + $(AM_V_PUB)cd $(@:%.build=%) && RPM_BUILD_DIR="" $(PUBLICAN_INTREE_ENV) \ +- $(PUBLICAN) build --publish --langs="$(DOCBOOK_LANGS)" \ +- --formats="$(DOCBOOK_FORMATS)" $(PUBLICAN_INTREE_OPT) $(PCMK_quiet) ++ $(PUBLICAN) build --src_dir="$(srcdir)" --publish \ ++ --langs="$(DOCBOOK_LANGS)" --formats="$(DOCBOOK_FORMATS)" \ ++ $(PUBLICAN_INTREE_OPT) $(PCMK_quiet) + rm -rf $(@:%.build=%)/tmp + touch $@ + + + # Update the translation template + pot: +- @for book in $(docbook); do \ +- echo "Updating translation templates in: $$book"; \ +- ( cd $$book && RPM_BUILD_DIR="" $(PUBLICAN) update_pot ); \ ++ @for book in $(docbook); do \ ++ echo "Updating translation templates in: $$book"; \ ++ ( cd $$book && RPM_BUILD_DIR="" \ ++ $(PUBLICAN) --src_dir="$(srcdir)" update_pot ); \ + done + + # Update the actual translations + po: pot +- @for book in $(docbook); do \ +- echo "Updating translations in: $$book"; \ +- ( cd $$book && RPM_BUILD_DIR="" $(PUBLICAN) update_po --langs=all );\ ++ @for book in $(docbook); do \ ++ echo "Updating translations in: $$book"; \ ++ ( cd $$book && RPM_BUILD_DIR="" \ ++ $(PUBLICAN) --src_dir="$(srcdir)" update_po \ ++ --langs=all ); \ + done + + if BUILD_DOCBOOK +@@ -310,7 +317,8 @@ brand-rpm-clean: + -find publican-clusterlabs -name "*.noarch.rpm" -exec rm -f \{\} \; + + brand-rpm-build: brand-rpm-clean brand-build +- cd publican-clusterlabs && $(PUBLICAN) package --binary ++ cd publican-clusterlabs && \ ++ $(PUBLICAN) --src_dir="$(srcdir)" package --binary + + brand-rpm-install: brand-rpm-build + find publican-clusterlabs -name "*.noarch.rpm" -exec sudo rpm -Uvh --force \{\} \; +-- +1.8.3.1 + + +From bc0bd42b67c80102a4a838319bb8aa0a1310c76b Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 23 Aug 2019 17:28:49 -0500 +Subject: [PATCH 94/96] Fix: tools: correct crm_report argument parsing + +There were a few instances where crm_report's option names passed to getopt, +option names listed in help, and option names checked for did not match. + +Where getopt and checks matched, I went with that, so that anything that +worked before continues to work. +--- + tools/crm_report.in | 10 ++++++---- + 1 file changed, 6 insertions(+), 4 deletions(-) + +diff --git a/tools/crm_report.in b/tools/crm_report.in +index 63c137c..947140f 100755 +--- a/tools/crm_report.in ++++ b/tools/crm_report.in +@@ -20,7 +20,7 @@ + + TEMP=`getopt \ + -o hv?xl:f:t:n:T:L:p:c:dSACHu:D:MVse: \ +- --long help,cts:,cts-log:,dest:,node:,nodes:,from:,to:,sos-mode,logfile:,as-directory,single-node,cluster:,user:,max-depth:,version,features,rsh: \ ++ --long help,corosync,cts:,cts-log:,dest:,heartbeat,node:,nodes:,--openais,from:,to:,sos-mode,logfile:,as-directory,single-node,cluster:,user:,max-depth:,version,features,rsh: \ + -n 'crm_report' -- "$@"` + # The quotes around $TEMP are essential + eval set -- "$TEMP" +@@ -54,6 +54,7 @@ Required option: + + Options: + -V increase verbosity (may be specified multiple times) ++ -h, --help display this message + -v, --version display software version + --features display software features + -t, --to TIME time at which all problems were resolved +@@ -77,9 +78,10 @@ Options: + -H, --heartbeat force the cluster type to be heartbeat + -u, --user USER username to use when collecting data from other nodes + (default root) +- -D, --depth search depth to use when attempting to locate files ++ -D, --max-depth search depth to use when attempting to locate files + -e, --rsh command to use to run commands on other nodes + (default ssh -T) ++ -d, --as-directory leave result as a directory tree instead of archiving + --sos-mode use defaults suitable for being called by sosreport tool + (behavior subject to change and not useful to end users) + DEST, --dest DEST custom destination directory or file name +@@ -119,13 +121,13 @@ while true; do + case "$1" in + -x) set -x; shift;; + -V) verbose=`expr $verbose + 1`; shift;; +- -T|--cts-test) tests="$tests $2"; shift; shift;; ++ -T|--cts) tests="$tests $2"; shift; shift;; + --cts-log) ctslog="$2"; shift; shift;; + -f|--from) start_time=`get_time "$2"`; shift; shift;; + -t|--to) end_time=`get_time "$2"`; shift; shift;; + -n|--node|--nodes) nodes="$nodes $2"; shift; shift;; + -S|--single-node) nodes="$host"; shift;; +- -E|-l|--logfile) extra_logs="$extra_logs $2"; shift; shift;; ++ -l|--logfile) extra_logs="$extra_logs $2"; shift; shift;; + -p) sanitize_patterns="$sanitize_patterns $2"; shift; shift;; + -L) log_patterns="$log_patterns `echo $2 | sed 's/ /\\\W/g'`"; shift; shift;; + -d|--as-directory) compress=0; shift;; +-- +1.8.3.1 + + +From 49c3055b932b732e0904d91cf49d4f80b7aa0e7d Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 23 Aug 2019 17:39:45 -0500 +Subject: [PATCH 95/96] Fix: tools: don't ignore log if unrelated file is too + large + +This fixes a regression in 1.1.12: since cb420a04, findln_by_time() would skip +a log if any file in the current working directory (rather than the log itself) +was larger than 1GB. +--- + tools/report.common.in | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tools/report.common.in b/tools/report.common.in +index ebdd8df..55b37da 100644 +--- a/tools/report.common.in ++++ b/tools/report.common.in +@@ -535,7 +535,7 @@ findln_by_time() { + # Some logs can be massive (over 1,500,000,000 lines have been seen in the wild) + # Even just 'wc -l' on these files can take 10+ minutes + +- local fileSize=`ls -lh | awk '{ print $5 }' | grep -ie G` ++ local fileSize=`ls -lh "$logf" | awk '{ print $5 }' | grep -ie G` + if [ x$fileSize != x ]; then + warning "$logf is ${fileSize} in size and could take many hours to process. Skipping." + return +-- +1.8.3.1 + + +From 456668b5afd781c61576c9b2d2feaf058fe1cc22 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 23 Aug 2019 22:38:51 -0500 +Subject: [PATCH 96/96] Fix: tools: check for tar in crm_report + +crm_report requires tar, so check for its existence up front. +--- + tools/crm_report.in | 4 ++++ + tools/report.collector.in | 2 ++ + tools/report.common.in | 10 ++++++++++ + 3 files changed, 16 insertions(+) + +diff --git a/tools/crm_report.in b/tools/crm_report.in +index 947140f..0ef4e6f 100755 +--- a/tools/crm_report.in ++++ b/tools/crm_report.in +@@ -475,6 +475,10 @@ getnodes() { + fi + } + ++if [ $compress -eq 1 ]; then ++ require_tar ++fi ++ + if [ "x$tests" != "x" ]; then + do_cts + +diff --git a/tools/report.collector.in b/tools/report.collector.in +index 48ee075..ab41df1 100644 +--- a/tools/report.collector.in ++++ b/tools/report.collector.in +@@ -821,6 +821,8 @@ collect_logs() { + trap "" 0 + } + ++require_tar ++ + debug "Initializing $REPORT_TARGET subdir" + if [ "$REPORT_MASTER" != "$REPORT_TARGET" ]; then + if [ -e $REPORT_HOME/$REPORT_TARGET ]; then +diff --git a/tools/report.common.in b/tools/report.common.in +index 55b37da..6d4f193 100644 +--- a/tools/report.common.in ++++ b/tools/report.common.in +@@ -128,6 +128,13 @@ fatal() { + exit 1 + } + ++require_tar() { ++ which tar >/dev/null 2>&1 ++ if [ $? -ne 0 ]; then ++ fatal "Required program 'tar' not found, please install and re-run" ++ fi ++} ++ + is_running() { + ps -ef | egrep -qs $(echo "$1" | sed -e 's/^\(.\)/[\1]/') + } +@@ -522,6 +529,9 @@ shrink() { + + cd $dir >/dev/null 2>&1 + tar $tar_options $target $base >/dev/null 2>&1 ++ if [ $? -ne 0 ]; then ++ fatal "Could not archive $base, please investigate and collect manually" ++ fi + cd $olddir >/dev/null 2>&1 + + echo $target +-- +1.8.3.1 + diff --git a/SOURCES/010-fork-callback.patch b/SOURCES/010-fork-callback.patch deleted file mode 100644 index 44db052..0000000 --- a/SOURCES/010-fork-callback.patch +++ /dev/null @@ -1,33 +0,0 @@ -From b3d9a6e313cb77ef0b22aa0a182f0cabbaa2a70e Mon Sep 17 00:00:00 2001 -From: Klaus Wenninger <klaus.wenninger@aon.at> -Date: Wed, 22 May 2019 16:34:58 +0200 -Subject: [PATCH] Fix: fence-lib: regression introduced with fork callback - -If it is a retry device is already moved from activating_on to -active_on. ---- - fencing/commands.c | 7 ++++++- - 1 file changed, 6 insertions(+), 1 deletion(-) - -diff --git a/fencing/commands.c b/fencing/commands.c -index d47a5ea..264697e 100644 ---- a/fencing/commands.c -+++ b/fencing/commands.c -@@ -306,8 +306,13 @@ static void - fork_cb(GPid pid, gpointer user_data) - { - async_command_t *cmd = (async_command_t *) user_data; -- stonith_device_t * device = cmd->activating_on; -+ stonith_device_t * device = -+ /* in case of a retry we've done the move from -+ * activating_on to active_on already -+ */ -+ cmd->activating_on?cmd->activating_on:cmd->active_on; - -+ CRM_ASSERT(device); - crm_debug("Operation %s%s%s on %s now running with pid=%d, timeout=%ds", - cmd->action, cmd->victim ? " for node " : "", cmd->victim ? cmd->victim : "", - device->id, pid, cmd->timeout); --- -1.8.3.1 - diff --git a/SOURCES/011-remote.patch b/SOURCES/011-remote.patch deleted file mode 100644 index c09a8e2..0000000 --- a/SOURCES/011-remote.patch +++ /dev/null @@ -1,786 +0,0 @@ -From a81ca9625e8d1ccd7f79fbe464b9f4221e8671f2 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot <kgaillot@redhat.com> -Date: Thu, 9 May 2019 20:26:08 -0500 -Subject: [PATCH 1/6] Refactor: libpe_status: functionize unfencing digest code - more - -... for readability, reusability, and avoiding unnecessary function calls or -memory allocation. ---- - lib/pengine/utils.c | 159 ++++++++++++++++++++++++++++++++++++++-------------- - 1 file changed, 118 insertions(+), 41 deletions(-) - -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index d09b0d8..b6a31d1 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -2091,57 +2091,134 @@ rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, - return data; - } - -+/*! -+ * \internal -+ * \brief Create an unfencing summary for use in special node attribute -+ * -+ * Create a string combining a fence device's resource ID, agent type, and -+ * parameter digest (whether for all parameters or just non-private parameters). -+ * This can be stored in a special node attribute, allowing us to detect changes -+ * in either the agent type or parameters, to know whether unfencing must be -+ * redone or can be safely skipped when the device's history is cleaned. -+ * -+ * \param[in] rsc_id Fence device resource ID -+ * \param[in] agent_type Fence device agent -+ * \param[in] param_digest Fence device parameter digest -+ * -+ * \return Newly allocated string with unfencing digest -+ * \note The caller is responsible for freeing the result. -+ */ -+static inline char * -+create_unfencing_summary(const char *rsc_id, const char *agent_type, -+ const char *param_digest) -+{ -+ return crm_strdup_printf("%s:%s:%s", rsc_id, agent_type, param_digest); -+} -+ -+/*! -+ * \internal -+ * \brief Check whether a node can skip unfencing -+ * -+ * Check whether a fence device's current definition matches a node's -+ * stored summary of when it was last unfenced by the device. -+ * -+ * \param[in] rsc_id Fence device's resource ID -+ * \param[in] agent Fence device's agent type -+ * \param[in] digest_calc Fence device's current parameter digest -+ * \param[in] node_summary Value of node's special unfencing node attribute -+ * (a comma-separated list of unfencing summaries for -+ * all devices that have unfenced this node) -+ * -+ * \return TRUE if digest matches, FALSE otherwise -+ */ -+static bool -+unfencing_digest_matches(const char *rsc_id, const char *agent, -+ const char *digest_calc, const char *node_summary) -+{ -+ bool matches = FALSE; -+ -+ if (rsc_id && agent && digest_calc && node_summary) { -+ char *search_secure = create_unfencing_summary(rsc_id, agent, -+ digest_calc); -+ -+ /* The digest was calculated including the device ID and agent, -+ * so there is no risk of collision using strstr(). -+ */ -+ matches = (strstr(node_summary, search_secure) != NULL); -+ crm_trace("Calculated unfencing digest '%s' %sfound in '%s'", -+ search_secure, matches? "" : "not ", node_summary); -+ free(search_secure); -+ } -+ return matches; -+} -+ -+/* Magic string to use as action name for digest cache entries used for -+ * unfencing checks. This is not a real action name (i.e. "on"), so -+ * check_action_definition() won't confuse these entries with real actions. -+ */ - #define STONITH_DIGEST_TASK "stonith-on" - -+/*! -+ * \internal -+ * \brief Calculate fence device digests and digest comparison result -+ * -+ * \param[in] rsc Fence device resource -+ * \param[in] agent Fence device's agent type -+ * \param[in] node Node with digest cache to use -+ * \param[in] data_set Cluster working set -+ * -+ * \return Node's digest cache entry -+ */ - static op_digest_cache_t * --fencing_action_digest_cmp(resource_t * rsc, node_t * node, pe_working_set_t * data_set) -+fencing_action_digest_cmp(pe_resource_t *rsc, const char *agent, -+ pe_node_t *node, pe_working_set_t *data_set) - { -- char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); -- op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, node, NULL, data_set); -+ const char *node_summary = NULL; - -- const char *digest_all = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL); -- const char *digest_secure = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE); -+ // Calculate device's current parameter digests -+ char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); -+ op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, -+ node, NULL, data_set); - -- /* No 'reloads' for fencing device changes -- * -- * We use the resource id + agent + digest so that we can detect -- * changes to the agent and/or the parameters used -- */ -- char *search_all = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_all_calc); -- char *search_secure = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_secure_calc); -+ free(key); - -- data->rc = RSC_DIGEST_ALL; -- if (digest_all == NULL) { -- /* it is unknown what the previous op digest was */ -+ // Check whether node has special unfencing summary node attribute -+ node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL); -+ if (node_summary == NULL) { - data->rc = RSC_DIGEST_UNKNOWN; -+ return data; -+ } - -- } else if (strstr(digest_all, search_all)) { -+ // Check whether full parameter digest matches -+ if (unfencing_digest_matches(rsc->id, agent, data->digest_all_calc, -+ node_summary)) { - data->rc = RSC_DIGEST_MATCH; -+ return data; -+ } - -- } else if(digest_secure && data->digest_secure_calc) { -- if(strstr(digest_secure, search_secure)) { -- if (is_set(data_set->flags, pe_flag_stdout)) { -- printf("Only 'private' parameters to %s for unfencing %s changed\n", -- rsc->id, node->details->uname); -- } -- data->rc = RSC_DIGEST_MATCH; -+ // Check whether secure parameter digest matches -+ node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE); -+ if (unfencing_digest_matches(rsc->id, agent, data->digest_secure_calc, -+ node_summary)) { -+ data->rc = RSC_DIGEST_MATCH; -+ if (is_set(data_set->flags, pe_flag_stdout)) { -+ printf("Only 'private' parameters to %s for unfencing %s changed\n", -+ rsc->id, node->details->uname); - } -+ return data; - } - -- if (is_set(data_set->flags, pe_flag_sanitized) -- && is_set(data_set->flags, pe_flag_stdout) -- && (data->rc == RSC_DIGEST_ALL) -+ // Parameters don't match -+ data->rc = RSC_DIGEST_ALL; -+ if (is_set(data_set->flags, (pe_flag_sanitized|pe_flag_stdout)) - && data->digest_secure_calc) { -- printf("Parameters to %s for unfencing %s changed, try '%s:%s:%s'\n", -- rsc->id, node->details->uname, rsc->id, -- (const char *) g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), -- data->digest_secure_calc); -- } -- -- free(key); -- free(search_all); -- free(search_secure); -+ char *digest = create_unfencing_summary(rsc->id, agent, -+ data->digest_secure_calc); - -+ printf("Parameters to %s for unfencing %s changed, try '%s'\n", -+ rsc->id, node->details->uname, digest); -+ free(digest); -+ } - return data; - } - -@@ -2228,9 +2305,6 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe - * - * We may do this for all nodes in the future, but for now - * the check_action_definition() based stuff works fine. -- * -- * Use "stonith-on" to avoid creating cache entries for -- * operations check_action_definition() would look for. - */ - long max = 1024; - long digests_all_offset = 0; -@@ -2242,8 +2316,11 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe - - for (GListPtr gIter = matches; gIter != NULL; gIter = gIter->next) { - resource_t *match = gIter->data; -- op_digest_cache_t *data = fencing_action_digest_cmp(match, node, data_set); -+ const char *agent = g_hash_table_lookup(match->meta, -+ XML_ATTR_TYPE); -+ op_digest_cache_t *data = NULL; - -+ data = fencing_action_digest_cmp(match, agent, node, data_set); - if(data->rc == RSC_DIGEST_ALL) { - optional = FALSE; - crm_notice("Unfencing %s (remote): because the definition of %s changed", node->details->uname, match->id); -@@ -2254,11 +2331,11 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe - - digests_all_offset += snprintf( - digests_all+digests_all_offset, max-digests_all_offset, -- "%s:%s:%s,", match->id, (const char*)g_hash_table_lookup(match->meta, XML_ATTR_TYPE), data->digest_all_calc); -+ "%s:%s:%s,", match->id, agent, data->digest_all_calc); - - digests_secure_offset += snprintf( - digests_secure+digests_secure_offset, max-digests_secure_offset, -- "%s:%s:%s,", match->id, (const char*)g_hash_table_lookup(match->meta, XML_ATTR_TYPE), data->digest_secure_calc); -+ "%s:%s:%s,", match->id, agent, data->digest_secure_calc); - } - g_hash_table_insert(stonith_op->meta, - strdup(XML_OP_ATTR_DIGESTS_ALL), --- -1.8.3.1 - - -From be34a73f9cfb6abdb3e2799593cb0358c01c2521 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot <kgaillot@redhat.com> -Date: Fri, 10 May 2019 11:57:31 -0500 -Subject: [PATCH 2/6] Fix: libpe_status: calculate secure digests for unfencing - ops - -The calculation of digests for detection of when unfencing is needed reused -rsc_action_digest(). However that would only add secure digests when the -pe_flag_sanitized flag was set, which is only set by crm_simulate, so secure -digests would never be added in normal cluster operation. This led to -node attributes like name="#digests-secure" -value="stonith-fence_compute-fence-nova:fence_compute:(null),". - -Now, rsc_action_digest() takes a new argument to select whether secure digests -are added, which is always set to TRUE when calculating unfencing digests. ---- - lib/pengine/utils.c | 27 ++++++++++++++++++++++----- - 1 file changed, 22 insertions(+), 5 deletions(-) - -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index b6a31d1..f52f1c7 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -1948,9 +1948,24 @@ append_versioned_params(xmlNode *versioned_params, const char *ra_version, xmlNo - } - #endif - -+/*! -+ * \internal -+ * \brief Calculate action digests and store in node's digest cache -+ * -+ * \param[in] rsc Resource that action was for -+ * \param[in] task Name of action performed -+ * \param[in] key Action's task key -+ * \param[in] node Node action was performed on -+ * \param[in] xml_op XML of operation in CIB status (if available) -+ * \param[in] calc_secure Whether to calculate secure digest -+ * \param[in] data_set Cluster working set -+ * -+ * \return Pointer to node's digest cache entry -+ */ - static op_digest_cache_t * --rsc_action_digest(resource_t * rsc, const char *task, const char *key, -- node_t * node, xmlNode * xml_op, pe_working_set_t * data_set) -+rsc_action_digest(pe_resource_t *rsc, const char *task, const char *key, -+ pe_node_t *node, xmlNode *xml_op, bool calc_secure, -+ pe_working_set_t *data_set) - { - op_digest_cache_t *data = NULL; - -@@ -2018,7 +2033,7 @@ rsc_action_digest(resource_t * rsc, const char *task, const char *key, - - data->digest_all_calc = calculate_operation_digest(data->params_all, op_version); - -- if (is_set(data_set->flags, pe_flag_sanitized)) { -+ if (calc_secure) { - data->params_secure = copy_xml(data->params_all); - if(secure_list) { - filter_parameters(data->params_secure, secure_list, FALSE); -@@ -2064,7 +2079,9 @@ rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, - - interval = crm_parse_int(interval_s, "0"); - key = generate_op_key(rsc->id, task, interval); -- data = rsc_action_digest(rsc, task, key, node, xml_op, data_set); -+ data = rsc_action_digest(rsc, task, key, node, xml_op, -+ is_set(data_set->flags, pe_flag_sanitized), -+ data_set); - - data->rc = RSC_DIGEST_MATCH; - if (digest_restart && data->digest_restart_calc && strcmp(data->digest_restart_calc, digest_restart) != 0) { -@@ -2178,7 +2195,7 @@ fencing_action_digest_cmp(pe_resource_t *rsc, const char *agent, - // Calculate device's current parameter digests - char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); - op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, -- node, NULL, data_set); -+ node, NULL, TRUE, data_set); - - free(key); - --- -1.8.3.1 - - -From 8819c2f96f74ab4b4979df5ed04c16dd6bdad5f1 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot <kgaillot@redhat.com> -Date: Sat, 8 Jun 2019 16:25:04 -0500 -Subject: [PATCH 3/6] Refactor: libpe_status: add function for checking - shutdown attribute - -... to reduce code duplication and allow further reuse ---- - include/crm/pengine/internal.h | 2 ++ - lib/pengine/unpack.c | 8 ++------ - lib/pengine/utils.c | 20 ++++++++++++++++++++ - 3 files changed, 24 insertions(+), 6 deletions(-) - -diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h -index c40b075..c3f9f70 100644 ---- a/include/crm/pengine/internal.h -+++ b/include/crm/pengine/internal.h -@@ -362,4 +362,6 @@ void pe__foreach_param_check(pe_working_set_t *data_set, - enum pe_check_parameters, - pe_working_set_t*)); - void pe__free_param_checks(pe_working_set_t *data_set); -+ -+bool pe__shutdown_requested(pe_node_t *node); - #endif -diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c -index 619ccbf..cf725a1 100644 ---- a/lib/pengine/unpack.c -+++ b/lib/pengine/unpack.c -@@ -1013,7 +1013,6 @@ unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * - const char *resource_discovery_enabled = NULL; - xmlNode *attrs = NULL; - resource_t *rsc = NULL; -- const char *shutdown = NULL; - - if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) { - return; -@@ -1035,8 +1034,7 @@ unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * - attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE); - add_node_attrs(attrs, this_node, TRUE, data_set); - -- shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); -- if (shutdown != NULL && safe_str_neq("0", shutdown)) { -+ if (pe__shutdown_requested(this_node)) { - crm_info("Node %s is shutting down", this_node->details->uname); - this_node->details->shutdown = TRUE; - if (rsc) { -@@ -1512,7 +1510,6 @@ gboolean - determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set_t * data_set) - { - gboolean online = FALSE; -- const char *shutdown = NULL; - const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED); - - if (this_node == NULL) { -@@ -1522,9 +1519,8 @@ determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set - - this_node->details->shutdown = FALSE; - this_node->details->expected_up = FALSE; -- shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); - -- if (shutdown != NULL && safe_str_neq("0", shutdown)) { -+ if (pe__shutdown_requested(this_node)) { - this_node->details->shutdown = TRUE; - - } else if (safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) { -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index f52f1c7..8eac2ce 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -2522,3 +2522,23 @@ void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrit - } - } - } -+ -+/*! -+ * \internal -+ * \brief Check whether shutdown has been requested for a node -+ * -+ * \param[in] node Node to check -+ * -+ * \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise -+ * \note This differs from simply using node->details->shutdown in that it can -+ * be used before that has been determined (and in fact to determine it), -+ * and it can also be used to distinguish requested shutdown from implicit -+ * shutdown of remote nodes by virtue of their connection stopping. -+ */ -+bool -+pe__shutdown_requested(pe_node_t *node) -+{ -+ const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN); -+ -+ return shutdown && strcmp(shutdown, "0"); -+} --- -1.8.3.1 - - -From 938e99f29ed5faaeb4015247e363ddc7e77208a3 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot <kgaillot@redhat.com> -Date: Wed, 5 Jun 2019 16:37:26 -0500 -Subject: [PATCH 4/6] Fix: scheduler: remote state is failed if node is - shutting down with connection failure - -When determining remote state, if the connection resource is failed and not -being started again, we consider the state to be unknown if the connection has -a reconnect interval, because we won't know whether the connection can be -recovered until the interval expires and we re-attempt connection. - -However, if the node is shutting down at the time, we won't re-attempt -connection, so consider the state failed in that case. (Note that we check the -actual shutdown node attribute, rather than node->details->shutdown, since that -is set for remote nodes whenever the connection is stopping.) - -This avoids a situation where actions that cannot succeed can be scheduled on a -remote node that's shutting down. ---- - pengine/allocate.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/pengine/allocate.c b/pengine/allocate.c -index 578db2f..c9877a4 100644 ---- a/pengine/allocate.c -+++ b/pengine/allocate.c -@@ -1998,7 +1998,8 @@ get_remote_node_state(pe_node_t *node) - - if ((remote_rsc->next_role == RSC_ROLE_STOPPED) - && remote_rsc->remote_reconnect_interval -- && node->details->remote_was_fenced) { -+ && node->details->remote_was_fenced -+ && !pe__shutdown_requested(node)) { - - /* We won't know whether the connection is recoverable until the - * reconnect interval expires and we reattempt connection. --- -1.8.3.1 - - -From c20f8920634f47bbdf699d80dafd50c6a72eac8b Mon Sep 17 00:00:00 2001 -From: Ken Gaillot <kgaillot@redhat.com> -Date: Wed, 5 Jun 2019 16:43:19 -0500 -Subject: [PATCH 5/6] Fix: libpe_status: don't order implied stops relative to - a remote connection - -Actions behind a remote connection are ordered relative to any start or stop of -the remote connection. However, if the action is a stop implied due to fencing, -it does not require the remote connection, and the ordering should not be done. - -This avoids a delay in the remote connection recovery if it is failed, e.g. -previously the ordering would look like: - - fence remote node -> implied stop of resource on remote -> stop connection - -Now, the connection stop can proceed simultaneously with the remote node -fencing. ---- - pengine/allocate.c | 11 +++++------ - 1 file changed, 5 insertions(+), 6 deletions(-) - -diff --git a/pengine/allocate.c b/pengine/allocate.c -index c9877a4..c7c68f8 100644 ---- a/pengine/allocate.c -+++ b/pengine/allocate.c -@@ -2091,14 +2091,13 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) - pe_order_implies_first, data_set); - - } else if(state == remote_state_failed) { -- /* We would only be here if the resource is -- * running on the remote node. Since we have no -- * way to stop it, it is necessary to fence the -- * node. -+ /* The resource is active on the node, but since we don't have a -+ * valid connection, the only way to stop the resource is by -+ * fencing the node. There is no need to order the stop relative -+ * to the remote connection, since the stop will become implied -+ * by the fencing. - */ - pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable"); -- order_action_then_stop(action, remote_rsc, -- pe_order_implies_first, data_set); - - } else if(remote_rsc->next_role == RSC_ROLE_STOPPED) { - /* State must be remote_state_unknown or remote_state_stopped. --- -1.8.3.1 - - -From 26a28ee80b7fc110125eedac377dfa4c0a8e8294 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot <kgaillot@redhat.com> -Date: Fri, 14 Jun 2019 14:08:47 -0500 -Subject: [PATCH 6/6] Test: pengine: update regression tests for remote - connection ordering change - ---- - pengine/test10/remote-connection-unrecoverable.dot | 2 -- - pengine/test10/remote-connection-unrecoverable.exp | 6 ------ - pengine/test10/remote-connection-unrecoverable.summary | 2 +- - pengine/test10/remote-fence-before-reconnect.dot | 1 - - pengine/test10/remote-fence-before-reconnect.exp | 6 +----- - pengine/test10/remote-fence-before-reconnect.summary | 2 +- - pengine/test10/remote-recover-all.dot | 2 -- - pengine/test10/remote-recover-all.exp | 6 ------ - pengine/test10/remote-recover-all.summary | 4 ++-- - pengine/test10/remote-recover-no-resources.dot | 1 - - pengine/test10/remote-recover-no-resources.exp | 3 --- - pengine/test10/remote-recover-no-resources.summary | 2 +- - pengine/test10/remote-recover-unknown.dot | 1 - - pengine/test10/remote-recover-unknown.exp | 3 --- - pengine/test10/remote-recover-unknown.summary | 2 +- - 15 files changed, 7 insertions(+), 36 deletions(-) - -diff --git a/pengine/test10/remote-connection-unrecoverable.dot b/pengine/test10/remote-connection-unrecoverable.dot -index 0360cd0..b5caca6 100644 ---- a/pengine/test10/remote-connection-unrecoverable.dot -+++ b/pengine/test10/remote-connection-unrecoverable.dot -@@ -7,14 +7,12 @@ digraph "g" { - "remote1_stop_0 node1" [ style=bold color="green" fontcolor="orange"] - "rsc1_delete_0 remote1" -> "rsc1_start_0 node2" [ style = dashed] - "rsc1_delete_0 remote1" [ style=dashed color="red" fontcolor="black"] --"rsc1_monitor_0 node2" -> "remote1_stop_0 node1" [ style = bold] - "rsc1_monitor_0 node2" -> "rsc1_start_0 node2" [ style = bold] - "rsc1_monitor_0 node2" -> "rsc2-master_demote_0" [ style = bold] - "rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black"] - "rsc1_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] - "rsc1_start_0 node2" -> "rsc1_monitor_10000 node2" [ style = bold] - "rsc1_start_0 node2" [ style=bold color="green" fontcolor="black"] --"rsc1_stop_0 remote1" -> "remote1_stop_0 node1" [ style = bold] - "rsc1_stop_0 remote1" -> "rsc1_delete_0 remote1" [ style = dashed] - "rsc1_stop_0 remote1" -> "rsc1_start_0 node2" [ style = bold] - "rsc1_stop_0 remote1" -> "rsc2-master_demote_0" [ style = bold] -diff --git a/pengine/test10/remote-connection-unrecoverable.exp b/pengine/test10/remote-connection-unrecoverable.exp -index 73fa7a1..339ad56 100644 ---- a/pengine/test10/remote-connection-unrecoverable.exp -+++ b/pengine/test10/remote-connection-unrecoverable.exp -@@ -9,12 +9,6 @@ - <trigger> - <crm_event id="1" operation="stonith" operation_key="stonith-node1-reboot" on_node="node1" on_node_uuid="1"/> - </trigger> -- <trigger> -- <pseudo_event id="6" operation="stop" operation_key="rsc1_stop_0"/> -- </trigger> -- <trigger> -- <rsc_op id="8" operation="monitor" operation_key="rsc1_monitor_0" on_node="node2" on_node_uuid="2"/> -- </trigger> - </inputs> - </synapse> - <synapse id="1"> -diff --git a/pengine/test10/remote-connection-unrecoverable.summary b/pengine/test10/remote-connection-unrecoverable.summary -index efeb765..18f7dc7 100644 ---- a/pengine/test10/remote-connection-unrecoverable.summary -+++ b/pengine/test10/remote-connection-unrecoverable.summary -@@ -24,12 +24,12 @@ Executing cluster transition: - * Resource action: killer stop on node2 - * Resource action: rsc1 monitor on node2 - * Fencing node1 (reboot) -+ * Pseudo action: remote1_stop_0 - * Fencing remote1 (reboot) - * Resource action: killer start on node2 - * Resource action: killer monitor=60000 on node2 - * Pseudo action: rsc1_stop_0 - * Pseudo action: rsc2-master_demote_0 -- * Pseudo action: remote1_stop_0 - * Resource action: rsc1 start on node2 - * Pseudo action: rsc2_demote_0 - * Pseudo action: rsc2-master_demoted_0 -diff --git a/pengine/test10/remote-fence-before-reconnect.dot b/pengine/test10/remote-fence-before-reconnect.dot -index 4ced43e..5812b7f 100644 ---- a/pengine/test10/remote-fence-before-reconnect.dot -+++ b/pengine/test10/remote-fence-before-reconnect.dot -@@ -3,7 +3,6 @@ - "fake2_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"] - "fake2_start_0 c7auto1" -> "fake2_monitor_10000 c7auto1" [ style = bold] - "fake2_start_0 c7auto1" [ style=bold color="green" fontcolor="black"] --"fake2_stop_0 c7auto4" -> "c7auto4_stop_0 c7auto1" [ style = bold] - "fake2_stop_0 c7auto4" -> "fake2_start_0 c7auto1" [ style = bold] - "fake2_stop_0 c7auto4" [ style=bold color="green" fontcolor="orange"] - "stonith 'reboot' c7auto4" -> "fake2_start_0 c7auto1" [ style = bold] -diff --git a/pengine/test10/remote-fence-before-reconnect.exp b/pengine/test10/remote-fence-before-reconnect.exp -index f99d9ef..f506f85 100644 ---- a/pengine/test10/remote-fence-before-reconnect.exp -+++ b/pengine/test10/remote-fence-before-reconnect.exp -@@ -9,11 +9,7 @@ - </downed> - </rsc_op> - </action_set> -- <inputs> -- <trigger> -- <pseudo_event id="13" operation="stop" operation_key="fake2_stop_0"/> -- </trigger> -- </inputs> -+ <inputs/> - </synapse> - <synapse id="1"> - <action_set> -diff --git a/pengine/test10/remote-fence-before-reconnect.summary b/pengine/test10/remote-fence-before-reconnect.summary -index f61e18b..03eac20 100644 ---- a/pengine/test10/remote-fence-before-reconnect.summary -+++ b/pengine/test10/remote-fence-before-reconnect.summary -@@ -17,9 +17,9 @@ Transition Summary: - * Move fake2 ( c7auto4 -> c7auto1 ) - - Executing cluster transition: -+ * Resource action: c7auto4 stop on c7auto1 - * Fencing c7auto4 (reboot) - * Pseudo action: fake2_stop_0 -- * Resource action: c7auto4 stop on c7auto1 - * Resource action: fake2 start on c7auto1 - * Resource action: fake2 monitor=10000 on c7auto1 - -diff --git a/pengine/test10/remote-recover-all.dot b/pengine/test10/remote-recover-all.dot -index 1f967c5..b48b04e 100644 ---- a/pengine/test10/remote-recover-all.dot -+++ b/pengine/test10/remote-recover-all.dot -@@ -19,7 +19,6 @@ digraph "g" { - "galera_demote_0 galera-2" -> "galera_stop_0 galera-2" [ style = bold] - "galera_demote_0 galera-2" [ style=bold color="green" fontcolor="orange"] - "galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"] --"galera_stop_0 galera-2" -> "galera-2_stop_0 controller-1" [ style = bold] - "galera_stop_0 galera-2" -> "galera-master_stopped_0" [ style = bold] - "galera_stop_0 galera-2" [ style=bold color="green" fontcolor="orange"] - "haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold] -@@ -60,7 +59,6 @@ digraph "g" { - "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] - "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] - "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] --"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] - "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] - "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] - "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] -diff --git a/pengine/test10/remote-recover-all.exp b/pengine/test10/remote-recover-all.exp -index 900781c..e61ad6a 100644 ---- a/pengine/test10/remote-recover-all.exp -+++ b/pengine/test10/remote-recover-all.exp -@@ -9,9 +9,6 @@ - <trigger> - <crm_event id="1" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/> - </trigger> -- <trigger> -- <pseudo_event id="39" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:2_stop_0"/> -- </trigger> - </inputs> - </synapse> - <synapse id="1"> -@@ -64,9 +61,6 @@ - <trigger> - <crm_event id="1" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/> - </trigger> -- <trigger> -- <pseudo_event id="49" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:1_stop_0"/> -- </trigger> - </inputs> - </synapse> - <synapse id="5" priority="1000000"> -diff --git a/pengine/test10/remote-recover-all.summary b/pengine/test10/remote-recover-all.summary -index 865f39a..cfeac3a 100644 ---- a/pengine/test10/remote-recover-all.summary -+++ b/pengine/test10/remote-recover-all.summary -@@ -63,6 +63,8 @@ Executing cluster transition: - * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 - * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 - * Fencing controller-1 (reboot) -+ * Pseudo action: messaging-1_stop_0 -+ * Pseudo action: galera-2_stop_0 - * Pseudo action: redis_post_notify_stop_0 - * Resource action: redis notify on controller-0 - * Resource action: redis notify on controller-2 -@@ -94,7 +96,6 @@ Executing cluster transition: - * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 - * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 - * Resource action: galera-0 monitor=20000 on controller-2 -- * Pseudo action: galera-2_stop_0 - * Resource action: rabbitmq notify on messaging-2 - * Resource action: rabbitmq notify on messaging-0 - * Pseudo action: rabbitmq_notified_0 -@@ -107,7 +108,6 @@ Executing cluster transition: - * Resource action: ip-172.17.1.17 start on controller-2 - * Resource action: ip-172.17.4.11 start on controller-2 - * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 -- * Pseudo action: messaging-1_stop_0 - * Pseudo action: redis_notified_0 - * Resource action: ip-172.17.1.14 monitor=10000 on controller-2 - * Resource action: ip-172.17.1.17 monitor=10000 on controller-2 -diff --git a/pengine/test10/remote-recover-no-resources.dot b/pengine/test10/remote-recover-no-resources.dot -index a46c305..a0b1ecc 100644 ---- a/pengine/test10/remote-recover-no-resources.dot -+++ b/pengine/test10/remote-recover-no-resources.dot -@@ -45,7 +45,6 @@ digraph "g" { - "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] - "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] - "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] --"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] - "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] - "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] - "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] -diff --git a/pengine/test10/remote-recover-no-resources.exp b/pengine/test10/remote-recover-no-resources.exp -index 4d82aa4..27f18b5 100644 ---- a/pengine/test10/remote-recover-no-resources.exp -+++ b/pengine/test10/remote-recover-no-resources.exp -@@ -9,9 +9,6 @@ - <trigger> - <crm_event id="1" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/> - </trigger> -- <trigger> -- <pseudo_event id="38" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:2_stop_0"/> -- </trigger> - </inputs> - </synapse> - <synapse id="1"> -diff --git a/pengine/test10/remote-recover-no-resources.summary b/pengine/test10/remote-recover-no-resources.summary -index 9527161..c01eb87 100644 ---- a/pengine/test10/remote-recover-no-resources.summary -+++ b/pengine/test10/remote-recover-no-resources.summary -@@ -60,6 +60,7 @@ Executing cluster transition: - * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 - * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 - * Fencing controller-1 (reboot) -+ * Pseudo action: messaging-1_stop_0 - * Pseudo action: galera-2_stop_0 - * Pseudo action: redis_post_notify_stop_0 - * Resource action: redis notify on controller-0 -@@ -92,7 +93,6 @@ Executing cluster transition: - * Pseudo action: ip-172.17.1.17_stop_0 - * Pseudo action: ip-172.17.4.11_stop_0 - * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 -- * Pseudo action: messaging-1_stop_0 - * Resource action: redis notify on controller-0 - * Resource action: redis notify on controller-2 - * Pseudo action: redis-master_confirmed-post_notify_stopped_0 -diff --git a/pengine/test10/remote-recover-unknown.dot b/pengine/test10/remote-recover-unknown.dot -index a883eb4..1d13e50 100644 ---- a/pengine/test10/remote-recover-unknown.dot -+++ b/pengine/test10/remote-recover-unknown.dot -@@ -46,7 +46,6 @@ digraph "g" { - "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] - "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] - "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] --"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] - "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] - "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] - "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] -diff --git a/pengine/test10/remote-recover-unknown.exp b/pengine/test10/remote-recover-unknown.exp -index 65677b4..13bd295 100644 ---- a/pengine/test10/remote-recover-unknown.exp -+++ b/pengine/test10/remote-recover-unknown.exp -@@ -9,9 +9,6 @@ - <trigger> - <crm_event id="1" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/> - </trigger> -- <trigger> -- <pseudo_event id="39" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:2_stop_0"/> -- </trigger> - </inputs> - </synapse> - <synapse id="1"> -diff --git a/pengine/test10/remote-recover-unknown.summary b/pengine/test10/remote-recover-unknown.summary -index 78a60d0..64f37cb 100644 ---- a/pengine/test10/remote-recover-unknown.summary -+++ b/pengine/test10/remote-recover-unknown.summary -@@ -61,6 +61,7 @@ Executing cluster transition: - * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 - * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 - * Fencing controller-1 (reboot) -+ * Pseudo action: messaging-1_stop_0 - * Pseudo action: galera-2_stop_0 - * Pseudo action: redis_post_notify_stop_0 - * Resource action: redis notify on controller-0 -@@ -94,7 +95,6 @@ Executing cluster transition: - * Pseudo action: ip-172.17.1.17_stop_0 - * Pseudo action: ip-172.17.4.11_stop_0 - * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 -- * Pseudo action: messaging-1_stop_0 - * Resource action: redis notify on controller-0 - * Resource action: redis notify on controller-2 - * Pseudo action: redis-master_confirmed-post_notify_stopped_0 --- -1.8.3.1 - diff --git a/SOURCES/012-tls-priorities.patch b/SOURCES/012-tls-priorities.patch deleted file mode 100644 index 34396a2..0000000 --- a/SOURCES/012-tls-priorities.patch +++ /dev/null @@ -1,189 +0,0 @@ -From 7c3bc762a9cede20a0193f64ca1a36f507aeeeb3 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot <kgaillot@redhat.com> -Date: Fri, 20 Apr 2018 13:23:10 -0500 -Subject: [PATCH 1/2] Build: libcrmcommon: configure option to specify GnuTLS - cipher priorities - -Default to current behavior, i.e. "NORMAL". Spec file overrides with "@SYSTEM" -on distros that have it. - -Pacemaker does not use option value as-is; it adds "+ANON-DH" for CIB remote -commands and "+DHE-PSK:+PSK" for Pacemaker Remote connections. In the longer -term, we could consider moving to certificate-based connections in both cases, -but that has backward compatibility issues as well as additional administrative -burden. ---- - configure.ac | 9 +++++++++ - lib/common/remote.c | 4 ++-- - pacemaker.spec.in | 4 ++++ - 3 files changed, 15 insertions(+), 2 deletions(-) - -diff --git a/configure.ac b/configure.ac -index ce02777..a7084e2 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -290,6 +290,12 @@ AC_ARG_WITH(cibsecrets, - [ SUPPORT_CIBSECRETS=no ], - ) - -+AC_ARG_WITH(gnutls-priorities, -+ [ --with-gnutls-priorities GnuTLS cipher priorities @<:@NORMAL@:>@ ], -+ [ PCMK_GNUTLS_PRIORITIES="$withval" ], -+ [ PCMK_GNUTLS_PRIORITIES="NORMAL" ], -+) -+ - CSPREFIX="" - AC_ARG_WITH(ais-prefix, - [ --with-ais-prefix=DIR Prefix used when Corosync was installed [$prefix]], -@@ -453,6 +459,9 @@ if test x"${BUG_URL}" = x""; then - fi - AC_SUBST(BUG_URL) - -+AC_DEFINE_UNQUOTED([PCMK_GNUTLS_PRIORITIES], ["$PCMK_GNUTLS_PRIORITIES"], -+ [GnuTLS cipher priorities]) -+ - for j in prefix exec_prefix bindir sbindir libexecdir datadir sysconfdir \ - sharedstatedir localstatedir libdir includedir oldincludedir infodir \ - mandir INITDIR docdir CONFIGDIR -diff --git a/lib/common/remote.c b/lib/common/remote.c -index 12d25fa..1e4f8d8 100644 ---- a/lib/common/remote.c -+++ b/lib/common/remote.c -@@ -244,9 +244,9 @@ pcmk__new_tls_session(int csock, unsigned int conn_type, - # ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT - if (cred_type == GNUTLS_CRD_ANON) { - // http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication -- prio = "NORMAL:+ANON-DH"; -+ prio = PCMK_GNUTLS_PRIORITIES ":+ANON-DH"; - } else { -- prio = "NORMAL:+DHE-PSK:+PSK"; -+ prio = PCMK_GNUTLS_PRIORITIES ":+DHE-PSK:+PSK"; - } - # endif - -diff --git a/pacemaker.spec.in b/pacemaker.spec.in -index 3a26572..fd0e3c8 100644 ---- a/pacemaker.spec.in -+++ b/pacemaker.spec.in -@@ -80,6 +80,9 @@ - } || %{?__transaction_systemd_inhibit:1}%{!?__transaction_systemd_inhibit:0}%{nil \ - } || %(test -f /usr/lib/os-release; test $? -ne 0; echo $?)) - -+%if 0%{?fedora} > 20 || 0%{?rhel} > 7 -+%global gnutls_priorities @SYSTEM -+%endif - - # Definitions for backward compatibility with older RPM versions - -@@ -403,6 +406,7 @@ export LDFLAGS_HARDENED_LIB="%{?_hardening_ldflags}" - --without-heartbeat \ - %{!?with_doc: --with-brand=} \ - %{!?with_hardening: --disable-hardening} \ -+ %{?gnutls_priorities: --with-gnutls-priorities="%{gnutls_priorities}"} \ - --with-initdir=%{_initrddir} \ - --localstatedir=%{_var} \ - --with-version=%{version}-%{release} --- -1.8.3.1 - - -From 99a83b172544102ec32585514e5808585f2ce31c Mon Sep 17 00:00:00 2001 -From: Ken Gaillot <kgaillot@redhat.com> -Date: Mon, 8 Jul 2019 17:39:12 -0500 -Subject: [PATCH 2/2] Feature: remote: allow run-time configurable TLS - priorities - -This also restores compilability with GnuTLS <2.1.7 (not that anyone is still -using that ...), unintentionally broken in 5bded36 (1.1.20). ---- - lib/common/remote.c | 34 +++++++++++++++++++++++++++------- - mcp/pacemaker.sysconfig | 9 +++++++++ - 2 files changed, 36 insertions(+), 7 deletions(-) - -diff --git a/lib/common/remote.c b/lib/common/remote.c -index 1e4f8d8..ccd0342 100644 ---- a/lib/common/remote.c -+++ b/lib/common/remote.c -@@ -237,17 +237,25 @@ pcmk__new_tls_session(int csock, unsigned int conn_type, - { - int rc = GNUTLS_E_SUCCESS; - # ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT -- const char *prio = NULL; -+ const char *prio_base = NULL; -+ char *prio = NULL; - # endif - gnutls_session_t *session = NULL; - - # ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT -- if (cred_type == GNUTLS_CRD_ANON) { -- // http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication -- prio = PCMK_GNUTLS_PRIORITIES ":+ANON-DH"; -- } else { -- prio = PCMK_GNUTLS_PRIORITIES ":+DHE-PSK:+PSK"; -+ /* Determine list of acceptable ciphers, etc. Pacemaker always adds the -+ * values required for its functionality. -+ * -+ * For an example of anonymous authentication, see: -+ * http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication -+ */ -+ -+ prio_base = getenv("PCMK_tls_priorities"); -+ if (prio_base == NULL) { -+ prio_base = PCMK_GNUTLS_PRIORITIES; - } -+ prio = crm_strdup_printf("%s:%s", prio_base, -+ (cred_type == GNUTLS_CRD_ANON)? "+ANON-DH" : "+DHE-PSK:+PSK"); - # endif - - session = gnutls_malloc(sizeof(gnutls_session_t)); -@@ -285,6 +293,9 @@ pcmk__new_tls_session(int csock, unsigned int conn_type, - if (rc != GNUTLS_E_SUCCESS) { - goto error; - } -+# ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT -+ free(prio); -+# endif - return session; - - error: -@@ -292,7 +303,16 @@ error: - CRM_XS " rc=%d priority='%s'", - (cred_type == GNUTLS_CRD_ANON)? "anonymous" : "PSK", - (conn_type == GNUTLS_SERVER)? "server" : "client", -- gnutls_strerror(rc), rc, prio); -+ gnutls_strerror(rc), rc, -+# ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT -+ prio -+# else -+ "default" -+# endif -+ ); -+# ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT -+ free(prio); -+# endif - if (session != NULL) { - gnutls_free(session); - } -diff --git a/mcp/pacemaker.sysconfig b/mcp/pacemaker.sysconfig -index a983011..0da401e 100644 ---- a/mcp/pacemaker.sysconfig -+++ b/mcp/pacemaker.sysconfig -@@ -101,6 +101,15 @@ - # value must be the same on all nodes. The default is "3121". - # PCMK_remote_port=3121 - -+# Use these GnuTLS cipher priorities for TLS connections. See: -+# -+# https://gnutls.org/manual/html_node/Priority-Strings.html -+# -+# Pacemaker will append ":+ANON-DH" for remote CIB access (when enabled) and -+# ":+DHE-PSK:+PSK" for Pacemaker Remote connections, as they are required for -+# the respective functionality. -+# PCMK_tls_priorities="NORMAL" -+ - # Set bounds on the bit length of the prime number generated for Diffie-Hellman - # parameters needed by TLS connections. The default is not to set any bounds. - # --- -1.8.3.1 - diff --git a/SOURCES/013-guest-node.patch b/SOURCES/013-guest-node.patch deleted file mode 100644 index 1291635..0000000 --- a/SOURCES/013-guest-node.patch +++ /dev/null @@ -1,126 +0,0 @@ -From 11685256d35035ae69985d1f4536d0ed68951efe Mon Sep 17 00:00:00 2001 -From: Ken Gaillot <kgaillot@redhat.com> -Date: Thu, 24 Oct 2019 17:35:48 -0500 -Subject: [PATCH 4/5] Fix: scheduler: properly detect whether guest node is - fenceable - -Guest nodes are "fenced" by stopping their container resource. Previously, we -assumed that this was always possible. However, it may not be if the -container's host is failed and not fenceable (e.g. due to lack of quorum). - -Now, we check guest nodes for fenceability as we do for other nodes, -with the criteria being that the guest's host must be either online or -fenceable. Additionally, when creating a new action that normally does not -require fencing, we make the action unrunnable if it is on an non-fenceable -guest node, because the action cannot be attempted in that case. ---- - lib/pengine/utils.c | 55 ++++++++++++++++++++++++++++++++++++++--------------- - pengine/allocate.c | 3 ++- - 2 files changed, 42 insertions(+), 16 deletions(-) - -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index 97241af..671ef76 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -92,36 +92,49 @@ pe_free_rsc_action_details(pe_action_t *action) - * \param[in] data_set Working set for cluster - * \param[in] node Name of node to check - * -- * \return TRUE if node can be fenced, FALSE otherwise -- * -- * \note This function should only be called for cluster nodes and baremetal -- * remote nodes; guest nodes are fenced by stopping their container -- * resource, so fence execution requirements do not apply to them. -+ * \return true if node can be fenced, false otherwise - */ --bool pe_can_fence(pe_working_set_t * data_set, node_t *node) -+bool -+pe_can_fence(pe_working_set_t *data_set, pe_node_t *node) - { -- if(is_not_set(data_set->flags, pe_flag_stonith_enabled)) { -- return FALSE; /* Turned off */ -+ if (is_container_remote_node(node)) { -+ /* Guest nodes are fenced by stopping their container resource. We can -+ * do that if the container's host is either online or fenceable. -+ */ -+ pe_resource_t *rsc = node->details->remote_rsc->container; -+ -+ for (GList *n = rsc->running_on; n != NULL; n = n->next) { -+ pe_node_t *container_node = n->data; -+ -+ if (!container_node->details->online -+ && !pe_can_fence(data_set, container_node)) { -+ return false; -+ } -+ } -+ return true; -+ -+ } else if(is_not_set(data_set->flags, pe_flag_stonith_enabled)) { -+ return false; /* Turned off */ - - } else if (is_not_set(data_set->flags, pe_flag_have_stonith_resource)) { -- return FALSE; /* No devices */ -+ return false; /* No devices */ - - } else if (is_set(data_set->flags, pe_flag_have_quorum)) { -- return TRUE; -+ return true; - - } else if (data_set->no_quorum_policy == no_quorum_ignore) { -- return TRUE; -+ return true; - - } else if(node == NULL) { -- return FALSE; -+ return false; - - } else if(node->details->online) { - crm_notice("We can fence %s without quorum because they're in our membership", node->details->uname); -- return TRUE; -+ return true; - } - - crm_trace("Cannot fence %s", node->details->uname); -- return FALSE; -+ return false; - } - - node_t * -@@ -576,7 +589,19 @@ custom_action(resource_t * rsc, char *key, const char *task, - } else if (action->needs == rsc_req_nothing) { - pe_rsc_trace(rsc, "Action %s does not require anything", action->uuid); - pe_action_set_reason(action, NULL, TRUE); -- pe_set_action_bit(action, pe_action_runnable); -+ if (is_container_remote_node(action->node) -+ && !pe_can_fence(data_set, action->node)) { -+ /* An action that requires nothing usually does not require any -+ * fencing in order to be runnable. However, there is an -+ * exception: an action cannot be completed if it is on a guest -+ * node whose host is unclean and cannot be fenced. -+ */ -+ pe_clear_action_bit(action, pe_action_runnable); -+ crm_debug("%s\t%s (cancelled : host cannot be fenced)", -+ action->node->details->uname, action->uuid); -+ } else { -+ pe_set_action_bit(action, pe_action_runnable); -+ } - #if 0 - /* - * No point checking this -diff --git a/pengine/allocate.c b/pengine/allocate.c -index e30cb1c..b819af3 100644 ---- a/pengine/allocate.c -+++ b/pengine/allocate.c -@@ -1584,7 +1584,8 @@ stage6(pe_working_set_t * data_set) - * so handle them separately. - */ - if (is_container_remote_node(node)) { -- if (node->details->remote_requires_reset && need_stonith) { -+ if (node->details->remote_requires_reset && need_stonith -+ && pe_can_fence(data_set, node)) { - fence_guest(node, data_set); - } - continue; --- -1.8.3.1 - diff --git a/SOURCES/014-guest-node-test.patch b/SOURCES/014-guest-node-test.patch deleted file mode 100644 index 95cf581..0000000 --- a/SOURCES/014-guest-node-test.patch +++ /dev/null @@ -1,1297 +0,0 @@ -From 739fb14ad27e3a8ab18e92a23f0926c773983ad6 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot <kgaillot@redhat.com> -Date: Tue, 29 Oct 2019 17:44:40 -0500 -Subject: [PATCH 5/5] Test: scheduler: add regression test for guest node with - unclean host that cannot be fenced - ---- - pengine/regression.sh | 1 + - pengine/test10/guest-host-not-fenceable.dot | 258 +++++++++++++++ - pengine/test10/guest-host-not-fenceable.exp | 340 +++++++++++++++++++ - pengine/test10/guest-host-not-fenceable.scores | 134 ++++++++ - pengine/test10/guest-host-not-fenceable.summary | 87 +++++ - pengine/test10/guest-host-not-fenceable.xml | 413 ++++++++++++++++++++++++ - 6 files changed, 1233 insertions(+) - create mode 100644 pengine/test10/guest-host-not-fenceable.dot - create mode 100644 pengine/test10/guest-host-not-fenceable.exp - create mode 100644 pengine/test10/guest-host-not-fenceable.scores - create mode 100644 pengine/test10/guest-host-not-fenceable.summary - create mode 100755 pengine/test10/guest-host-not-fenceable.xml - -diff --git a/pengine/regression.sh b/pengine/regression.sh -index 25d9e3f..f2226ed 100755 ---- a/pengine/regression.sh -+++ b/pengine/regression.sh -@@ -862,6 +862,7 @@ do_test whitebox-imply-stop-on-fence "imply stop action on container node rsc wh - do_test whitebox-nested-group "Verify guest remote-node works nested in a group" - do_test guest-node-host-dies "Verify guest node is recovered if host goes away" - do_test guest-node-cleanup "Order guest node connection recovery after container probe" -+do_test guest-host-not-fenceable "Actions on guest node are unrunnable if host is unclean and cannot be fenced" - - echo "" - do_test remote-startup-probes "Baremetal remote-node startup probes" -diff --git a/pengine/test10/guest-host-not-fenceable.dot b/pengine/test10/guest-host-not-fenceable.dot -new file mode 100644 -index 0000000..a510aaf ---- /dev/null -+++ b/pengine/test10/guest-host-not-fenceable.dot -@@ -0,0 +1,258 @@ -+ digraph "g" { -+"galera-bundle-0_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] -+"galera-bundle-0_start_0 node1" -> "galera-bundle-0_monitor_60000 node1" [ style = dashed] -+"galera-bundle-0_start_0 node1" -> "galera_clear_failcount_0 galera-bundle-0" [ style = dashed] -+"galera-bundle-0_start_0 node1" -> "galera_monitor_10000 galera-bundle-0" [ style = dashed] -+"galera-bundle-0_start_0 node1" -> "galera_start_0 galera-bundle-0" [ style = dashed] -+"galera-bundle-0_start_0 node1" [ style=dashed color="red" fontcolor="black"] -+"galera-bundle-0_stop_0 node1" -> "galera-bundle-0_start_0 node1" [ style = dashed] -+"galera-bundle-0_stop_0 node1" -> "galera-bundle-docker-0_stop_0 node1" [ style = bold] -+"galera-bundle-0_stop_0 node1" [ style=bold color="green" fontcolor="black"] -+"galera-bundle-1_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] -+"galera-bundle-1_start_0 node1" -> "galera-bundle-1_monitor_60000 node1" [ style = dashed] -+"galera-bundle-1_start_0 node1" -> "galera_monitor_10000 galera-bundle-1" [ style = dashed] -+"galera-bundle-1_start_0 node1" -> "galera_start_0 galera-bundle-1" [ style = dashed] -+"galera-bundle-1_start_0 node1" [ style=dashed color="red" fontcolor="black"] -+"galera-bundle-1_stop_0 node2" -> "galera-bundle-1_start_0 node1" [ style = dashed] -+"galera-bundle-1_stop_0 node2" -> "galera-bundle-docker-1_stop_0 node2" [ style = dashed] -+"galera-bundle-1_stop_0 node2" [ style=dashed color="red" fontcolor="black"] -+"galera-bundle-2_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] -+"galera-bundle-2_start_0 node1" -> "galera-bundle-2_monitor_60000 node1" [ style = dashed] -+"galera-bundle-2_start_0 node1" -> "galera_monitor_20000 galera-bundle-2" [ style = dashed] -+"galera-bundle-2_start_0 node1" -> "galera_monitor_30000 galera-bundle-2" [ style = dashed] -+"galera-bundle-2_start_0 node1" -> "galera_start_0 galera-bundle-2" [ style = dashed] -+"galera-bundle-2_start_0 node1" [ style=dashed color="red" fontcolor="black"] -+"galera-bundle-2_stop_0 node3" -> "galera-bundle-2_start_0 node1" [ style = dashed] -+"galera-bundle-2_stop_0 node3" -> "galera-bundle-docker-2_stop_0 node3" [ style = dashed] -+"galera-bundle-2_stop_0 node3" [ style=dashed color="red" fontcolor="black"] -+"galera-bundle-docker-0_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] -+"galera-bundle-docker-0_start_0 node1" -> "galera-bundle-0_start_0 node1" [ style = dashed] -+"galera-bundle-docker-0_start_0 node1" -> "galera-bundle-docker-0_monitor_60000 node1" [ style = dashed] -+"galera-bundle-docker-0_start_0 node1" -> "galera-bundle_running_0" [ style = dashed] -+"galera-bundle-docker-0_start_0 node1" -> "galera_start_0 galera-bundle-0" [ style = dashed] -+"galera-bundle-docker-0_start_0 node1" [ style=dashed color="red" fontcolor="black"] -+"galera-bundle-docker-0_stop_0 node1" -> "galera-bundle-docker-0_start_0 node1" [ style = dashed] -+"galera-bundle-docker-0_stop_0 node1" -> "galera-bundle_stopped_0" [ style = bold] -+"galera-bundle-docker-0_stop_0 node1" [ style=bold color="green" fontcolor="black"] -+"galera-bundle-docker-1_stop_0 node2" -> "galera-bundle_stopped_0" [ style = dashed] -+"galera-bundle-docker-1_stop_0 node2" [ style=dashed color="red" fontcolor="black"] -+"galera-bundle-docker-2_stop_0 node3" -> "galera-bundle_stopped_0" [ style = dashed] -+"galera-bundle-docker-2_stop_0 node3" [ style=dashed color="red" fontcolor="black"] -+"galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold] -+"galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold] -+"galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-1" [ style = dashed] -+"galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-2" [ style = dashed] -+"galera-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] -+"galera-bundle-master_demoted_0" -> "galera-bundle-master_start_0" [ style = dashed] -+"galera-bundle-master_demoted_0" -> "galera-bundle-master_stop_0" [ style = bold] -+"galera-bundle-master_demoted_0" -> "galera-bundle_demoted_0" [ style = bold] -+"galera-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = dashed] -+"galera-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"] -+"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = dashed] -+"galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-0" [ style = dashed] -+"galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-1" [ style = dashed] -+"galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-2" [ style = dashed] -+"galera-bundle-master_start_0" [ style=dashed color="red" fontcolor="orange"] -+"galera-bundle-master_stop_0" -> "galera-bundle-master_stopped_0" [ style = bold] -+"galera-bundle-master_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] -+"galera-bundle-master_stop_0" -> "galera_stop_0 galera-bundle-1" [ style = dashed] -+"galera-bundle-master_stop_0" -> "galera_stop_0 galera-bundle-2" [ style = dashed] -+"galera-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] -+"galera-bundle-master_stopped_0" -> "galera-bundle-master_start_0" [ style = dashed] -+"galera-bundle-master_stopped_0" -> "galera-bundle_stopped_0" [ style = bold] -+"galera-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"galera-bundle_demote_0" -> "galera-bundle-master_demote_0" [ style = bold] -+"galera-bundle_demote_0" -> "galera-bundle_demoted_0" [ style = bold] -+"galera-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] -+"galera-bundle_demoted_0" -> "galera-bundle_start_0" [ style = dashed] -+"galera-bundle_demoted_0" -> "galera-bundle_stop_0" [ style = bold] -+"galera-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"galera-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] -+"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 node1" [ style = dashed] -+"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = dashed] -+"galera-bundle_start_0" [ style=dashed color="red" fontcolor="orange"] -+"galera-bundle_stop_0" -> "galera-bundle-docker-0_stop_0 node1" [ style = bold] -+"galera-bundle_stop_0" -> "galera-bundle-docker-1_stop_0 node2" [ style = dashed] -+"galera-bundle_stop_0" -> "galera-bundle-docker-2_stop_0 node3" [ style = dashed] -+"galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold] -+"galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] -+"galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-1" [ style = dashed] -+"galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-2" [ style = dashed] -+"galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] -+"galera-bundle_stopped_0" -> "galera-bundle_start_0" [ style = dashed] -+"galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"galera_clear_failcount_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] -+"galera_demote_0 galera-bundle-0" -> "galera-bundle-0_stop_0 node1" [ style = bold] -+"galera_demote_0 galera-bundle-0" -> "galera-bundle-master_demoted_0" [ style = bold] -+"galera_demote_0 galera-bundle-0" -> "galera_stop_0 galera-bundle-0" [ style = bold] -+"galera_demote_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] -+"galera_demote_0 galera-bundle-1" -> "galera-bundle-master_demoted_0" [ style = dashed] -+"galera_demote_0 galera-bundle-1" -> "galera_demote_0 galera-bundle-0" [ style = dashed] -+"galera_demote_0 galera-bundle-1" -> "galera_stop_0 galera-bundle-1" [ style = dashed] -+"galera_demote_0 galera-bundle-1" [ style=dashed color="red" fontcolor="black"] -+"galera_demote_0 galera-bundle-2" -> "galera-bundle-master_demoted_0" [ style = dashed] -+"galera_demote_0 galera-bundle-2" -> "galera_demote_0 galera-bundle-1" [ style = dashed] -+"galera_demote_0 galera-bundle-2" -> "galera_monitor_20000 galera-bundle-2" [ style = dashed] -+"galera_demote_0 galera-bundle-2" -> "galera_monitor_30000 galera-bundle-2" [ style = dashed] -+"galera_demote_0 galera-bundle-2" -> "galera_stop_0 galera-bundle-2" [ style = dashed] -+"galera_demote_0 galera-bundle-2" [ style=dashed color="red" fontcolor="black"] -+"galera_monitor_10000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] -+"galera_monitor_10000 galera-bundle-1" [ style=dashed color="red" fontcolor="black"] -+"galera_monitor_20000 galera-bundle-2" [ style=dashed color="red" fontcolor="black"] -+"galera_monitor_30000 galera-bundle-2" [ style=dashed color="red" fontcolor="black"] -+"galera_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = dashed] -+"galera_start_0 galera-bundle-0" -> "galera_monitor_10000 galera-bundle-0" [ style = dashed] -+"galera_start_0 galera-bundle-0" -> "galera_start_0 galera-bundle-1" [ style = dashed] -+"galera_start_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] -+"galera_start_0 galera-bundle-1" -> "galera-bundle-master_running_0" [ style = dashed] -+"galera_start_0 galera-bundle-1" -> "galera_monitor_10000 galera-bundle-1" [ style = dashed] -+"galera_start_0 galera-bundle-1" -> "galera_start_0 galera-bundle-2" [ style = dashed] -+"galera_start_0 galera-bundle-1" [ style=dashed color="red" fontcolor="black"] -+"galera_start_0 galera-bundle-2" -> "galera-bundle-master_running_0" [ style = dashed] -+"galera_start_0 galera-bundle-2" -> "galera_monitor_20000 galera-bundle-2" [ style = dashed] -+"galera_start_0 galera-bundle-2" -> "galera_monitor_30000 galera-bundle-2" [ style = dashed] -+"galera_start_0 galera-bundle-2" [ style=dashed color="red" fontcolor="black"] -+"galera_stop_0 galera-bundle-0" -> "galera-bundle-0_stop_0 node1" [ style = bold] -+"galera_stop_0 galera-bundle-0" -> "galera-bundle-master_stopped_0" [ style = bold] -+"galera_stop_0 galera-bundle-0" -> "galera_start_0 galera-bundle-0" [ style = dashed] -+"galera_stop_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] -+"galera_stop_0 galera-bundle-1" -> "galera-bundle-master_stopped_0" [ style = dashed] -+"galera_stop_0 galera-bundle-1" -> "galera_start_0 galera-bundle-1" [ style = dashed] -+"galera_stop_0 galera-bundle-1" -> "galera_stop_0 galera-bundle-0" [ style = dashed] -+"galera_stop_0 galera-bundle-1" [ style=dashed color="red" fontcolor="black"] -+"galera_stop_0 galera-bundle-2" -> "galera-bundle-master_stopped_0" [ style = dashed] -+"galera_stop_0 galera-bundle-2" -> "galera_start_0 galera-bundle-2" [ style = dashed] -+"galera_stop_0 galera-bundle-2" -> "galera_stop_0 galera-bundle-1" [ style = dashed] -+"galera_stop_0 galera-bundle-2" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq-bundle-0_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq-bundle-0_start_0 node1" -> "rabbitmq-bundle-0_monitor_60000 node1" [ style = dashed] -+"rabbitmq-bundle-0_start_0 node1" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] -+"rabbitmq-bundle-0_start_0 node1" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] -+"rabbitmq-bundle-0_start_0 node1" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq-bundle-0_stop_0 node1" -> "rabbitmq-bundle-0_start_0 node1" [ style = dashed] -+"rabbitmq-bundle-0_stop_0 node1" -> "rabbitmq-bundle-docker-0_stop_0 node1" [ style = bold] -+"rabbitmq-bundle-0_stop_0 node1" [ style=bold color="green" fontcolor="black"] -+"rabbitmq-bundle-1_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq-bundle-1_start_0 node1" -> "rabbitmq-bundle-1_monitor_60000 node1" [ style = dashed] -+"rabbitmq-bundle-1_start_0 node1" -> "rabbitmq_monitor_10000 rabbitmq-bundle-1" [ style = dashed] -+"rabbitmq-bundle-1_start_0 node1" -> "rabbitmq_start_0 rabbitmq-bundle-1" [ style = dashed] -+"rabbitmq-bundle-1_start_0 node1" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq-bundle-1_stop_0 node2" -> "rabbitmq-bundle-1_start_0 node1" [ style = dashed] -+"rabbitmq-bundle-1_stop_0 node2" -> "rabbitmq-bundle-docker-1_stop_0 node2" [ style = dashed] -+"rabbitmq-bundle-1_stop_0 node2" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq-bundle-2_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq-bundle-2_start_0 node1" -> "rabbitmq-bundle-2_monitor_60000 node1" [ style = dashed] -+"rabbitmq-bundle-2_start_0 node1" -> "rabbitmq_monitor_10000 rabbitmq-bundle-2" [ style = dashed] -+"rabbitmq-bundle-2_start_0 node1" -> "rabbitmq_start_0 rabbitmq-bundle-2" [ style = dashed] -+"rabbitmq-bundle-2_start_0 node1" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq-bundle-2_stop_0 node3" -> "rabbitmq-bundle-2_start_0 node1" [ style = dashed] -+"rabbitmq-bundle-2_stop_0 node3" -> "rabbitmq-bundle-docker-2_stop_0 node3" [ style = dashed] -+"rabbitmq-bundle-2_stop_0 node3" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = dashed] -+"rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=dashed color="red" fontcolor="orange"] -+"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle-clone_pre_notify_start_0" [ style = dashed] -+"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle_stopped_0" [ style = bold] -+"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = dashed] -+"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed] -+"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"] -+"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold] -+"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] -+"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] -+"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = dashed] -+"rabbitmq-bundle-clone_post_notify_running_0" [ style=dashed color="red" fontcolor="orange"] -+"rabbitmq-bundle-clone_post_notify_stopped_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style = bold] -+"rabbitmq-bundle-clone_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = dashed] -+"rabbitmq-bundle-clone_pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"] -+"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] -+"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" [ style = bold] -+"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" [ style = dashed] -+"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" [ style = dashed] -+"rabbitmq-bundle-clone_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] -+"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = dashed] -+"rabbitmq-bundle-clone_running_0" [ style=dashed color="red" fontcolor="orange"] -+"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] -+"rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] -+"rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-1" [ style = dashed] -+"rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-2" [ style = dashed] -+"rabbitmq-bundle-clone_start_0" [ style=dashed color="red" fontcolor="orange"] -+"rabbitmq-bundle-clone_stop_0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] -+"rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] -+"rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-1" [ style = dashed] -+"rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-2" [ style = dashed] -+"rabbitmq-bundle-clone_stop_0" [ style=bold color="green" fontcolor="orange"] -+"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold] -+"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed] -+"rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"rabbitmq-bundle-docker-0_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq-bundle-docker-0_start_0 node1" -> "rabbitmq-bundle-0_start_0 node1" [ style = dashed] -+"rabbitmq-bundle-docker-0_start_0 node1" -> "rabbitmq-bundle-docker-0_monitor_60000 node1" [ style = dashed] -+"rabbitmq-bundle-docker-0_start_0 node1" -> "rabbitmq-bundle_running_0" [ style = dashed] -+"rabbitmq-bundle-docker-0_start_0 node1" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] -+"rabbitmq-bundle-docker-0_start_0 node1" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq-bundle-docker-0_stop_0 node1" -> "rabbitmq-bundle-docker-0_start_0 node1" [ style = dashed] -+"rabbitmq-bundle-docker-0_stop_0 node1" -> "rabbitmq-bundle_stopped_0" [ style = bold] -+"rabbitmq-bundle-docker-0_stop_0 node1" [ style=bold color="green" fontcolor="black"] -+"rabbitmq-bundle-docker-1_stop_0 node2" -> "rabbitmq-bundle_stopped_0" [ style = dashed] -+"rabbitmq-bundle-docker-1_stop_0 node2" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq-bundle-docker-2_stop_0 node3" -> "rabbitmq-bundle_stopped_0" [ style = dashed] -+"rabbitmq-bundle-docker-2_stop_0 node3" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] -+"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed] -+"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-docker-0_start_0 node1" [ style = dashed] -+"rabbitmq-bundle_start_0" [ style=dashed color="red" fontcolor="orange"] -+"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] -+"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-docker-0_stop_0 node1" [ style = bold] -+"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-docker-1_stop_0 node2" [ style = dashed] -+"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-docker-2_stop_0 node3" [ style = dashed] -+"rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] -+"rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-1" [ style = dashed] -+"rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-2" [ style = dashed] -+"rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] -+"rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq_monitor_10000 rabbitmq-bundle-1" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq_monitor_10000 rabbitmq-bundle-2" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] -+"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] -+"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = dashed] -+"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = dashed] -+"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] -+"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] -+"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq_start_0 rabbitmq-bundle-1" [ style = dashed] -+"rabbitmq_start_0 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq_start_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] -+"rabbitmq_start_0 rabbitmq-bundle-1" -> "rabbitmq_monitor_10000 rabbitmq-bundle-1" [ style = dashed] -+"rabbitmq_start_0 rabbitmq-bundle-1" -> "rabbitmq_start_0 rabbitmq-bundle-2" [ style = dashed] -+"rabbitmq_start_0 rabbitmq-bundle-1" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq_start_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] -+"rabbitmq_start_0 rabbitmq-bundle-2" -> "rabbitmq_monitor_10000 rabbitmq-bundle-2" [ style = dashed] -+"rabbitmq_start_0 rabbitmq-bundle-2" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-0_stop_0 node1" [ style = bold] -+"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] -+"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] -+"rabbitmq_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] -+"rabbitmq_stop_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_stopped_0" [ style = dashed] -+"rabbitmq_stop_0 rabbitmq-bundle-1" -> "rabbitmq_start_0 rabbitmq-bundle-1" [ style = dashed] -+"rabbitmq_stop_0 rabbitmq-bundle-1" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = dashed] -+"rabbitmq_stop_0 rabbitmq-bundle-1" [ style=dashed color="red" fontcolor="black"] -+"rabbitmq_stop_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_stopped_0" [ style = dashed] -+"rabbitmq_stop_0 rabbitmq-bundle-2" -> "rabbitmq_start_0 rabbitmq-bundle-2" [ style = dashed] -+"rabbitmq_stop_0 rabbitmq-bundle-2" -> "rabbitmq_stop_0 rabbitmq-bundle-1" [ style = dashed] -+"rabbitmq_stop_0 rabbitmq-bundle-2" [ style=dashed color="red" fontcolor="black"] -+"stonith-fence_ipmilan-node1_stop_0 node2" [ style=dashed color="red" fontcolor="black"] -+"stonith-fence_ipmilan-node2_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] -+"stonith-fence_ipmilan-node2_start_0 node1" -> "stonith-fence_ipmilan-node2_monitor_60000 node1" [ style = dashed] -+"stonith-fence_ipmilan-node2_start_0 node1" [ style=dashed color="red" fontcolor="black"] -+"stonith-fence_ipmilan-node2_stop_0 node3" -> "stonith-fence_ipmilan-node2_start_0 node1" [ style = dashed] -+"stonith-fence_ipmilan-node2_stop_0 node3" [ style=dashed color="red" fontcolor="black"] -+"stonith-fence_ipmilan-node3_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] -+"stonith-fence_ipmilan-node3_start_0 node1" -> "stonith-fence_ipmilan-node3_monitor_60000 node1" [ style = dashed] -+"stonith-fence_ipmilan-node3_start_0 node1" [ style=dashed color="red" fontcolor="black"] -+"stonith-fence_ipmilan-node3_stop_0 node2" -> "stonith-fence_ipmilan-node3_start_0 node1" [ style = dashed] -+"stonith-fence_ipmilan-node3_stop_0 node2" [ style=dashed color="red" fontcolor="black"] -+} -diff --git a/pengine/test10/guest-host-not-fenceable.exp b/pengine/test10/guest-host-not-fenceable.exp -new file mode 100644 -index 0000000..ed28f2b ---- /dev/null -+++ b/pengine/test10/guest-host-not-fenceable.exp -@@ -0,0 +1,340 @@ -+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0"> -+ <synapse id="0"> -+ <action_set> -+ <rsc_op id="108" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1"> -+ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/> -+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_active_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_all_hosts="node1 node2 node3 node1 node2 node3 node1 node2 node3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 node1 node2 node3 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="node1 node2 node3 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_physical_host="node1" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/> -+ </rsc_op> -+ </action_set> -+ <inputs> -+ <trigger> -+ <pseudo_event id="47" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="1"> -+ <action_set> -+ <rsc_op id="31" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1"> -+ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/> -+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_active_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_all_hosts="node1 node2 node3 node1 node2 node3 node1 node2 node3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 node1 node2 node3 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="node1 node2 node3 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_physical_host="node1" CRM_meta_timeout="200000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/> -+ </rsc_op> -+ </action_set> -+ <inputs> -+ <trigger> -+ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle_stop_0"/> -+ </trigger> -+ <trigger> -+ <pseudo_event id="45" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="2" priority="1000000"> -+ <action_set> -+ <pseudo_event id="50" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-post_notify_stopped_0"> -+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs> -+ <trigger> -+ <pseudo_event id="49" operation="notify" operation_key="rabbitmq-bundle-clone_post_notify_stopped_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="3" priority="1000000"> -+ <action_set> -+ <pseudo_event id="49" operation="notify" operation_key="rabbitmq-bundle-clone_post_notify_stopped_0"> -+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs> -+ <trigger> -+ <pseudo_event id="46" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0"/> -+ </trigger> -+ <trigger> -+ <pseudo_event id="48" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-pre_notify_stop_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="4"> -+ <action_set> -+ <pseudo_event id="48" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-pre_notify_stop_0"> -+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs> -+ <trigger> -+ <pseudo_event id="47" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"/> -+ </trigger> -+ <trigger> -+ <rsc_op id="108" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="5"> -+ <action_set> -+ <pseudo_event id="47" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"> -+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs/> -+ </synapse> -+ <synapse id="6" priority="1000000"> -+ <action_set> -+ <pseudo_event id="46" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0"> -+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs> -+ <trigger> -+ <rsc_op id="31" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1"/> -+ </trigger> -+ <trigger> -+ <pseudo_event id="45" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="7"> -+ <action_set> -+ <pseudo_event id="45" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0"> -+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs> -+ <trigger> -+ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle_stop_0"/> -+ </trigger> -+ <trigger> -+ <pseudo_event id="48" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-pre_notify_stop_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="8"> -+ <action_set> -+ <rsc_op id="16" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="node1" on_node_uuid="1"> -+ <primitive id="rabbitmq-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/> -+ <attributes CRM_meta_on_node="node1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.122.139:8787/rhosp13/openstack-rabbitmq:pcmklatest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/rabbitmq-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3122 -v /var/lib/kolla/config_files/rabbitmq.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/puppet-generated/rabbitmq/:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/rabbitmq:/var/lib/rabbitmq:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /var/log/containers/rabbitmq:/var/log/rabbitmq:rw -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/rabbitmq-bundle-0:/var/log --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/> -+ </rsc_op> -+ </action_set> -+ <inputs> -+ <trigger> -+ <rsc_op id="17" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="node1" on_node_uuid="1"/> -+ </trigger> -+ <trigger> -+ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle_stop_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="9"> -+ <action_set> -+ <rsc_op id="17" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="node1" on_node_uuid="1"> -+ <primitive id="rabbitmq-bundle-0" class="ocf" provider="pacemaker" type="remote"/> -+ <attributes CRM_meta_container="rabbitmq-bundle-docker-0" CRM_meta_on_node="node1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="node1" port="3122"/> -+ <downed> -+ <node id="rabbitmq-bundle-0"/> -+ </downed> -+ </rsc_op> -+ </action_set> -+ <inputs> -+ <trigger> -+ <rsc_op id="31" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="10"> -+ <action_set> -+ <rsc_op id="66" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"> -+ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/> -+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_physical_host="node1" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="node1:node1.internalapi.localdomain;node2:node2.internalapi.localdomain;node3:node3.internalapi.localdomain" enable_creation="true" log="/var/log/mysql/mysqld.log" wsrep_cluster_address="gcomm://node1.internalapi.localdomain,node2.internalapi.localdomain,node3.internalapi.localdomain"/> -+ </rsc_op> -+ </action_set> -+ <inputs> -+ <trigger> -+ <pseudo_event id="82" operation="demote" operation_key="galera-bundle-master_demote_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="11"> -+ <action_set> -+ <rsc_op id="10" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"> -+ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/> -+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_physical_host="node1" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="node1:node1.internalapi.localdomain;node2:node2.internalapi.localdomain;node3:node3.internalapi.localdomain" enable_creation="true" log="/var/log/mysql/mysqld.log" wsrep_cluster_address="gcomm://node1.internalapi.localdomain,node2.internalapi.localdomain,node3.internalapi.localdomain"/> -+ </rsc_op> -+ </action_set> -+ <inputs> -+ <trigger> -+ <pseudo_event id="64" operation="stop" operation_key="galera-bundle_stop_0"/> -+ </trigger> -+ <trigger> -+ <rsc_op id="66" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"/> -+ </trigger> -+ <trigger> -+ <pseudo_event id="78" operation="stop" operation_key="galera-bundle-master_stop_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="12" priority="1000000"> -+ <action_set> -+ <pseudo_event id="83" operation="demoted" operation_key="galera-bundle-master_demoted_0"> -+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs> -+ <trigger> -+ <rsc_op id="66" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"/> -+ </trigger> -+ <trigger> -+ <pseudo_event id="82" operation="demote" operation_key="galera-bundle-master_demote_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="13"> -+ <action_set> -+ <pseudo_event id="82" operation="demote" operation_key="galera-bundle-master_demote_0"> -+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs> -+ <trigger> -+ <pseudo_event id="86" operation="demote" operation_key="galera-bundle_demote_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="14" priority="1000000"> -+ <action_set> -+ <pseudo_event id="79" operation="stopped" operation_key="galera-bundle-master_stopped_0"> -+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs> -+ <trigger> -+ <rsc_op id="10" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"/> -+ </trigger> -+ <trigger> -+ <pseudo_event id="78" operation="stop" operation_key="galera-bundle-master_stop_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="15"> -+ <action_set> -+ <pseudo_event id="78" operation="stop" operation_key="galera-bundle-master_stop_0"> -+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs> -+ <trigger> -+ <pseudo_event id="64" operation="stop" operation_key="galera-bundle_stop_0"/> -+ </trigger> -+ <trigger> -+ <pseudo_event id="83" operation="demoted" operation_key="galera-bundle-master_demoted_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="16"> -+ <action_set> -+ <rsc_op id="51" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="node1" on_node_uuid="1"> -+ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/> -+ <attributes CRM_meta_on_node="node1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.122.139:8787/rhosp13/openstack-mariadb:pcmklatest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/puppet-generated/mysql/:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /var/log/containers/mysql:/var/log/mysql:rw -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/> -+ </rsc_op> -+ </action_set> -+ <inputs> -+ <trigger> -+ <rsc_op id="52" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="node1" on_node_uuid="1"/> -+ </trigger> -+ <trigger> -+ <pseudo_event id="64" operation="stop" operation_key="galera-bundle_stop_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="17"> -+ <action_set> -+ <rsc_op id="52" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="node1" on_node_uuid="1"> -+ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/> -+ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_on_node="node1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="node1" port="3123"/> -+ <downed> -+ <node id="galera-bundle-0"/> -+ </downed> -+ </rsc_op> -+ </action_set> -+ <inputs> -+ <trigger> -+ <rsc_op id="10" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"/> -+ </trigger> -+ <trigger> -+ <rsc_op id="66" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="18" priority="1000000"> -+ <action_set> -+ <pseudo_event id="87" operation="demoted" operation_key="galera-bundle_demoted_0"> -+ <attributes CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs> -+ <trigger> -+ <pseudo_event id="83" operation="demoted" operation_key="galera-bundle-master_demoted_0"/> -+ </trigger> -+ <trigger> -+ <pseudo_event id="86" operation="demote" operation_key="galera-bundle_demote_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="19"> -+ <action_set> -+ <pseudo_event id="86" operation="demote" operation_key="galera-bundle_demote_0"> -+ <attributes CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs/> -+ </synapse> -+ <synapse id="20" priority="1000000"> -+ <action_set> -+ <pseudo_event id="65" operation="stopped" operation_key="galera-bundle_stopped_0"> -+ <attributes CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs> -+ <trigger> -+ <rsc_op id="51" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="node1" on_node_uuid="1"/> -+ </trigger> -+ <trigger> -+ <pseudo_event id="79" operation="stopped" operation_key="galera-bundle-master_stopped_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="21"> -+ <action_set> -+ <pseudo_event id="64" operation="stop" operation_key="galera-bundle_stop_0"> -+ <attributes CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs> -+ <trigger> -+ <pseudo_event id="87" operation="demoted" operation_key="galera-bundle_demoted_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="22" priority="1000000"> -+ <action_set> -+ <pseudo_event id="30" operation="stopped" operation_key="rabbitmq-bundle_stopped_0"> -+ <attributes CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs> -+ <trigger> -+ <rsc_op id="16" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="node1" on_node_uuid="1"/> -+ </trigger> -+ <trigger> -+ <pseudo_event id="50" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-post_notify_stopped_0"/> -+ </trigger> -+ </inputs> -+ </synapse> -+ <synapse id="23"> -+ <action_set> -+ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle_stop_0"> -+ <attributes CRM_meta_timeout="20000" /> -+ </pseudo_event> -+ </action_set> -+ <inputs/> -+ </synapse> -+</transition_graph> -diff --git a/pengine/test10/guest-host-not-fenceable.scores b/pengine/test10/guest-host-not-fenceable.scores -new file mode 100644 -index 0000000..80bd0d4 ---- /dev/null -+++ b/pengine/test10/guest-host-not-fenceable.scores -@@ -0,0 +1,134 @@ -+Allocation scores: -+Using the original execution date of: 2019-08-26 04:52:42Z -+clone_color: galera-bundle-master allocation score on galera-bundle-0: 0 -+clone_color: galera-bundle-master allocation score on galera-bundle-1: 0 -+clone_color: galera-bundle-master allocation score on galera-bundle-2: 0 -+clone_color: galera-bundle-master allocation score on node1: -INFINITY -+clone_color: galera-bundle-master allocation score on node2: -INFINITY -+clone_color: galera-bundle-master allocation score on node3: -INFINITY -+clone_color: galera:0 allocation score on galera-bundle-0: INFINITY -+clone_color: galera:1 allocation score on galera-bundle-1: INFINITY -+clone_color: galera:2 allocation score on galera-bundle-2: INFINITY -+clone_color: rabbitmq-bundle-clone allocation score on node1: -INFINITY -+clone_color: rabbitmq-bundle-clone allocation score on node2: -INFINITY -+clone_color: rabbitmq-bundle-clone allocation score on node3: -INFINITY -+clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 -+clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: 0 -+clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: 0 -+clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY -+clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY -+clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY -+container_color: galera-bundle allocation score on node1: 0 -+container_color: galera-bundle allocation score on node2: 0 -+container_color: galera-bundle allocation score on node3: 0 -+container_color: galera-bundle-0 allocation score on node1: INFINITY -+container_color: galera-bundle-0 allocation score on node2: 0 -+container_color: galera-bundle-0 allocation score on node3: 0 -+container_color: galera-bundle-1 allocation score on node1: 0 -+container_color: galera-bundle-1 allocation score on node2: INFINITY -+container_color: galera-bundle-1 allocation score on node3: 0 -+container_color: galera-bundle-2 allocation score on node1: 0 -+container_color: galera-bundle-2 allocation score on node2: 0 -+container_color: galera-bundle-2 allocation score on node3: INFINITY -+container_color: galera-bundle-docker-0 allocation score on node1: INFINITY -+container_color: galera-bundle-docker-0 allocation score on node2: 0 -+container_color: galera-bundle-docker-0 allocation score on node3: 0 -+container_color: galera-bundle-docker-1 allocation score on node1: 0 -+container_color: galera-bundle-docker-1 allocation score on node2: INFINITY -+container_color: galera-bundle-docker-1 allocation score on node3: 0 -+container_color: galera-bundle-docker-2 allocation score on node1: 0 -+container_color: galera-bundle-docker-2 allocation score on node2: 0 -+container_color: galera-bundle-docker-2 allocation score on node3: INFINITY -+container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY -+container_color: galera-bundle-master allocation score on galera-bundle-1: -INFINITY -+container_color: galera-bundle-master allocation score on galera-bundle-2: -INFINITY -+container_color: galera-bundle-master allocation score on node1: 0 -+container_color: galera-bundle-master allocation score on node2: 0 -+container_color: galera-bundle-master allocation score on node3: 0 -+container_color: galera:0 allocation score on galera-bundle-0: INFINITY -+container_color: galera:1 allocation score on galera-bundle-1: INFINITY -+container_color: galera:2 allocation score on galera-bundle-2: INFINITY -+container_color: rabbitmq-bundle allocation score on node1: 0 -+container_color: rabbitmq-bundle allocation score on node2: 0 -+container_color: rabbitmq-bundle allocation score on node3: 0 -+container_color: rabbitmq-bundle-0 allocation score on node1: INFINITY -+container_color: rabbitmq-bundle-0 allocation score on node2: 0 -+container_color: rabbitmq-bundle-0 allocation score on node3: 0 -+container_color: rabbitmq-bundle-1 allocation score on node1: 0 -+container_color: rabbitmq-bundle-1 allocation score on node2: INFINITY -+container_color: rabbitmq-bundle-1 allocation score on node3: 0 -+container_color: rabbitmq-bundle-2 allocation score on node1: 0 -+container_color: rabbitmq-bundle-2 allocation score on node2: 0 -+container_color: rabbitmq-bundle-2 allocation score on node3: INFINITY -+container_color: rabbitmq-bundle-clone allocation score on node1: 0 -+container_color: rabbitmq-bundle-clone allocation score on node2: 0 -+container_color: rabbitmq-bundle-clone allocation score on node3: 0 -+container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: -INFINITY -+container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: -INFINITY -+container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: -INFINITY -+container_color: rabbitmq-bundle-docker-0 allocation score on node1: INFINITY -+container_color: rabbitmq-bundle-docker-0 allocation score on node2: 0 -+container_color: rabbitmq-bundle-docker-0 allocation score on node3: 0 -+container_color: rabbitmq-bundle-docker-1 allocation score on node1: 0 -+container_color: rabbitmq-bundle-docker-1 allocation score on node2: INFINITY -+container_color: rabbitmq-bundle-docker-1 allocation score on node3: 0 -+container_color: rabbitmq-bundle-docker-2 allocation score on node1: 0 -+container_color: rabbitmq-bundle-docker-2 allocation score on node2: 0 -+container_color: rabbitmq-bundle-docker-2 allocation score on node3: INFINITY -+container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY -+container_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY -+container_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY -+galera:0 promotion score on galera-bundle-0: 100 -+galera:1 promotion score on galera-bundle-1: 100 -+galera:2 promotion score on galera-bundle-2: -1 -+native_color: galera-bundle-0 allocation score on node1: INFINITY -+native_color: galera-bundle-0 allocation score on node2: 0 -+native_color: galera-bundle-0 allocation score on node3: 0 -+native_color: galera-bundle-1 allocation score on node1: 0 -+native_color: galera-bundle-1 allocation score on node2: INFINITY -+native_color: galera-bundle-1 allocation score on node3: 0 -+native_color: galera-bundle-2 allocation score on node1: 0 -+native_color: galera-bundle-2 allocation score on node2: 0 -+native_color: galera-bundle-2 allocation score on node3: INFINITY -+native_color: galera-bundle-docker-0 allocation score on node1: INFINITY -+native_color: galera-bundle-docker-0 allocation score on node2: -INFINITY -+native_color: galera-bundle-docker-0 allocation score on node3: -INFINITY -+native_color: galera-bundle-docker-1 allocation score on node1: -INFINITY -+native_color: galera-bundle-docker-1 allocation score on node2: -INFINITY -+native_color: galera-bundle-docker-1 allocation score on node3: -INFINITY -+native_color: galera-bundle-docker-2 allocation score on node1: -INFINITY -+native_color: galera-bundle-docker-2 allocation score on node2: -INFINITY -+native_color: galera-bundle-docker-2 allocation score on node3: -INFINITY -+native_color: galera:0 allocation score on galera-bundle-0: INFINITY -+native_color: galera:1 allocation score on galera-bundle-1: INFINITY -+native_color: galera:2 allocation score on galera-bundle-2: INFINITY -+native_color: rabbitmq-bundle-0 allocation score on node1: INFINITY -+native_color: rabbitmq-bundle-0 allocation score on node2: 0 -+native_color: rabbitmq-bundle-0 allocation score on node3: 0 -+native_color: rabbitmq-bundle-1 allocation score on node1: 0 -+native_color: rabbitmq-bundle-1 allocation score on node2: INFINITY -+native_color: rabbitmq-bundle-1 allocation score on node3: 0 -+native_color: rabbitmq-bundle-2 allocation score on node1: 0 -+native_color: rabbitmq-bundle-2 allocation score on node2: 0 -+native_color: rabbitmq-bundle-2 allocation score on node3: INFINITY -+native_color: rabbitmq-bundle-docker-0 allocation score on node1: INFINITY -+native_color: rabbitmq-bundle-docker-0 allocation score on node2: -INFINITY -+native_color: rabbitmq-bundle-docker-0 allocation score on node3: -INFINITY -+native_color: rabbitmq-bundle-docker-1 allocation score on node1: -INFINITY -+native_color: rabbitmq-bundle-docker-1 allocation score on node2: -INFINITY -+native_color: rabbitmq-bundle-docker-1 allocation score on node3: -INFINITY -+native_color: rabbitmq-bundle-docker-2 allocation score on node1: -INFINITY -+native_color: rabbitmq-bundle-docker-2 allocation score on node2: -INFINITY -+native_color: rabbitmq-bundle-docker-2 allocation score on node3: -INFINITY -+native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY -+native_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY -+native_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY -+native_color: stonith-fence_ipmilan-node1 allocation score on node1: -INFINITY -+native_color: stonith-fence_ipmilan-node1 allocation score on node2: INFINITY -+native_color: stonith-fence_ipmilan-node1 allocation score on node3: 0 -+native_color: stonith-fence_ipmilan-node2 allocation score on node1: 0 -+native_color: stonith-fence_ipmilan-node2 allocation score on node2: -INFINITY -+native_color: stonith-fence_ipmilan-node2 allocation score on node3: INFINITY -+native_color: stonith-fence_ipmilan-node3 allocation score on node1: 0 -+native_color: stonith-fence_ipmilan-node3 allocation score on node2: INFINITY -+native_color: stonith-fence_ipmilan-node3 allocation score on node3: -INFINITY -diff --git a/pengine/test10/guest-host-not-fenceable.summary b/pengine/test10/guest-host-not-fenceable.summary -new file mode 100644 -index 0000000..54a4d7b ---- /dev/null -+++ b/pengine/test10/guest-host-not-fenceable.summary -@@ -0,0 +1,87 @@ -+Using the original execution date of: 2019-08-26 04:52:42Z -+ -+Current cluster status: -+Node node2 (2): UNCLEAN (offline) -+Node node3 (3): UNCLEAN (offline) -+Online: [ node1 ] -+Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 ] -+ -+ Docker container set: rabbitmq-bundle [192.168.122.139:8787/rhosp13/openstack-rabbitmq:pcmklatest] -+ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started node1 -+ rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): FAILED node2 (UNCLEAN) -+ rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): FAILED node3 (UNCLEAN) -+ Docker container set: galera-bundle [192.168.122.139:8787/rhosp13/openstack-mariadb:pcmklatest] -+ galera-bundle-0 (ocf::heartbeat:galera): FAILED Master node1 -+ galera-bundle-1 (ocf::heartbeat:galera): FAILED Master node2 (UNCLEAN) -+ galera-bundle-2 (ocf::heartbeat:galera): FAILED Master node3 (UNCLEAN) -+ stonith-fence_ipmilan-node1 (stonith:fence_ipmilan): Started node2 (UNCLEAN) -+ stonith-fence_ipmilan-node3 (stonith:fence_ipmilan): Started node2 (UNCLEAN) -+ stonith-fence_ipmilan-node2 (stonith:fence_ipmilan): Started node3 (UNCLEAN) -+ -+Transition Summary: -+ * Stop rabbitmq-bundle-docker-0 ( node1 ) due to no quorum -+ * Stop rabbitmq-bundle-0 ( node1 ) due to no quorum -+ * Stop rabbitmq:0 ( rabbitmq-bundle-0 ) due to no quorum -+ * Stop rabbitmq-bundle-docker-1 ( node2 ) due to node availability (blocked) -+ * Stop rabbitmq-bundle-1 ( node2 ) due to no quorum (blocked) -+ * Stop rabbitmq:1 ( rabbitmq-bundle-1 ) due to no quorum (blocked) -+ * Stop rabbitmq-bundle-docker-2 ( node3 ) due to node availability (blocked) -+ * Stop rabbitmq-bundle-2 ( node3 ) due to no quorum (blocked) -+ * Stop rabbitmq:2 ( rabbitmq-bundle-2 ) due to no quorum (blocked) -+ * Stop galera-bundle-docker-0 ( node1 ) due to no quorum -+ * Stop galera-bundle-0 ( node1 ) due to no quorum -+ * Stop galera:0 ( Master galera-bundle-0 ) due to no quorum -+ * Stop galera-bundle-docker-1 ( node2 ) due to node availability (blocked) -+ * Stop galera-bundle-1 ( node2 ) due to no quorum (blocked) -+ * Stop galera:1 ( Master galera-bundle-1 ) due to no quorum (blocked) -+ * Stop galera-bundle-docker-2 ( node3 ) due to node availability (blocked) -+ * Stop galera-bundle-2 ( node3 ) due to no quorum (blocked) -+ * Stop galera:2 ( Master galera-bundle-2 ) due to no quorum (blocked) -+ * Stop stonith-fence_ipmilan-node1 ( node2 ) due to node availability (blocked) -+ * Stop stonith-fence_ipmilan-node3 ( node2 ) due to no quorum (blocked) -+ * Stop stonith-fence_ipmilan-node2 ( node3 ) due to no quorum (blocked) -+ -+Executing cluster transition: -+ * Pseudo action: rabbitmq-bundle-clone_pre_notify_stop_0 -+ * Pseudo action: galera-bundle_demote_0 -+ * Pseudo action: rabbitmq-bundle_stop_0 -+ * Resource action: rabbitmq notify on rabbitmq-bundle-0 -+ * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_stop_0 -+ * Pseudo action: rabbitmq-bundle-clone_stop_0 -+ * Pseudo action: galera-bundle-master_demote_0 -+ * Resource action: rabbitmq stop on rabbitmq-bundle-0 -+ * Pseudo action: rabbitmq-bundle-clone_stopped_0 -+ * Resource action: rabbitmq-bundle-0 stop on node1 -+ * Resource action: galera demote on galera-bundle-0 -+ * Pseudo action: galera-bundle-master_demoted_0 -+ * Pseudo action: galera-bundle_demoted_0 -+ * Pseudo action: galera-bundle_stop_0 -+ * Pseudo action: rabbitmq-bundle-clone_post_notify_stopped_0 -+ * Resource action: rabbitmq-bundle-docker-0 stop on node1 -+ * Pseudo action: galera-bundle-master_stop_0 -+ * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_stopped_0 -+ * Resource action: galera stop on galera-bundle-0 -+ * Pseudo action: galera-bundle-master_stopped_0 -+ * Resource action: galera-bundle-0 stop on node1 -+ * Pseudo action: rabbitmq-bundle_stopped_0 -+ * Resource action: galera-bundle-docker-0 stop on node1 -+ * Pseudo action: galera-bundle_stopped_0 -+Using the original execution date of: 2019-08-26 04:52:42Z -+ -+Revised cluster status: -+Node node2 (2): UNCLEAN (offline) -+Node node3 (3): UNCLEAN (offline) -+Online: [ node1 ] -+ -+ Docker container set: rabbitmq-bundle [192.168.122.139:8787/rhosp13/openstack-rabbitmq:pcmklatest] -+ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped -+ rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): FAILED node2 (UNCLEAN) -+ rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): FAILED node3 (UNCLEAN) -+ Docker container set: galera-bundle [192.168.122.139:8787/rhosp13/openstack-mariadb:pcmklatest] -+ galera-bundle-0 (ocf::heartbeat:galera): Stopped -+ galera-bundle-1 (ocf::heartbeat:galera): FAILED Master node2 (UNCLEAN) -+ galera-bundle-2 (ocf::heartbeat:galera): FAILED Master node3 (UNCLEAN) -+ stonith-fence_ipmilan-node1 (stonith:fence_ipmilan): Started node2 (UNCLEAN) -+ stonith-fence_ipmilan-node3 (stonith:fence_ipmilan): Started node2 (UNCLEAN) -+ stonith-fence_ipmilan-node2 (stonith:fence_ipmilan): Started node3 (UNCLEAN) -+ -diff --git a/pengine/test10/guest-host-not-fenceable.xml b/pengine/test10/guest-host-not-fenceable.xml -new file mode 100755 -index 0000000..a1ccdc8 ---- /dev/null -+++ b/pengine/test10/guest-host-not-fenceable.xml -@@ -0,0 +1,413 @@ -+<cib crm_feature_set="3.0.14" validate-with="pacemaker-2.10" epoch="71" num_updates="166" admin_epoch="0" cib-last-written="Wed Aug 21 12:00:45 2019" update-origin="node1" update-client="crm_resource" update-user="root" have-quorum="0" dc-uuid="1" no-quorum-panic="1" execution-date="1566795162"> -+ <configuration> -+ <crm_config> -+ <cluster_property_set id="cib-bootstrap-options"> -+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/> -+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.19-8.el7_6.4-c3c624ea3d"/> -+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/> -+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/> -+ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/> -+ <nvpair id="cib-bootstrap-options-cluster-recheck-interval" name="cluster-recheck-interval" value="600s"/> -+ </cluster_property_set> -+ </crm_config> -+ <nodes> -+ <node id="1" uname="node1"> -+ <instance_attributes id="nodes-1"> -+ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/> -+ <nvpair id="nodes-1-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@node1"/> -+ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/> -+ </instance_attributes> -+ </node> -+ <node id="2" uname="node2"> -+ <instance_attributes id="nodes-2"> -+ <nvpair id="nodes-2-rabbitmq-role" name="rabbitmq-role" value="true"/> -+ <nvpair id="nodes-2-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@node2"/> -+ <nvpair id="nodes-2-galera-role" name="galera-role" value="true"/> -+ </instance_attributes> -+ </node> -+ <node id="3" uname="node3"> -+ <instance_attributes id="nodes-3"> -+ <nvpair id="nodes-3-rabbitmq-role" name="rabbitmq-role" value="true"/> -+ <nvpair id="nodes-3-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@node3"/> -+ <nvpair id="nodes-3-galera-role" name="galera-role" value="true"/> -+ </instance_attributes> -+ </node> -+ </nodes> -+ <resources> -+ <bundle id="rabbitmq-bundle"> -+ <docker image="192.168.122.139:8787/rhosp13/openstack-rabbitmq:pcmklatest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="3" run-command="/bin/bash /usr/local/bin/kolla_start"/> -+ <network control-port="3122"/> -+ <storage> -+ <storage-mapping id="rabbitmq-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/rabbitmq.json" target-dir="/var/lib/kolla/config_files/config.json"/> -+ <storage-mapping id="rabbitmq-cfg-data" options="ro" source-dir="/var/lib/config-data/puppet-generated/rabbitmq/" target-dir="/var/lib/kolla/config_files/src"/> -+ <storage-mapping id="rabbitmq-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/> -+ <storage-mapping id="rabbitmq-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/> -+ <storage-mapping id="rabbitmq-lib" options="rw" source-dir="/var/lib/rabbitmq" target-dir="/var/lib/rabbitmq"/> -+ <storage-mapping id="rabbitmq-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/> -+ <storage-mapping id="rabbitmq-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/> -+ <storage-mapping id="rabbitmq-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/> -+ <storage-mapping id="rabbitmq-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/> -+ <storage-mapping id="rabbitmq-log" options="rw" source-dir="/var/log/containers/rabbitmq" target-dir="/var/log/rabbitmq"/> -+ <storage-mapping id="rabbitmq-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/> -+ </storage> -+ <primitive class="ocf" id="rabbitmq" provider="heartbeat" type="rabbitmq-cluster"> -+ <instance_attributes id="rabbitmq-instance_attributes"> -+ <nvpair id="rabbitmq-instance_attributes-set_policy" name="set_policy" value="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/> -+ </instance_attributes> -+ <meta_attributes id="rabbitmq-meta_attributes"> -+ <nvpair id="rabbitmq-meta_attributes-container-attribute-target" name="container-attribute-target" value="host"/> -+ <nvpair id="rabbitmq-meta_attributes-notify" name="notify" value="true"/> -+ </meta_attributes> -+ <operations> -+ <op id="rabbitmq-monitor-interval-10s" interval="10s" name="monitor" timeout="40s"/> -+ <op id="rabbitmq-start-interval-0s" interval="0s" name="start" timeout="200s"/> -+ <op id="rabbitmq-stop-interval-0s" interval="0s" name="stop" timeout="200s"/> -+ </operations> -+ </primitive> -+ <meta_attributes id="rabbitmq-bundle-meta_attributes"/> -+ </bundle> -+ <bundle id="galera-bundle"> -+ <docker image="192.168.122.139:8787/rhosp13/openstack-mariadb:pcmklatest" masters="3" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="3" run-command="/bin/bash /usr/local/bin/kolla_start"/> -+ <network control-port="3123"/> -+ <storage> -+ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/> -+ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/puppet-generated/mysql/" target-dir="/var/lib/kolla/config_files/src"/> -+ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/> -+ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/> -+ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/> -+ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/> -+ <storage-mapping id="mysql-log" options="rw" source-dir="/var/log/containers/mysql" target-dir="/var/log/mysql"/> -+ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/> -+ </storage> -+ <primitive class="ocf" id="galera" provider="heartbeat" type="galera"> -+ <instance_attributes id="galera-instance_attributes"> -+ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/> -+ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="node1:node1.internalapi.localdomain;node2:node2.internalapi.localdomain;node3:node3.internalapi.localdomain"/> -+ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/> -+ <nvpair id="galera-instance_attributes-log" name="log" value="/var/log/mysql/mysqld.log"/> -+ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://node1.internalapi.localdomain,node2.internalapi.localdomain,node3.internalapi.localdomain"/> -+ </instance_attributes> -+ <meta_attributes id="galera-meta_attributes"> -+ <nvpair id="galera-meta_attributes-container-attribute-target" name="container-attribute-target" value="host"/> -+ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="3"/> -+ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/> -+ </meta_attributes> -+ <operations> -+ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120s"/> -+ <op id="galera-monitor-interval-20s" interval="20s" name="monitor" timeout="30s"/> -+ <op id="galera-monitor-interval-10s" interval="10s" name="monitor" role="Master" timeout="30s"/> -+ <op id="galera-monitor-interval-30s" interval="30s" name="monitor" role="Slave" timeout="30s"/> -+ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/> -+ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120s"/> -+ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120s"/> -+ </operations> -+ </primitive> -+ </bundle> -+ <primitive class="stonith" id="stonith-fence_ipmilan-node1" type="fence_ipmilan"> -+ <instance_attributes id="stonith-fence_ipmilan-node1-instance_attributes"> -+ <nvpair id="stonith-fence_ipmilan-node1-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="node1"/> -+ </instance_attributes> -+ <meta_attributes id="stonith-fence_ipmilan-node1-meta_attributes"> -+ <nvpair id="stonith-fence_ipmilan-node1-meta_attributes-failure-timeout" name="failure-timeout" value="600s"/> -+ <nvpair id="stonith-fence_ipmilan-node1-meta_attributes-migration-threshold" name="migration-threshold" value="5"/> -+ </meta_attributes> -+ <operations> -+ <op id="stonith-fence_ipmilan-node1-monitor-interval-60s" interval="60s" name="monitor"/> -+ </operations> -+ </primitive> -+ <primitive class="stonith" id="stonith-fence_ipmilan-node3" type="fence_ipmilan"> -+ <instance_attributes id="stonith-fence_ipmilan-node3-instance_attributes"> -+ <nvpair id="stonith-fence_ipmilan-node3-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="node3"/> -+ </instance_attributes> -+ <meta_attributes id="stonith-fence_ipmilan-node3-meta_attributes"> -+ <nvpair id="stonith-fence_ipmilan-node3-meta_attributes-failure-timeout" name="failure-timeout" value="600s"/> -+ <nvpair id="stonith-fence_ipmilan-node3-meta_attributes-migration-threshold" name="migration-threshold" value="5"/> -+ </meta_attributes> -+ <operations> -+ <op id="stonith-fence_ipmilan-node3-monitor-interval-60s" interval="60s" name="monitor"/> -+ </operations> -+ </primitive> -+ <primitive class="stonith" id="stonith-fence_ipmilan-node2" type="fence_ipmilan"> -+ <instance_attributes id="stonith-fence_ipmilan-node2-instance_attributes"> -+ <nvpair id="stonith-fence_ipmilan-node2-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="node2"/> -+ </instance_attributes> -+ <meta_attributes id="stonith-fence_ipmilan-node2-meta_attributes"> -+ <nvpair id="stonith-fence_ipmilan-node2-meta_attributes-failure-timeout" name="failure-timeout" value="600s"/> -+ <nvpair id="stonith-fence_ipmilan-node2-meta_attributes-migration-threshold" name="migration-threshold" value="5"/> -+ </meta_attributes> -+ <operations> -+ <op id="stonith-fence_ipmilan-node2-monitor-interval-60s" interval="60s" name="monitor"/> -+ </operations> -+ </primitive> -+ </resources> -+ <constraints> -+ <rsc_location id="location-rabbitmq-bundle" resource-discovery="exclusive" rsc="rabbitmq-bundle"> -+ <rule id="location-rabbitmq-bundle-rule" score="0"> -+ <expression attribute="rabbitmq-role" id="location-rabbitmq-bundle-rule-expr" operation="eq" value="true"/> -+ </rule> -+ </rsc_location> -+ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle"> -+ <rule id="location-galera-bundle-rule" score="0"> -+ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/> -+ </rule> -+ </rsc_location> -+ <rsc_location id="location-stonith-fence_ipmilan-node1-node1--INFINITY" node="node1" rsc="stonith-fence_ipmilan-node1" score="-INFINITY"/> -+ <rsc_location id="location-stonith-fence_ipmilan-node3-node3--INFINITY" node="node3" rsc="stonith-fence_ipmilan-node3" score="-INFINITY"/> -+ <rsc_location id="location-stonith-fence_ipmilan-node2-node2--INFINITY" node="node2" rsc="stonith-fence_ipmilan-node2" score="-INFINITY"/> -+ </constraints> -+ <rsc_defaults> -+ <meta_attributes id="rsc_defaults-options"> -+ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/> -+ </meta_attributes> -+ </rsc_defaults> -+ </configuration> -+ <status> -+ <node_state id="1" uname="node1" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member"> -+ <transient_attributes id="1"> -+ <instance_attributes id="status-1"> -+ <nvpair id="status-1-master-galera" name="master-galera" value="100"/> -+ <nvpair id="status-1-rmq-node-attr-rabbitmq" name="rmq-node-attr-rabbitmq" value="rabbit@node1"/> -+ </instance_attributes> -+ </transient_attributes> -+ <lrm id="1"> -+ <lrm_resources> -+ <lrm_resource id="rabbitmq-bundle-1" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-1"> -+ <lrm_rsc_op id="rabbitmq-bundle-1_last_0" operation_key="rabbitmq-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="26:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;26:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="0" queue-time="0" op-digest="4e9836c1fe6ca784363329f38f1a6bab" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-2" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-2"> -+ <lrm_rsc_op id="rabbitmq-bundle-2_last_0" operation_key="rabbitmq-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="28:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;28:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="0" queue-time="0" op-digest="6c8e8fc40a3a8bc0990cb4086a91cb5a" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ </lrm_resource> -+ <lrm_resource id="stonith-fence_ipmilan-node1" type="fence_ipmilan" class="stonith"> -+ <lrm_rsc_op id="stonith-fence_ipmilan-node1_last_0" operation_key="stonith-fence_ipmilan-node1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="42:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;42:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1566435865" last-rc-change="1566435865" exec-time="139" queue-time="0" op-digest="40ba273e494269d4d7bded2368059e8d" op-secure-params=" password passwd " op-secure-digest="3ffa7c4b1189e29b87161a1808ba7d9f"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="46:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;46:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="75" rc-code="0" op-status="0" interval="0" last-run="1566435865" last-rc-change="1566435865" exec-time="2053" queue-time="0" op-digest="14ba0643d7f91ddb1789cd6c29f9ac3f"/> -+ <lrm_rsc_op id="rabbitmq-bundle-docker-0_monitor_60000" operation_key="rabbitmq-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="47:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;47:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="80" rc-code="0" op-status="0" interval="60000" last-rc-change="1566435867" exec-time="161" queue-time="0" op-digest="f18bc65b8186d36c889d9d67beeef1b3"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-docker-1" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="rabbitmq-bundle-docker-1_last_0" operation_key="rabbitmq-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="25:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;25:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="9" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="196" queue-time="0" op-digest="6611df38a8c054188fb3906bac2a4ac6"/> -+ </lrm_resource> -+ <lrm_resource id="stonith-fence_ipmilan-node2" type="fence_ipmilan" class="stonith"> -+ <lrm_rsc_op id="stonith-fence_ipmilan-node2_last_0" operation_key="stonith-fence_ipmilan-node2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="44:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;44:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="69" rc-code="7" op-status="0" interval="0" last-run="1566435865" last-rc-change="1566435865" exec-time="0" queue-time="0" op-digest="7252381349b2722d90a4ce9fba7ba3b5" op-secure-params=" password passwd " op-secure-digest="5265a046a980e16fb93d35f07f19a6d4"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-docker-0" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="80:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;80:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="74" rc-code="0" op-status="0" interval="0" last-run="1566435865" last-rc-change="1566435865" exec-time="1784" queue-time="0" op-digest="c9e60e09c1b2adcdac0169e69cbeb76e"/> -+ <lrm_rsc_op id="galera-bundle-docker-0_monitor_60000" operation_key="galera-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="81:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;81:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="78" rc-code="0" op-status="0" interval="60000" last-rc-change="1566435867" exec-time="235" queue-time="0" op-digest="5826a0cfeef03a64959f74a25e81bbc4"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-1" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-1"> -+ <lrm_rsc_op id="galera-bundle-1_last_0" operation_key="galera-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="32:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;32:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="0" queue-time="0" op-digest="5902e2bf7cfd38a4e959ce335356dc1f" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-2" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-2"> -+ <lrm_rsc_op id="galera-bundle-2_last_0" operation_key="galera-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="34:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;34:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="0" queue-time="0" op-digest="f724287411b756a9cac4dba67d3bddf6" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-docker-2" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="rabbitmq-bundle-docker-2_last_0" operation_key="rabbitmq-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="27:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;27:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="14" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="166" queue-time="0" op-digest="7ade29d4352e8007c537badef8a2e0b0"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-docker-1" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="galera-bundle-docker-1_last_0" operation_key="galera-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="31:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;31:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="23" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="155" queue-time="0" op-digest="c6f303ef421236554d0dea3151b97cb3"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-docker-2" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="galera-bundle-docker-2_last_0" operation_key="galera-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="33:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;33:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="28" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="174" queue-time="0" op-digest="bb5745d5e3a6dedaf2449265ce05625b"/> -+ </lrm_resource> -+ <lrm_resource id="stonith-fence_ipmilan-node3" type="fence_ipmilan" class="stonith"> -+ <lrm_rsc_op id="stonith-fence_ipmilan-node3_last_0" operation_key="stonith-fence_ipmilan-node3_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="43:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;43:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="65" rc-code="7" op-status="0" interval="0" last-run="1566435865" last-rc-change="1566435865" exec-time="0" queue-time="0" op-digest="094b99e257ba32c2b718b61813e80256" op-secure-params=" password passwd " op-secure-digest="765c80b4e1ed30f2ef368a8296f7481c"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-0" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-0"> -+ <lrm_rsc_op id="galera-bundle-0_last_0" operation_key="galera-bundle-0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="82:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;82:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1566435867" last-rc-change="1566435867" exec-time="0" queue-time="0" op-digest="8193f24828986601946578a7993cbb74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ <lrm_rsc_op id="galera-bundle-0_monitor_60000" operation_key="galera-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="67:890:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;67:890:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="10" rc-code="0" op-status="0" interval="60000" last-rc-change="1566435871" exec-time="0" queue-time="0" op-digest="aa427f07df30a17686d41e30145bf557"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0"> -+ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="48:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;48:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="8" rc-code="0" op-status="0" interval="0" last-run="1566435867" last-rc-change="1566435867" exec-time="0" queue-time="0" op-digest="2c6afa7ac6b48957892b7238a3270a5d" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ <lrm_rsc_op id="rabbitmq-bundle-0_monitor_60000" operation_key="rabbitmq-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="32:890:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;32:890:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="9" rc-code="0" op-status="0" interval="60000" last-rc-change="1566435871" exec-time="0" queue-time="0" op-digest="deffe42b0e1f26d58665d8c3a846a8eb"/> -+ </lrm_resource> -+ </lrm_resources> -+ </lrm> -+ </node_state> -+ <node_state id="2" uname="node2" in_ccm="false" crmd="offline" crm-debug-origin="do_state_transition" join="down" expected="down"> -+ <transient_attributes id="2"> -+ <instance_attributes id="status-2"> -+ <nvpair id="status-2-master-galera" name="master-galera" value="100"/> -+ <nvpair id="status-2-rmq-node-attr-rabbitmq" name="rmq-node-attr-rabbitmq" value="rabbit@node2"/> -+ </instance_attributes> -+ </transient_attributes> -+ <lrm id="2"> -+ <lrm_resources> -+ <lrm_resource id="rabbitmq-bundle-1" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-1"> -+ <lrm_rsc_op id="rabbitmq-bundle-1_last_0" operation_key="rabbitmq-bundle-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="27:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;27:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="12" rc-code="0" op-status="0" interval="0" last-run="1566356445" last-rc-change="1566356445" exec-time="0" queue-time="0" op-digest="4e9836c1fe6ca784363329f38f1a6bab" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ <lrm_rsc_op id="rabbitmq-bundle-1_monitor_60000" operation_key="rabbitmq-bundle-1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="36:754:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;36:754:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="13" rc-code="0" op-status="0" interval="60000" last-rc-change="1566356446" exec-time="0" queue-time="0" op-digest="7fe12a754b9debfddd5a529c95d4ea6c"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-2" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-2"> -+ <lrm_rsc_op id="rabbitmq-bundle-2_last_0" operation_key="rabbitmq-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="28:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;28:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="0" queue-time="0" op-digest="6c8e8fc40a3a8bc0990cb4086a91cb5a" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-docker-0" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="29:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;29:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="19" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="204" queue-time="0" op-digest="c9e60e09c1b2adcdac0169e69cbeb76e"/> -+ </lrm_resource> -+ <lrm_resource id="stonith-fence_ipmilan-node1" type="fence_ipmilan" class="stonith"> -+ <lrm_rsc_op id="stonith-fence_ipmilan-node1_last_0" operation_key="stonith-fence_ipmilan-node1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="125:17:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;125:17:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="82" rc-code="0" op-status="0" interval="0" last-run="1565266314" last-rc-change="1565266314" exec-time="322" queue-time="0" op-digest="40ba273e494269d4d7bded2368059e8d" op-secure-params=" password passwd " op-secure-digest="3ffa7c4b1189e29b87161a1808ba7d9f"/> -+ <lrm_rsc_op id="stonith-fence_ipmilan-node1_monitor_60000" operation_key="stonith-fence_ipmilan-node1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="126:17:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;126:17:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="83" rc-code="0" op-status="0" interval="60000" last-rc-change="1565266315" exec-time="239" queue-time="0" op-digest="865620c5b61a242892a200a6a0e1cd35" op-secure-params=" password passwd " op-secure-digest="3ffa7c4b1189e29b87161a1808ba7d9f"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="23:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;23:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="215" queue-time="0" op-digest="14ba0643d7f91ddb1789cd6c29f9ac3f"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-docker-1" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="rabbitmq-bundle-docker-1_last_0" operation_key="rabbitmq-bundle-docker-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="25:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;25:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="94" rc-code="0" op-status="0" interval="0" last-run="1566356445" last-rc-change="1566356445" exec-time="598" queue-time="0" op-digest="6611df38a8c054188fb3906bac2a4ac6"/> -+ <lrm_rsc_op id="rabbitmq-bundle-docker-1_monitor_60000" operation_key="rabbitmq-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="26:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;26:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="95" rc-code="0" op-status="0" interval="60000" last-rc-change="1566356445" exec-time="163" queue-time="0" op-digest="256b8a21685f8d8237dfff64197a60b0"/> -+ </lrm_resource> -+ <lrm_resource id="stonith-fence_ipmilan-node2" type="fence_ipmilan" class="stonith"> -+ <lrm_rsc_op id="stonith-fence_ipmilan-node2_last_0" operation_key="stonith-fence_ipmilan-node2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="44:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;44:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="69" rc-code="7" op-status="0" interval="0" last-run="1565265997" last-rc-change="1565265997" exec-time="0" queue-time="0" op-digest="7252381349b2722d90a4ce9fba7ba3b5" op-secure-params=" password passwd " op-secure-digest="5265a046a980e16fb93d35f07f19a6d4"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-0" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-0"> -+ <lrm_rsc_op id="galera-bundle-0_last_0" operation_key="galera-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="30:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;30:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="0" queue-time="0" op-digest="8193f24828986601946578a7993cbb74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-docker-1" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="galera-bundle-docker-1_last_0" operation_key="galera-bundle-docker-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="84:12:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;84:12:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="76" rc-code="0" op-status="0" interval="0" last-run="1565265997" last-rc-change="1565265997" exec-time="2055" queue-time="0" op-digest="c6f303ef421236554d0dea3151b97cb3"/> -+ <lrm_rsc_op id="galera-bundle-docker-1_monitor_60000" operation_key="galera-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="85:12:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;85:12:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="80" rc-code="0" op-status="0" interval="60000" last-rc-change="1565265999" exec-time="240" queue-time="0" op-digest="b6b5570d0c79fe1f2c57f97e38d0c454"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-2" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-2"> -+ <lrm_rsc_op id="galera-bundle-2_last_0" operation_key="galera-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="34:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;34:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="0" queue-time="0" op-digest="f724287411b756a9cac4dba67d3bddf6" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-docker-2" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="galera-bundle-docker-2_last_0" operation_key="galera-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="33:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;33:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="28" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="161" queue-time="0" op-digest="bb5745d5e3a6dedaf2449265ce05625b"/> -+ </lrm_resource> -+ <lrm_resource id="stonith-fence_ipmilan-node3" type="fence_ipmilan" class="stonith"> -+ <lrm_rsc_op id="stonith-fence_ipmilan-node3_last_0" operation_key="stonith-fence_ipmilan-node3_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="117:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;117:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="88" rc-code="0" op-status="0" interval="0" last-run="1565925136" last-rc-change="1565925136" exec-time="196" queue-time="0" op-digest="094b99e257ba32c2b718b61813e80256" op-secure-params=" password passwd " op-secure-digest="765c80b4e1ed30f2ef368a8296f7481c"/> -+ <lrm_rsc_op id="stonith-fence_ipmilan-node3_monitor_60000" operation_key="stonith-fence_ipmilan-node3_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="118:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;118:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="89" rc-code="0" op-status="0" interval="60000" last-rc-change="1565925136" exec-time="197" queue-time="0" op-digest="14dda17b1320f5f4ce99f062334a12da" op-secure-params=" password passwd " op-secure-digest="765c80b4e1ed30f2ef368a8296f7481c"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-docker-2" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="rabbitmq-bundle-docker-2_last_0" operation_key="rabbitmq-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="27:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;27:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="14" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="171" queue-time="0" op-digest="7ade29d4352e8007c537badef8a2e0b0"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-1" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-1"> -+ <lrm_rsc_op id="galera-bundle-1_last_0" operation_key="galera-bundle-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="86:12:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;86:12:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="8" rc-code="0" op-status="0" interval="0" last-run="1565265999" last-rc-change="1565265999" exec-time="0" queue-time="0" op-digest="5902e2bf7cfd38a4e959ce335356dc1f" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ <lrm_rsc_op id="galera-bundle-1_monitor_60000" operation_key="galera-bundle-1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="71:13:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;71:13:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="10" rc-code="0" op-status="0" interval="60000" last-rc-change="1565266003" exec-time="0" queue-time="0" op-digest="9cd0e6ae49720131ba1b94817d9df3c7"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0"> -+ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="24:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;24:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="0" queue-time="0" op-digest="2c6afa7ac6b48957892b7238a3270a5d" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ </lrm_resource> -+ </lrm_resources> -+ </lrm> -+ </node_state> -+ <node_state id="3" uname="node3" in_ccm="false" crmd="offline" crm-debug-origin="do_state_transition" join="down" expected="member"> -+ <lrm id="3"> -+ <lrm_resources> -+ <lrm_resource id="rabbitmq-bundle-1" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-1"> -+ <lrm_rsc_op id="rabbitmq-bundle-1_last_0" operation_key="rabbitmq-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="26:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;26:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="0" queue-time="0" op-digest="4e9836c1fe6ca784363329f38f1a6bab" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-2" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-2"> -+ <lrm_rsc_op id="rabbitmq-bundle-2_last_0" operation_key="rabbitmq-bundle-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="31:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;31:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="12" rc-code="0" op-status="0" interval="0" last-run="1566356445" last-rc-change="1566356445" exec-time="0" queue-time="0" op-digest="6c8e8fc40a3a8bc0990cb4086a91cb5a" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ <lrm_rsc_op id="rabbitmq-bundle-2_monitor_60000" operation_key="rabbitmq-bundle-2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="41:754:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;41:754:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="13" rc-code="0" op-status="0" interval="60000" last-rc-change="1566356446" exec-time="0" queue-time="0" op-digest="2633a5ceafe55cf96d15c4984e9cb635"/> -+ </lrm_resource> -+ <lrm_resource id="stonith-fence_ipmilan-node1" type="fence_ipmilan" class="stonith"> -+ <lrm_rsc_op id="stonith-fence_ipmilan-node1_last_0" operation_key="stonith-fence_ipmilan-node1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="42:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;42:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1565267341" last-rc-change="1565267341" exec-time="143" queue-time="1" op-digest="40ba273e494269d4d7bded2368059e8d" op-secure-params=" password passwd " op-secure-digest="3ffa7c4b1189e29b87161a1808ba7d9f"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="23:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;23:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="195" queue-time="0" op-digest="14ba0643d7f91ddb1789cd6c29f9ac3f"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-docker-1" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="rabbitmq-bundle-docker-1_last_0" operation_key="rabbitmq-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="25:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;25:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="10" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="176" queue-time="0" op-digest="6611df38a8c054188fb3906bac2a4ac6"/> -+ </lrm_resource> -+ <lrm_resource id="stonith-fence_ipmilan-node2" type="fence_ipmilan" class="stonith"> -+ <lrm_rsc_op id="stonith-fence_ipmilan-node2_last_0" operation_key="stonith-fence_ipmilan-node2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="120:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;120:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="82" rc-code="0" op-status="0" interval="0" last-run="1565925136" last-rc-change="1565925136" exec-time="251" queue-time="1" op-digest="7252381349b2722d90a4ce9fba7ba3b5" op-secure-params=" password passwd " op-secure-digest="5265a046a980e16fb93d35f07f19a6d4"/> -+ <lrm_rsc_op id="stonith-fence_ipmilan-node2_monitor_60000" operation_key="stonith-fence_ipmilan-node2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="121:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;121:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="83" rc-code="0" op-status="0" interval="60000" last-rc-change="1565925136" exec-time="194" queue-time="0" op-digest="8fa93fafac88b73db2fde1123f0fac16" op-secure-params=" password passwd " op-secure-digest="5265a046a980e16fb93d35f07f19a6d4"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-0" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-0"> -+ <lrm_rsc_op id="galera-bundle-0_last_0" operation_key="galera-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="30:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;30:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="0" queue-time="0" op-digest="8193f24828986601946578a7993cbb74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-1" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-1"> -+ <lrm_rsc_op id="galera-bundle-1_last_0" operation_key="galera-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="32:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;32:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="0" queue-time="0" op-digest="5902e2bf7cfd38a4e959ce335356dc1f" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-docker-2" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="galera-bundle-docker-2_last_0" operation_key="galera-bundle-docker-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="88:12:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:0;88:12:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="76" rc-code="0" op-status="0" interval="0" last-run="1565267341" last-rc-change="1565267341" exec-time="1505" queue-time="0" op-digest="bb5745d5e3a6dedaf2449265ce05625b"/> -+ <lrm_rsc_op id="galera-bundle-docker-2_monitor_60000" operation_key="galera-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="89:12:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:0;89:12:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="78" rc-code="0" op-status="0" interval="60000" last-rc-change="1565267343" exec-time="255" queue-time="0" op-digest="382bc0d634f7dc834b2e7106a1f740f4"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-docker-0" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="29:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;29:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="19" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="163" queue-time="0" op-digest="c9e60e09c1b2adcdac0169e69cbeb76e"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-docker-1" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="galera-bundle-docker-1_last_0" operation_key="galera-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="31:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;31:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="24" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="205" queue-time="0" op-digest="c6f303ef421236554d0dea3151b97cb3"/> -+ </lrm_resource> -+ <lrm_resource id="galera-bundle-2" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-2"> -+ <lrm_rsc_op id="galera-bundle-2_last_0" operation_key="galera-bundle-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="90:12:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:0;90:12:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="7" rc-code="0" op-status="0" interval="0" last-run="1565267343" last-rc-change="1565267343" exec-time="0" queue-time="0" op-digest="f724287411b756a9cac4dba67d3bddf6" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ <lrm_rsc_op id="galera-bundle-2_monitor_60000" operation_key="galera-bundle-2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="75:13:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:0;75:13:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="10" rc-code="0" op-status="0" interval="60000" last-rc-change="1565267347" exec-time="0" queue-time="0" op-digest="d7f8ff67dde73741c39fc99235612a69"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-docker-2" type="docker" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="rabbitmq-bundle-docker-2_last_0" operation_key="rabbitmq-bundle-docker-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="29:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;29:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="90" rc-code="0" op-status="0" interval="0" last-run="1566356445" last-rc-change="1566356445" exec-time="589" queue-time="0" op-digest="7ade29d4352e8007c537badef8a2e0b0"/> -+ <lrm_rsc_op id="rabbitmq-bundle-docker-2_monitor_60000" operation_key="rabbitmq-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="30:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;30:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="91" rc-code="0" op-status="0" interval="60000" last-rc-change="1566356445" exec-time="159" queue-time="0" op-digest="43e4544c25234613205b2a31701bbaa5"/> -+ </lrm_resource> -+ <lrm_resource id="stonith-fence_ipmilan-node3" type="fence_ipmilan" class="stonith"> -+ <lrm_rsc_op id="stonith-fence_ipmilan-node3_last_0" operation_key="stonith-fence_ipmilan-node3_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="43:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;43:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="65" rc-code="7" op-status="0" interval="0" last-run="1565267341" last-rc-change="1565267341" exec-time="0" queue-time="0" op-digest="094b99e257ba32c2b718b61813e80256" op-secure-params=" password passwd " op-secure-digest="765c80b4e1ed30f2ef368a8296f7481c"/> -+ </lrm_resource> -+ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0"> -+ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="24:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;24:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="0" queue-time="0" op-digest="2c6afa7ac6b48957892b7238a3270a5d" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> -+ </lrm_resource> -+ </lrm_resources> -+ </lrm> -+ </node_state> -+ <node_state remote_node="true" id="rabbitmq-bundle-2" uname="rabbitmq-bundle-2" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0"> -+ <lrm id="rabbitmq-bundle-2"> -+ <lrm_resources> -+ <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="48:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;48:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1566356483" last-rc-change="1566356483" exec-time="21171" queue-time="0" op-digest="780d433233eb4f94c1a151623d002e84"/> -+ <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="49:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;49:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="32" rc-code="0" op-status="0" interval="10000" last-rc-change="1566356505" exec-time="6280" queue-time="1" op-digest="6b46cdf9111345cbd0460b2540d3b2c7"/> -+ </lrm_resource> -+ </lrm_resources> -+ </lrm> -+ </node_state> -+ <node_state remote_node="true" id="rabbitmq-bundle-1" uname="rabbitmq-bundle-1" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0"> -+ <lrm id="rabbitmq-bundle-1"> -+ <lrm_resources> -+ <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="46:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;46:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1566356462" last-rc-change="1566356462" exec-time="20783" queue-time="0" op-digest="780d433233eb4f94c1a151623d002e84"/> -+ <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="47:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;47:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="32" rc-code="0" op-status="0" interval="10000" last-rc-change="1566356505" exec-time="6246" queue-time="0" op-digest="6b46cdf9111345cbd0460b2540d3b2c7"/> -+ </lrm_resource> -+ </lrm_resources> -+ </lrm> -+ </node_state> -+ <node_state remote_node="true" id="rabbitmq-bundle-0" uname="rabbitmq-bundle-0" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0"> -+ <lrm id="rabbitmq-bundle-0"> -+ <lrm_resources> -+ <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="49:890:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;49:890:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1566435880" last-rc-change="1566435880" exec-time="24518" queue-time="0" op-digest="780d433233eb4f94c1a151623d002e84"/> -+ <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="46:891:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;46:891:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="32" rc-code="0" op-status="0" interval="10000" last-rc-change="1566435907" exec-time="6252" queue-time="0" op-digest="6b46cdf9111345cbd0460b2540d3b2c7"/> -+ </lrm_resource> -+ </lrm_resources> -+ </lrm> -+ </node_state> -+ <node_state remote_node="true" id="galera-bundle-2" uname="galera-bundle-2" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0"> -+ <lrm id="galera-bundle-2"> -+ <lrm_resources> -+ <lrm_resource id="galera" type="galera" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="galera_last_0" operation_key="galera_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="88:14:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:0;88:14:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1565267357" last-rc-change="1565267357" exec-time="38152" queue-time="0" op-digest="1b6366d05865bce625517a6aaed95684" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> -+ <lrm_rsc_op id="galera_monitor_10000" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="91:15:8:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:8;91:15:8:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="66" rc-code="8" op-status="0" interval="10000" last-rc-change="1565267396" exec-time="1009" queue-time="0" op-digest="29e39b115884dbdd932fbce6501ece1b" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> -+ </lrm_resource> -+ </lrm_resources> -+ </lrm> -+ <transient_attributes id="galera-bundle-2"> -+ <instance_attributes id="status-galera-bundle-2"/> -+ </transient_attributes> -+ </node_state> -+ <node_state remote_node="true" id="galera-bundle-1" uname="galera-bundle-1" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0"> -+ <lrm id="galera-bundle-1"> -+ <lrm_resources> -+ <lrm_resource id="galera" type="galera" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="galera_last_0" operation_key="galera_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="88:14:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;88:14:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1565266015" last-rc-change="1565266015" exec-time="27052" queue-time="0" op-digest="1b6366d05865bce625517a6aaed95684" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> -+ <lrm_rsc_op id="galera_monitor_10000" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="86:15:8:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:8;86:15:8:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="66" rc-code="8" op-status="0" interval="10000" last-rc-change="1565266043" exec-time="961" queue-time="0" op-digest="29e39b115884dbdd932fbce6501ece1b" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> -+ </lrm_resource> -+ </lrm_resources> -+ </lrm> -+ <transient_attributes id="galera-bundle-1"> -+ <instance_attributes id="status-galera-bundle-1"/> -+ </transient_attributes> -+ </node_state> -+ <node_state remote_node="true" id="galera-bundle-0" uname="galera-bundle-0" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0"> -+ <lrm id="galera-bundle-0"> -+ <lrm_resources> -+ <lrm_resource id="galera" type="galera" class="ocf" provider="heartbeat"> -+ <lrm_rsc_op id="galera_last_0" operation_key="galera_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="89:891:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;89:891:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1566435907" last-rc-change="1566435907" exec-time="10647" queue-time="0" op-digest="1b6366d05865bce625517a6aaed95684" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> -+ <lrm_rsc_op id="galera_monitor_10000" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="83:892:8:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:8;83:892:8:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="66" rc-code="8" op-status="0" interval="10000" last-rc-change="1566435919" exec-time="1059" queue-time="0" op-digest="29e39b115884dbdd932fbce6501ece1b" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> -+ <lrm_rsc_op id="galera_last_failure_0" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="83:892:8:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:1;83:892:8:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="local node <node1> is started, but not in primary mode. Unknown state." on_node="node1" call-id="66" rc-code="1" op-status="0" interval="10000" last-rc-change="1566795150" exec-time="0" queue-time="0" op-digest="29e39b115884dbdd932fbce6501ece1b" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> -+ </lrm_resource> -+ </lrm_resources> -+ </lrm> -+ </node_state> -+ </status> -+</cib> --- -1.8.3.1 - diff --git a/SOURCES/02-regression.patch b/SOURCES/02-regression.patch new file mode 100644 index 0000000..63046e8 --- /dev/null +++ b/SOURCES/02-regression.patch @@ -0,0 +1,57 @@ +From a5c40afde45e0a4837baacb3204b90c045b49f69 Mon Sep 17 00:00:00 2001 +From: "Gao,Yan" <ygao@suse.com> +Date: Wed, 9 Oct 2019 12:28:14 +0200 +Subject: [PATCH] Fix: fencing: Do not block concurrent fencing actions on a + device + +Switching to common service interface for fencing actions as of +18c321e79 introduced a regression that concurrent fencing actions on a +fencing device would get blocked. The impact would be very obvious for +"slow" fencing mechanisms such as sbd. +--- + include/crm/services.h | 1 + + lib/fencing/st_client.c | 1 + + lib/services/services.c | 3 ++- + 3 files changed, 4 insertions(+), 1 deletion(-) + +diff --git a/include/crm/services.h b/include/crm/services.h +index cbb2354..61fec27 100644 +--- a/include/crm/services.h ++++ b/include/crm/services.h +@@ -156,6 +156,7 @@ enum nagios_exitcode { + enum svc_action_flags { + /* On timeout, only kill pid, do not kill entire pid group */ + SVC_ACTION_LEAVE_GROUP = 0x01, ++ SVC_ACTION_NON_BLOCKED = 0x02, + }; + + typedef struct svc_action_private_s svc_action_private_t; +diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c +index 60c07d5..95b3c2f 100644 +--- a/lib/fencing/st_client.c ++++ b/lib/fencing/st_client.c +@@ -914,6 +914,7 @@ internal_stonith_action_execute(stonith_action_t * action) + svc_action->sequence = stonith_sequence++; + svc_action->params = action->args; + svc_action->cb_data = (void *) action; ++ set_bit(svc_action->flags, SVC_ACTION_NON_BLOCKED); + + /* keep retries from executing out of control and free previous results */ + if (is_retry) { +diff --git a/lib/services/services.c b/lib/services/services.c +index 1d06c5d..f86c6cc 100644 +--- a/lib/services/services.c ++++ b/lib/services/services.c +@@ -827,7 +827,8 @@ services_action_async_fork_notify(svc_action_t * op, + g_hash_table_replace(recurring_actions, op->id, op); + } + +- if (op->rsc && is_op_blocked(op->rsc)) { ++ if (is_not_set(op->flags, SVC_ACTION_NON_BLOCKED) ++ && op->rsc && is_op_blocked(op->rsc)) { + blocked_ops = g_list_append(blocked_ops, op); + return TRUE; + } +-- +1.8.3.1 + diff --git a/SOURCES/03-guest-node.patch b/SOURCES/03-guest-node.patch new file mode 100644 index 0000000..1291635 --- /dev/null +++ b/SOURCES/03-guest-node.patch @@ -0,0 +1,126 @@ +From 11685256d35035ae69985d1f4536d0ed68951efe Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 24 Oct 2019 17:35:48 -0500 +Subject: [PATCH 4/5] Fix: scheduler: properly detect whether guest node is + fenceable + +Guest nodes are "fenced" by stopping their container resource. Previously, we +assumed that this was always possible. However, it may not be if the +container's host is failed and not fenceable (e.g. due to lack of quorum). + +Now, we check guest nodes for fenceability as we do for other nodes, +with the criteria being that the guest's host must be either online or +fenceable. Additionally, when creating a new action that normally does not +require fencing, we make the action unrunnable if it is on an non-fenceable +guest node, because the action cannot be attempted in that case. +--- + lib/pengine/utils.c | 55 ++++++++++++++++++++++++++++++++++++++--------------- + pengine/allocate.c | 3 ++- + 2 files changed, 42 insertions(+), 16 deletions(-) + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 97241af..671ef76 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -92,36 +92,49 @@ pe_free_rsc_action_details(pe_action_t *action) + * \param[in] data_set Working set for cluster + * \param[in] node Name of node to check + * +- * \return TRUE if node can be fenced, FALSE otherwise +- * +- * \note This function should only be called for cluster nodes and baremetal +- * remote nodes; guest nodes are fenced by stopping their container +- * resource, so fence execution requirements do not apply to them. ++ * \return true if node can be fenced, false otherwise + */ +-bool pe_can_fence(pe_working_set_t * data_set, node_t *node) ++bool ++pe_can_fence(pe_working_set_t *data_set, pe_node_t *node) + { +- if(is_not_set(data_set->flags, pe_flag_stonith_enabled)) { +- return FALSE; /* Turned off */ ++ if (is_container_remote_node(node)) { ++ /* Guest nodes are fenced by stopping their container resource. We can ++ * do that if the container's host is either online or fenceable. ++ */ ++ pe_resource_t *rsc = node->details->remote_rsc->container; ++ ++ for (GList *n = rsc->running_on; n != NULL; n = n->next) { ++ pe_node_t *container_node = n->data; ++ ++ if (!container_node->details->online ++ && !pe_can_fence(data_set, container_node)) { ++ return false; ++ } ++ } ++ return true; ++ ++ } else if(is_not_set(data_set->flags, pe_flag_stonith_enabled)) { ++ return false; /* Turned off */ + + } else if (is_not_set(data_set->flags, pe_flag_have_stonith_resource)) { +- return FALSE; /* No devices */ ++ return false; /* No devices */ + + } else if (is_set(data_set->flags, pe_flag_have_quorum)) { +- return TRUE; ++ return true; + + } else if (data_set->no_quorum_policy == no_quorum_ignore) { +- return TRUE; ++ return true; + + } else if(node == NULL) { +- return FALSE; ++ return false; + + } else if(node->details->online) { + crm_notice("We can fence %s without quorum because they're in our membership", node->details->uname); +- return TRUE; ++ return true; + } + + crm_trace("Cannot fence %s", node->details->uname); +- return FALSE; ++ return false; + } + + node_t * +@@ -576,7 +589,19 @@ custom_action(resource_t * rsc, char *key, const char *task, + } else if (action->needs == rsc_req_nothing) { + pe_rsc_trace(rsc, "Action %s does not require anything", action->uuid); + pe_action_set_reason(action, NULL, TRUE); +- pe_set_action_bit(action, pe_action_runnable); ++ if (is_container_remote_node(action->node) ++ && !pe_can_fence(data_set, action->node)) { ++ /* An action that requires nothing usually does not require any ++ * fencing in order to be runnable. However, there is an ++ * exception: an action cannot be completed if it is on a guest ++ * node whose host is unclean and cannot be fenced. ++ */ ++ pe_clear_action_bit(action, pe_action_runnable); ++ crm_debug("%s\t%s (cancelled : host cannot be fenced)", ++ action->node->details->uname, action->uuid); ++ } else { ++ pe_set_action_bit(action, pe_action_runnable); ++ } + #if 0 + /* + * No point checking this +diff --git a/pengine/allocate.c b/pengine/allocate.c +index e30cb1c..b819af3 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1584,7 +1584,8 @@ stage6(pe_working_set_t * data_set) + * so handle them separately. + */ + if (is_container_remote_node(node)) { +- if (node->details->remote_requires_reset && need_stonith) { ++ if (node->details->remote_requires_reset && need_stonith ++ && pe_can_fence(data_set, node)) { + fence_guest(node, data_set); + } + continue; +-- +1.8.3.1 + diff --git a/SOURCES/04-guest-node-test.patch b/SOURCES/04-guest-node-test.patch new file mode 100644 index 0000000..95cf581 --- /dev/null +++ b/SOURCES/04-guest-node-test.patch @@ -0,0 +1,1297 @@ +From 739fb14ad27e3a8ab18e92a23f0926c773983ad6 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Tue, 29 Oct 2019 17:44:40 -0500 +Subject: [PATCH 5/5] Test: scheduler: add regression test for guest node with + unclean host that cannot be fenced + +--- + pengine/regression.sh | 1 + + pengine/test10/guest-host-not-fenceable.dot | 258 +++++++++++++++ + pengine/test10/guest-host-not-fenceable.exp | 340 +++++++++++++++++++ + pengine/test10/guest-host-not-fenceable.scores | 134 ++++++++ + pengine/test10/guest-host-not-fenceable.summary | 87 +++++ + pengine/test10/guest-host-not-fenceable.xml | 413 ++++++++++++++++++++++++ + 6 files changed, 1233 insertions(+) + create mode 100644 pengine/test10/guest-host-not-fenceable.dot + create mode 100644 pengine/test10/guest-host-not-fenceable.exp + create mode 100644 pengine/test10/guest-host-not-fenceable.scores + create mode 100644 pengine/test10/guest-host-not-fenceable.summary + create mode 100755 pengine/test10/guest-host-not-fenceable.xml + +diff --git a/pengine/regression.sh b/pengine/regression.sh +index 25d9e3f..f2226ed 100755 +--- a/pengine/regression.sh ++++ b/pengine/regression.sh +@@ -862,6 +862,7 @@ do_test whitebox-imply-stop-on-fence "imply stop action on container node rsc wh + do_test whitebox-nested-group "Verify guest remote-node works nested in a group" + do_test guest-node-host-dies "Verify guest node is recovered if host goes away" + do_test guest-node-cleanup "Order guest node connection recovery after container probe" ++do_test guest-host-not-fenceable "Actions on guest node are unrunnable if host is unclean and cannot be fenced" + + echo "" + do_test remote-startup-probes "Baremetal remote-node startup probes" +diff --git a/pengine/test10/guest-host-not-fenceable.dot b/pengine/test10/guest-host-not-fenceable.dot +new file mode 100644 +index 0000000..a510aaf +--- /dev/null ++++ b/pengine/test10/guest-host-not-fenceable.dot +@@ -0,0 +1,258 @@ ++ digraph "g" { ++"galera-bundle-0_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-0_start_0 node1" -> "galera-bundle-0_monitor_60000 node1" [ style = dashed] ++"galera-bundle-0_start_0 node1" -> "galera_clear_failcount_0 galera-bundle-0" [ style = dashed] ++"galera-bundle-0_start_0 node1" -> "galera_monitor_10000 galera-bundle-0" [ style = dashed] ++"galera-bundle-0_start_0 node1" -> "galera_start_0 galera-bundle-0" [ style = dashed] ++"galera-bundle-0_start_0 node1" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-0_stop_0 node1" -> "galera-bundle-0_start_0 node1" [ style = dashed] ++"galera-bundle-0_stop_0 node1" -> "galera-bundle-docker-0_stop_0 node1" [ style = bold] ++"galera-bundle-0_stop_0 node1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-1_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-1_start_0 node1" -> "galera-bundle-1_monitor_60000 node1" [ style = dashed] ++"galera-bundle-1_start_0 node1" -> "galera_monitor_10000 galera-bundle-1" [ style = dashed] ++"galera-bundle-1_start_0 node1" -> "galera_start_0 galera-bundle-1" [ style = dashed] ++"galera-bundle-1_start_0 node1" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-1_stop_0 node2" -> "galera-bundle-1_start_0 node1" [ style = dashed] ++"galera-bundle-1_stop_0 node2" -> "galera-bundle-docker-1_stop_0 node2" [ style = dashed] ++"galera-bundle-1_stop_0 node2" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-2_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-2_start_0 node1" -> "galera-bundle-2_monitor_60000 node1" [ style = dashed] ++"galera-bundle-2_start_0 node1" -> "galera_monitor_20000 galera-bundle-2" [ style = dashed] ++"galera-bundle-2_start_0 node1" -> "galera_monitor_30000 galera-bundle-2" [ style = dashed] ++"galera-bundle-2_start_0 node1" -> "galera_start_0 galera-bundle-2" [ style = dashed] ++"galera-bundle-2_start_0 node1" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-2_stop_0 node3" -> "galera-bundle-2_start_0 node1" [ style = dashed] ++"galera-bundle-2_stop_0 node3" -> "galera-bundle-docker-2_stop_0 node3" [ style = dashed] ++"galera-bundle-2_stop_0 node3" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-docker-0_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-docker-0_start_0 node1" -> "galera-bundle-0_start_0 node1" [ style = dashed] ++"galera-bundle-docker-0_start_0 node1" -> "galera-bundle-docker-0_monitor_60000 node1" [ style = dashed] ++"galera-bundle-docker-0_start_0 node1" -> "galera-bundle_running_0" [ style = dashed] ++"galera-bundle-docker-0_start_0 node1" -> "galera_start_0 galera-bundle-0" [ style = dashed] ++"galera-bundle-docker-0_start_0 node1" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-docker-0_stop_0 node1" -> "galera-bundle-docker-0_start_0 node1" [ style = dashed] ++"galera-bundle-docker-0_stop_0 node1" -> "galera-bundle_stopped_0" [ style = bold] ++"galera-bundle-docker-0_stop_0 node1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-1_stop_0 node2" -> "galera-bundle_stopped_0" [ style = dashed] ++"galera-bundle-docker-1_stop_0 node2" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-docker-2_stop_0 node3" -> "galera-bundle_stopped_0" [ style = dashed] ++"galera-bundle-docker-2_stop_0 node3" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold] ++"galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold] ++"galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-1" [ style = dashed] ++"galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-2" [ style = dashed] ++"galera-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_demoted_0" -> "galera-bundle-master_start_0" [ style = dashed] ++"galera-bundle-master_demoted_0" -> "galera-bundle-master_stop_0" [ style = bold] ++"galera-bundle-master_demoted_0" -> "galera-bundle_demoted_0" [ style = bold] ++"galera-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = dashed] ++"galera-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"] ++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = dashed] ++"galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-0" [ style = dashed] ++"galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-1" [ style = dashed] ++"galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-2" [ style = dashed] ++"galera-bundle-master_start_0" [ style=dashed color="red" fontcolor="orange"] ++"galera-bundle-master_stop_0" -> "galera-bundle-master_stopped_0" [ style = bold] ++"galera-bundle-master_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] ++"galera-bundle-master_stop_0" -> "galera_stop_0 galera-bundle-1" [ style = dashed] ++"galera-bundle-master_stop_0" -> "galera_stop_0 galera-bundle-2" [ style = dashed] ++"galera-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_stopped_0" -> "galera-bundle-master_start_0" [ style = dashed] ++"galera-bundle-master_stopped_0" -> "galera-bundle_stopped_0" [ style = bold] ++"galera-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_demote_0" -> "galera-bundle-master_demote_0" [ style = bold] ++"galera-bundle_demote_0" -> "galera-bundle_demoted_0" [ style = bold] ++"galera-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_demoted_0" -> "galera-bundle_start_0" [ style = dashed] ++"galera-bundle_demoted_0" -> "galera-bundle_stop_0" [ style = bold] ++"galera-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] ++"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 node1" [ style = dashed] ++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = dashed] ++"galera-bundle_start_0" [ style=dashed color="red" fontcolor="orange"] ++"galera-bundle_stop_0" -> "galera-bundle-docker-0_stop_0 node1" [ style = bold] ++"galera-bundle_stop_0" -> "galera-bundle-docker-1_stop_0 node2" [ style = dashed] ++"galera-bundle_stop_0" -> "galera-bundle-docker-2_stop_0 node3" [ style = dashed] ++"galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold] ++"galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] ++"galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-1" [ style = dashed] ++"galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-2" [ style = dashed] ++"galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_stopped_0" -> "galera-bundle_start_0" [ style = dashed] ++"galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"galera_clear_failcount_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"galera_demote_0 galera-bundle-0" -> "galera-bundle-0_stop_0 node1" [ style = bold] ++"galera_demote_0 galera-bundle-0" -> "galera-bundle-master_demoted_0" [ style = bold] ++"galera_demote_0 galera-bundle-0" -> "galera_stop_0 galera-bundle-0" [ style = bold] ++"galera_demote_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"galera_demote_0 galera-bundle-1" -> "galera-bundle-master_demoted_0" [ style = dashed] ++"galera_demote_0 galera-bundle-1" -> "galera_demote_0 galera-bundle-0" [ style = dashed] ++"galera_demote_0 galera-bundle-1" -> "galera_stop_0 galera-bundle-1" [ style = dashed] ++"galera_demote_0 galera-bundle-1" [ style=dashed color="red" fontcolor="black"] ++"galera_demote_0 galera-bundle-2" -> "galera-bundle-master_demoted_0" [ style = dashed] ++"galera_demote_0 galera-bundle-2" -> "galera_demote_0 galera-bundle-1" [ style = dashed] ++"galera_demote_0 galera-bundle-2" -> "galera_monitor_20000 galera-bundle-2" [ style = dashed] ++"galera_demote_0 galera-bundle-2" -> "galera_monitor_30000 galera-bundle-2" [ style = dashed] ++"galera_demote_0 galera-bundle-2" -> "galera_stop_0 galera-bundle-2" [ style = dashed] ++"galera_demote_0 galera-bundle-2" [ style=dashed color="red" fontcolor="black"] ++"galera_monitor_10000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"galera_monitor_10000 galera-bundle-1" [ style=dashed color="red" fontcolor="black"] ++"galera_monitor_20000 galera-bundle-2" [ style=dashed color="red" fontcolor="black"] ++"galera_monitor_30000 galera-bundle-2" [ style=dashed color="red" fontcolor="black"] ++"galera_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = dashed] ++"galera_start_0 galera-bundle-0" -> "galera_monitor_10000 galera-bundle-0" [ style = dashed] ++"galera_start_0 galera-bundle-0" -> "galera_start_0 galera-bundle-1" [ style = dashed] ++"galera_start_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"galera_start_0 galera-bundle-1" -> "galera-bundle-master_running_0" [ style = dashed] ++"galera_start_0 galera-bundle-1" -> "galera_monitor_10000 galera-bundle-1" [ style = dashed] ++"galera_start_0 galera-bundle-1" -> "galera_start_0 galera-bundle-2" [ style = dashed] ++"galera_start_0 galera-bundle-1" [ style=dashed color="red" fontcolor="black"] ++"galera_start_0 galera-bundle-2" -> "galera-bundle-master_running_0" [ style = dashed] ++"galera_start_0 galera-bundle-2" -> "galera_monitor_20000 galera-bundle-2" [ style = dashed] ++"galera_start_0 galera-bundle-2" -> "galera_monitor_30000 galera-bundle-2" [ style = dashed] ++"galera_start_0 galera-bundle-2" [ style=dashed color="red" fontcolor="black"] ++"galera_stop_0 galera-bundle-0" -> "galera-bundle-0_stop_0 node1" [ style = bold] ++"galera_stop_0 galera-bundle-0" -> "galera-bundle-master_stopped_0" [ style = bold] ++"galera_stop_0 galera-bundle-0" -> "galera_start_0 galera-bundle-0" [ style = dashed] ++"galera_stop_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"galera_stop_0 galera-bundle-1" -> "galera-bundle-master_stopped_0" [ style = dashed] ++"galera_stop_0 galera-bundle-1" -> "galera_start_0 galera-bundle-1" [ style = dashed] ++"galera_stop_0 galera-bundle-1" -> "galera_stop_0 galera-bundle-0" [ style = dashed] ++"galera_stop_0 galera-bundle-1" [ style=dashed color="red" fontcolor="black"] ++"galera_stop_0 galera-bundle-2" -> "galera-bundle-master_stopped_0" [ style = dashed] ++"galera_stop_0 galera-bundle-2" -> "galera_start_0 galera-bundle-2" [ style = dashed] ++"galera_stop_0 galera-bundle-2" -> "galera_stop_0 galera-bundle-1" [ style = dashed] ++"galera_stop_0 galera-bundle-2" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-0_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-0_start_0 node1" -> "rabbitmq-bundle-0_monitor_60000 node1" [ style = dashed] ++"rabbitmq-bundle-0_start_0 node1" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq-bundle-0_start_0 node1" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq-bundle-0_start_0 node1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-0_stop_0 node1" -> "rabbitmq-bundle-0_start_0 node1" [ style = dashed] ++"rabbitmq-bundle-0_stop_0 node1" -> "rabbitmq-bundle-docker-0_stop_0 node1" [ style = bold] ++"rabbitmq-bundle-0_stop_0 node1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-1_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-1_start_0 node1" -> "rabbitmq-bundle-1_monitor_60000 node1" [ style = dashed] ++"rabbitmq-bundle-1_start_0 node1" -> "rabbitmq_monitor_10000 rabbitmq-bundle-1" [ style = dashed] ++"rabbitmq-bundle-1_start_0 node1" -> "rabbitmq_start_0 rabbitmq-bundle-1" [ style = dashed] ++"rabbitmq-bundle-1_start_0 node1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-1_stop_0 node2" -> "rabbitmq-bundle-1_start_0 node1" [ style = dashed] ++"rabbitmq-bundle-1_stop_0 node2" -> "rabbitmq-bundle-docker-1_stop_0 node2" [ style = dashed] ++"rabbitmq-bundle-1_stop_0 node2" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-2_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-2_start_0 node1" -> "rabbitmq-bundle-2_monitor_60000 node1" [ style = dashed] ++"rabbitmq-bundle-2_start_0 node1" -> "rabbitmq_monitor_10000 rabbitmq-bundle-2" [ style = dashed] ++"rabbitmq-bundle-2_start_0 node1" -> "rabbitmq_start_0 rabbitmq-bundle-2" [ style = dashed] ++"rabbitmq-bundle-2_start_0 node1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-2_stop_0 node3" -> "rabbitmq-bundle-2_start_0 node1" [ style = dashed] ++"rabbitmq-bundle-2_stop_0 node3" -> "rabbitmq-bundle-docker-2_stop_0 node3" [ style = dashed] ++"rabbitmq-bundle-2_stop_0 node3" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle-clone_pre_notify_start_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_post_notify_running_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_post_notify_stopped_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = dashed] ++"rabbitmq-bundle-clone_pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" [ style = dashed] ++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" [ style = dashed] ++"rabbitmq-bundle-clone_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_running_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-1" [ style = dashed] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-2" [ style = dashed] ++"rabbitmq-bundle-clone_start_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-1" [ style = dashed] ++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-2" [ style = dashed] ++"rabbitmq-bundle-clone_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed] ++"rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-docker-0_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-docker-0_start_0 node1" -> "rabbitmq-bundle-0_start_0 node1" [ style = dashed] ++"rabbitmq-bundle-docker-0_start_0 node1" -> "rabbitmq-bundle-docker-0_monitor_60000 node1" [ style = dashed] ++"rabbitmq-bundle-docker-0_start_0 node1" -> "rabbitmq-bundle_running_0" [ style = dashed] ++"rabbitmq-bundle-docker-0_start_0 node1" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq-bundle-docker-0_start_0 node1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-docker-0_stop_0 node1" -> "rabbitmq-bundle-docker-0_start_0 node1" [ style = dashed] ++"rabbitmq-bundle-docker-0_stop_0 node1" -> "rabbitmq-bundle_stopped_0" [ style = bold] ++"rabbitmq-bundle-docker-0_stop_0 node1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-docker-1_stop_0 node2" -> "rabbitmq-bundle_stopped_0" [ style = dashed] ++"rabbitmq-bundle-docker-1_stop_0 node2" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-docker-2_stop_0 node3" -> "rabbitmq-bundle_stopped_0" [ style = dashed] ++"rabbitmq-bundle-docker-2_stop_0 node3" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed] ++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-docker-0_start_0 node1" [ style = dashed] ++"rabbitmq-bundle_start_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] ++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-docker-0_stop_0 node1" [ style = bold] ++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-docker-1_stop_0 node2" [ style = dashed] ++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-docker-2_stop_0 node3" [ style = dashed] ++"rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-1" [ style = dashed] ++"rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-2" [ style = dashed] ++"rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_monitor_10000 rabbitmq-bundle-1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_monitor_10000 rabbitmq-bundle-2" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = dashed] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = dashed] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq_start_0 rabbitmq-bundle-1" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_start_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-1" -> "rabbitmq_monitor_10000 rabbitmq-bundle-1" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-1" -> "rabbitmq_start_0 rabbitmq-bundle-2" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_start_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-2" -> "rabbitmq_monitor_10000 rabbitmq-bundle-2" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-2" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-0_stop_0 node1" [ style = bold] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq_stop_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_stopped_0" [ style = dashed] ++"rabbitmq_stop_0 rabbitmq-bundle-1" -> "rabbitmq_start_0 rabbitmq-bundle-1" [ style = dashed] ++"rabbitmq_stop_0 rabbitmq-bundle-1" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq_stop_0 rabbitmq-bundle-1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_stop_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_stopped_0" [ style = dashed] ++"rabbitmq_stop_0 rabbitmq-bundle-2" -> "rabbitmq_start_0 rabbitmq-bundle-2" [ style = dashed] ++"rabbitmq_stop_0 rabbitmq-bundle-2" -> "rabbitmq_stop_0 rabbitmq-bundle-1" [ style = dashed] ++"rabbitmq_stop_0 rabbitmq-bundle-2" [ style=dashed color="red" fontcolor="black"] ++"stonith-fence_ipmilan-node1_stop_0 node2" [ style=dashed color="red" fontcolor="black"] ++"stonith-fence_ipmilan-node2_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] ++"stonith-fence_ipmilan-node2_start_0 node1" -> "stonith-fence_ipmilan-node2_monitor_60000 node1" [ style = dashed] ++"stonith-fence_ipmilan-node2_start_0 node1" [ style=dashed color="red" fontcolor="black"] ++"stonith-fence_ipmilan-node2_stop_0 node3" -> "stonith-fence_ipmilan-node2_start_0 node1" [ style = dashed] ++"stonith-fence_ipmilan-node2_stop_0 node3" [ style=dashed color="red" fontcolor="black"] ++"stonith-fence_ipmilan-node3_monitor_60000 node1" [ style=dashed color="red" fontcolor="black"] ++"stonith-fence_ipmilan-node3_start_0 node1" -> "stonith-fence_ipmilan-node3_monitor_60000 node1" [ style = dashed] ++"stonith-fence_ipmilan-node3_start_0 node1" [ style=dashed color="red" fontcolor="black"] ++"stonith-fence_ipmilan-node3_stop_0 node2" -> "stonith-fence_ipmilan-node3_start_0 node1" [ style = dashed] ++"stonith-fence_ipmilan-node3_stop_0 node2" [ style=dashed color="red" fontcolor="black"] ++} +diff --git a/pengine/test10/guest-host-not-fenceable.exp b/pengine/test10/guest-host-not-fenceable.exp +new file mode 100644 +index 0000000..ed28f2b +--- /dev/null ++++ b/pengine/test10/guest-host-not-fenceable.exp +@@ -0,0 +1,340 @@ ++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0"> ++ <synapse id="0"> ++ <action_set> ++ <rsc_op id="108" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1"> ++ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/> ++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_active_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_all_hosts="node1 node2 node3 node1 node2 node3 node1 node2 node3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 node1 node2 node3 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="node1 node2 node3 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_physical_host="node1" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/> ++ </rsc_op> ++ </action_set> ++ <inputs> ++ <trigger> ++ <pseudo_event id="47" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="1"> ++ <action_set> ++ <rsc_op id="31" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1"> ++ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/> ++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_active_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_all_hosts="node1 node2 node3 node1 node2 node3 node1 node2 node3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 node1 node2 node3 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="node1 node2 node3 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_physical_host="node1" CRM_meta_timeout="200000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/> ++ </rsc_op> ++ </action_set> ++ <inputs> ++ <trigger> ++ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle_stop_0"/> ++ </trigger> ++ <trigger> ++ <pseudo_event id="45" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="2" priority="1000000"> ++ <action_set> ++ <pseudo_event id="50" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-post_notify_stopped_0"> ++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <pseudo_event id="49" operation="notify" operation_key="rabbitmq-bundle-clone_post_notify_stopped_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="3" priority="1000000"> ++ <action_set> ++ <pseudo_event id="49" operation="notify" operation_key="rabbitmq-bundle-clone_post_notify_stopped_0"> ++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <pseudo_event id="46" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0"/> ++ </trigger> ++ <trigger> ++ <pseudo_event id="48" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-pre_notify_stop_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="4"> ++ <action_set> ++ <pseudo_event id="48" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-pre_notify_stop_0"> ++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <pseudo_event id="47" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"/> ++ </trigger> ++ <trigger> ++ <rsc_op id="108" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="5"> ++ <action_set> ++ <pseudo_event id="47" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"> ++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs/> ++ </synapse> ++ <synapse id="6" priority="1000000"> ++ <action_set> ++ <pseudo_event id="46" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0"> ++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="31" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1"/> ++ </trigger> ++ <trigger> ++ <pseudo_event id="45" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="7"> ++ <action_set> ++ <pseudo_event id="45" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0"> ++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle_stop_0"/> ++ </trigger> ++ <trigger> ++ <pseudo_event id="48" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-pre_notify_stop_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="8"> ++ <action_set> ++ <rsc_op id="16" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="node1" on_node_uuid="1"> ++ <primitive id="rabbitmq-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/> ++ <attributes CRM_meta_on_node="node1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.122.139:8787/rhosp13/openstack-rabbitmq:pcmklatest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/rabbitmq-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3122 -v /var/lib/kolla/config_files/rabbitmq.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/puppet-generated/rabbitmq/:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/rabbitmq:/var/lib/rabbitmq:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /var/log/containers/rabbitmq:/var/log/rabbitmq:rw -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/rabbitmq-bundle-0:/var/log --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/> ++ </rsc_op> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="17" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="node1" on_node_uuid="1"/> ++ </trigger> ++ <trigger> ++ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle_stop_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="9"> ++ <action_set> ++ <rsc_op id="17" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="node1" on_node_uuid="1"> ++ <primitive id="rabbitmq-bundle-0" class="ocf" provider="pacemaker" type="remote"/> ++ <attributes CRM_meta_container="rabbitmq-bundle-docker-0" CRM_meta_on_node="node1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="node1" port="3122"/> ++ <downed> ++ <node id="rabbitmq-bundle-0"/> ++ </downed> ++ </rsc_op> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="31" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="10"> ++ <action_set> ++ <rsc_op id="66" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"> ++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/> ++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_physical_host="node1" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="node1:node1.internalapi.localdomain;node2:node2.internalapi.localdomain;node3:node3.internalapi.localdomain" enable_creation="true" log="/var/log/mysql/mysqld.log" wsrep_cluster_address="gcomm://node1.internalapi.localdomain,node2.internalapi.localdomain,node3.internalapi.localdomain"/> ++ </rsc_op> ++ </action_set> ++ <inputs> ++ <trigger> ++ <pseudo_event id="82" operation="demote" operation_key="galera-bundle-master_demote_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="11"> ++ <action_set> ++ <rsc_op id="10" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"> ++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/> ++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_physical_host="node1" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="node1:node1.internalapi.localdomain;node2:node2.internalapi.localdomain;node3:node3.internalapi.localdomain" enable_creation="true" log="/var/log/mysql/mysqld.log" wsrep_cluster_address="gcomm://node1.internalapi.localdomain,node2.internalapi.localdomain,node3.internalapi.localdomain"/> ++ </rsc_op> ++ </action_set> ++ <inputs> ++ <trigger> ++ <pseudo_event id="64" operation="stop" operation_key="galera-bundle_stop_0"/> ++ </trigger> ++ <trigger> ++ <rsc_op id="66" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"/> ++ </trigger> ++ <trigger> ++ <pseudo_event id="78" operation="stop" operation_key="galera-bundle-master_stop_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="12" priority="1000000"> ++ <action_set> ++ <pseudo_event id="83" operation="demoted" operation_key="galera-bundle-master_demoted_0"> ++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="66" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"/> ++ </trigger> ++ <trigger> ++ <pseudo_event id="82" operation="demote" operation_key="galera-bundle-master_demote_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="13"> ++ <action_set> ++ <pseudo_event id="82" operation="demote" operation_key="galera-bundle-master_demote_0"> ++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <pseudo_event id="86" operation="demote" operation_key="galera-bundle_demote_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="14" priority="1000000"> ++ <action_set> ++ <pseudo_event id="79" operation="stopped" operation_key="galera-bundle-master_stopped_0"> ++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="10" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"/> ++ </trigger> ++ <trigger> ++ <pseudo_event id="78" operation="stop" operation_key="galera-bundle-master_stop_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="15"> ++ <action_set> ++ <pseudo_event id="78" operation="stop" operation_key="galera-bundle-master_stop_0"> ++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <pseudo_event id="64" operation="stop" operation_key="galera-bundle_stop_0"/> ++ </trigger> ++ <trigger> ++ <pseudo_event id="83" operation="demoted" operation_key="galera-bundle-master_demoted_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="16"> ++ <action_set> ++ <rsc_op id="51" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="node1" on_node_uuid="1"> ++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/> ++ <attributes CRM_meta_on_node="node1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.122.139:8787/rhosp13/openstack-mariadb:pcmklatest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/puppet-generated/mysql/:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /var/log/containers/mysql:/var/log/mysql:rw -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/> ++ </rsc_op> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="52" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="node1" on_node_uuid="1"/> ++ </trigger> ++ <trigger> ++ <pseudo_event id="64" operation="stop" operation_key="galera-bundle_stop_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="17"> ++ <action_set> ++ <rsc_op id="52" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="node1" on_node_uuid="1"> ++ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/> ++ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_on_node="node1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="node1" port="3123"/> ++ <downed> ++ <node id="galera-bundle-0"/> ++ </downed> ++ </rsc_op> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="10" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"/> ++ </trigger> ++ <trigger> ++ <rsc_op id="66" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="18" priority="1000000"> ++ <action_set> ++ <pseudo_event id="87" operation="demoted" operation_key="galera-bundle_demoted_0"> ++ <attributes CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <pseudo_event id="83" operation="demoted" operation_key="galera-bundle-master_demoted_0"/> ++ </trigger> ++ <trigger> ++ <pseudo_event id="86" operation="demote" operation_key="galera-bundle_demote_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="19"> ++ <action_set> ++ <pseudo_event id="86" operation="demote" operation_key="galera-bundle_demote_0"> ++ <attributes CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs/> ++ </synapse> ++ <synapse id="20" priority="1000000"> ++ <action_set> ++ <pseudo_event id="65" operation="stopped" operation_key="galera-bundle_stopped_0"> ++ <attributes CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="51" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="node1" on_node_uuid="1"/> ++ </trigger> ++ <trigger> ++ <pseudo_event id="79" operation="stopped" operation_key="galera-bundle-master_stopped_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="21"> ++ <action_set> ++ <pseudo_event id="64" operation="stop" operation_key="galera-bundle_stop_0"> ++ <attributes CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <pseudo_event id="87" operation="demoted" operation_key="galera-bundle_demoted_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="22" priority="1000000"> ++ <action_set> ++ <pseudo_event id="30" operation="stopped" operation_key="rabbitmq-bundle_stopped_0"> ++ <attributes CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="16" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="node1" on_node_uuid="1"/> ++ </trigger> ++ <trigger> ++ <pseudo_event id="50" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-post_notify_stopped_0"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="23"> ++ <action_set> ++ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle_stop_0"> ++ <attributes CRM_meta_timeout="20000" /> ++ </pseudo_event> ++ </action_set> ++ <inputs/> ++ </synapse> ++</transition_graph> +diff --git a/pengine/test10/guest-host-not-fenceable.scores b/pengine/test10/guest-host-not-fenceable.scores +new file mode 100644 +index 0000000..80bd0d4 +--- /dev/null ++++ b/pengine/test10/guest-host-not-fenceable.scores +@@ -0,0 +1,134 @@ ++Allocation scores: ++Using the original execution date of: 2019-08-26 04:52:42Z ++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0 ++clone_color: galera-bundle-master allocation score on galera-bundle-1: 0 ++clone_color: galera-bundle-master allocation score on galera-bundle-2: 0 ++clone_color: galera-bundle-master allocation score on node1: -INFINITY ++clone_color: galera-bundle-master allocation score on node2: -INFINITY ++clone_color: galera-bundle-master allocation score on node3: -INFINITY ++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY ++clone_color: galera:1 allocation score on galera-bundle-1: INFINITY ++clone_color: galera:2 allocation score on galera-bundle-2: INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on node1: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on node2: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on node3: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: 0 ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: 0 ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY ++clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY ++container_color: galera-bundle allocation score on node1: 0 ++container_color: galera-bundle allocation score on node2: 0 ++container_color: galera-bundle allocation score on node3: 0 ++container_color: galera-bundle-0 allocation score on node1: INFINITY ++container_color: galera-bundle-0 allocation score on node2: 0 ++container_color: galera-bundle-0 allocation score on node3: 0 ++container_color: galera-bundle-1 allocation score on node1: 0 ++container_color: galera-bundle-1 allocation score on node2: INFINITY ++container_color: galera-bundle-1 allocation score on node3: 0 ++container_color: galera-bundle-2 allocation score on node1: 0 ++container_color: galera-bundle-2 allocation score on node2: 0 ++container_color: galera-bundle-2 allocation score on node3: INFINITY ++container_color: galera-bundle-docker-0 allocation score on node1: INFINITY ++container_color: galera-bundle-docker-0 allocation score on node2: 0 ++container_color: galera-bundle-docker-0 allocation score on node3: 0 ++container_color: galera-bundle-docker-1 allocation score on node1: 0 ++container_color: galera-bundle-docker-1 allocation score on node2: INFINITY ++container_color: galera-bundle-docker-1 allocation score on node3: 0 ++container_color: galera-bundle-docker-2 allocation score on node1: 0 ++container_color: galera-bundle-docker-2 allocation score on node2: 0 ++container_color: galera-bundle-docker-2 allocation score on node3: INFINITY ++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-master allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-master allocation score on node1: 0 ++container_color: galera-bundle-master allocation score on node2: 0 ++container_color: galera-bundle-master allocation score on node3: 0 ++container_color: galera:0 allocation score on galera-bundle-0: INFINITY ++container_color: galera:1 allocation score on galera-bundle-1: INFINITY ++container_color: galera:2 allocation score on galera-bundle-2: INFINITY ++container_color: rabbitmq-bundle allocation score on node1: 0 ++container_color: rabbitmq-bundle allocation score on node2: 0 ++container_color: rabbitmq-bundle allocation score on node3: 0 ++container_color: rabbitmq-bundle-0 allocation score on node1: INFINITY ++container_color: rabbitmq-bundle-0 allocation score on node2: 0 ++container_color: rabbitmq-bundle-0 allocation score on node3: 0 ++container_color: rabbitmq-bundle-1 allocation score on node1: 0 ++container_color: rabbitmq-bundle-1 allocation score on node2: INFINITY ++container_color: rabbitmq-bundle-1 allocation score on node3: 0 ++container_color: rabbitmq-bundle-2 allocation score on node1: 0 ++container_color: rabbitmq-bundle-2 allocation score on node2: 0 ++container_color: rabbitmq-bundle-2 allocation score on node3: INFINITY ++container_color: rabbitmq-bundle-clone allocation score on node1: 0 ++container_color: rabbitmq-bundle-clone allocation score on node2: 0 ++container_color: rabbitmq-bundle-clone allocation score on node3: 0 ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on node1: INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on node2: 0 ++container_color: rabbitmq-bundle-docker-0 allocation score on node3: 0 ++container_color: rabbitmq-bundle-docker-1 allocation score on node1: 0 ++container_color: rabbitmq-bundle-docker-1 allocation score on node2: INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on node3: 0 ++container_color: rabbitmq-bundle-docker-2 allocation score on node1: 0 ++container_color: rabbitmq-bundle-docker-2 allocation score on node2: 0 ++container_color: rabbitmq-bundle-docker-2 allocation score on node3: INFINITY ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++container_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY ++container_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY ++galera:0 promotion score on galera-bundle-0: 100 ++galera:1 promotion score on galera-bundle-1: 100 ++galera:2 promotion score on galera-bundle-2: -1 ++native_color: galera-bundle-0 allocation score on node1: INFINITY ++native_color: galera-bundle-0 allocation score on node2: 0 ++native_color: galera-bundle-0 allocation score on node3: 0 ++native_color: galera-bundle-1 allocation score on node1: 0 ++native_color: galera-bundle-1 allocation score on node2: INFINITY ++native_color: galera-bundle-1 allocation score on node3: 0 ++native_color: galera-bundle-2 allocation score on node1: 0 ++native_color: galera-bundle-2 allocation score on node2: 0 ++native_color: galera-bundle-2 allocation score on node3: INFINITY ++native_color: galera-bundle-docker-0 allocation score on node1: INFINITY ++native_color: galera-bundle-docker-0 allocation score on node2: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on node3: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on node1: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on node2: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on node3: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on node1: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on node2: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on node3: -INFINITY ++native_color: galera:0 allocation score on galera-bundle-0: INFINITY ++native_color: galera:1 allocation score on galera-bundle-1: INFINITY ++native_color: galera:2 allocation score on galera-bundle-2: INFINITY ++native_color: rabbitmq-bundle-0 allocation score on node1: INFINITY ++native_color: rabbitmq-bundle-0 allocation score on node2: 0 ++native_color: rabbitmq-bundle-0 allocation score on node3: 0 ++native_color: rabbitmq-bundle-1 allocation score on node1: 0 ++native_color: rabbitmq-bundle-1 allocation score on node2: INFINITY ++native_color: rabbitmq-bundle-1 allocation score on node3: 0 ++native_color: rabbitmq-bundle-2 allocation score on node1: 0 ++native_color: rabbitmq-bundle-2 allocation score on node2: 0 ++native_color: rabbitmq-bundle-2 allocation score on node3: INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on node1: INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on node2: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on node3: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on node1: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on node2: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on node3: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on node1: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on node2: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on node3: -INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++native_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY ++native_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY ++native_color: stonith-fence_ipmilan-node1 allocation score on node1: -INFINITY ++native_color: stonith-fence_ipmilan-node1 allocation score on node2: INFINITY ++native_color: stonith-fence_ipmilan-node1 allocation score on node3: 0 ++native_color: stonith-fence_ipmilan-node2 allocation score on node1: 0 ++native_color: stonith-fence_ipmilan-node2 allocation score on node2: -INFINITY ++native_color: stonith-fence_ipmilan-node2 allocation score on node3: INFINITY ++native_color: stonith-fence_ipmilan-node3 allocation score on node1: 0 ++native_color: stonith-fence_ipmilan-node3 allocation score on node2: INFINITY ++native_color: stonith-fence_ipmilan-node3 allocation score on node3: -INFINITY +diff --git a/pengine/test10/guest-host-not-fenceable.summary b/pengine/test10/guest-host-not-fenceable.summary +new file mode 100644 +index 0000000..54a4d7b +--- /dev/null ++++ b/pengine/test10/guest-host-not-fenceable.summary +@@ -0,0 +1,87 @@ ++Using the original execution date of: 2019-08-26 04:52:42Z ++ ++Current cluster status: ++Node node2 (2): UNCLEAN (offline) ++Node node3 (3): UNCLEAN (offline) ++Online: [ node1 ] ++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 ] ++ ++ Docker container set: rabbitmq-bundle [192.168.122.139:8787/rhosp13/openstack-rabbitmq:pcmklatest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started node1 ++ rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): FAILED node2 (UNCLEAN) ++ rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): FAILED node3 (UNCLEAN) ++ Docker container set: galera-bundle [192.168.122.139:8787/rhosp13/openstack-mariadb:pcmklatest] ++ galera-bundle-0 (ocf::heartbeat:galera): FAILED Master node1 ++ galera-bundle-1 (ocf::heartbeat:galera): FAILED Master node2 (UNCLEAN) ++ galera-bundle-2 (ocf::heartbeat:galera): FAILED Master node3 (UNCLEAN) ++ stonith-fence_ipmilan-node1 (stonith:fence_ipmilan): Started node2 (UNCLEAN) ++ stonith-fence_ipmilan-node3 (stonith:fence_ipmilan): Started node2 (UNCLEAN) ++ stonith-fence_ipmilan-node2 (stonith:fence_ipmilan): Started node3 (UNCLEAN) ++ ++Transition Summary: ++ * Stop rabbitmq-bundle-docker-0 ( node1 ) due to no quorum ++ * Stop rabbitmq-bundle-0 ( node1 ) due to no quorum ++ * Stop rabbitmq:0 ( rabbitmq-bundle-0 ) due to no quorum ++ * Stop rabbitmq-bundle-docker-1 ( node2 ) due to node availability (blocked) ++ * Stop rabbitmq-bundle-1 ( node2 ) due to no quorum (blocked) ++ * Stop rabbitmq:1 ( rabbitmq-bundle-1 ) due to no quorum (blocked) ++ * Stop rabbitmq-bundle-docker-2 ( node3 ) due to node availability (blocked) ++ * Stop rabbitmq-bundle-2 ( node3 ) due to no quorum (blocked) ++ * Stop rabbitmq:2 ( rabbitmq-bundle-2 ) due to no quorum (blocked) ++ * Stop galera-bundle-docker-0 ( node1 ) due to no quorum ++ * Stop galera-bundle-0 ( node1 ) due to no quorum ++ * Stop galera:0 ( Master galera-bundle-0 ) due to no quorum ++ * Stop galera-bundle-docker-1 ( node2 ) due to node availability (blocked) ++ * Stop galera-bundle-1 ( node2 ) due to no quorum (blocked) ++ * Stop galera:1 ( Master galera-bundle-1 ) due to no quorum (blocked) ++ * Stop galera-bundle-docker-2 ( node3 ) due to node availability (blocked) ++ * Stop galera-bundle-2 ( node3 ) due to no quorum (blocked) ++ * Stop galera:2 ( Master galera-bundle-2 ) due to no quorum (blocked) ++ * Stop stonith-fence_ipmilan-node1 ( node2 ) due to node availability (blocked) ++ * Stop stonith-fence_ipmilan-node3 ( node2 ) due to no quorum (blocked) ++ * Stop stonith-fence_ipmilan-node2 ( node3 ) due to no quorum (blocked) ++ ++Executing cluster transition: ++ * Pseudo action: rabbitmq-bundle-clone_pre_notify_stop_0 ++ * Pseudo action: galera-bundle_demote_0 ++ * Pseudo action: rabbitmq-bundle_stop_0 ++ * Resource action: rabbitmq notify on rabbitmq-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_stop_0 ++ * Pseudo action: rabbitmq-bundle-clone_stop_0 ++ * Pseudo action: galera-bundle-master_demote_0 ++ * Resource action: rabbitmq stop on rabbitmq-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_stopped_0 ++ * Resource action: rabbitmq-bundle-0 stop on node1 ++ * Resource action: galera demote on galera-bundle-0 ++ * Pseudo action: galera-bundle-master_demoted_0 ++ * Pseudo action: galera-bundle_demoted_0 ++ * Pseudo action: galera-bundle_stop_0 ++ * Pseudo action: rabbitmq-bundle-clone_post_notify_stopped_0 ++ * Resource action: rabbitmq-bundle-docker-0 stop on node1 ++ * Pseudo action: galera-bundle-master_stop_0 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_stopped_0 ++ * Resource action: galera stop on galera-bundle-0 ++ * Pseudo action: galera-bundle-master_stopped_0 ++ * Resource action: galera-bundle-0 stop on node1 ++ * Pseudo action: rabbitmq-bundle_stopped_0 ++ * Resource action: galera-bundle-docker-0 stop on node1 ++ * Pseudo action: galera-bundle_stopped_0 ++Using the original execution date of: 2019-08-26 04:52:42Z ++ ++Revised cluster status: ++Node node2 (2): UNCLEAN (offline) ++Node node3 (3): UNCLEAN (offline) ++Online: [ node1 ] ++ ++ Docker container set: rabbitmq-bundle [192.168.122.139:8787/rhosp13/openstack-rabbitmq:pcmklatest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped ++ rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): FAILED node2 (UNCLEAN) ++ rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): FAILED node3 (UNCLEAN) ++ Docker container set: galera-bundle [192.168.122.139:8787/rhosp13/openstack-mariadb:pcmklatest] ++ galera-bundle-0 (ocf::heartbeat:galera): Stopped ++ galera-bundle-1 (ocf::heartbeat:galera): FAILED Master node2 (UNCLEAN) ++ galera-bundle-2 (ocf::heartbeat:galera): FAILED Master node3 (UNCLEAN) ++ stonith-fence_ipmilan-node1 (stonith:fence_ipmilan): Started node2 (UNCLEAN) ++ stonith-fence_ipmilan-node3 (stonith:fence_ipmilan): Started node2 (UNCLEAN) ++ stonith-fence_ipmilan-node2 (stonith:fence_ipmilan): Started node3 (UNCLEAN) ++ +diff --git a/pengine/test10/guest-host-not-fenceable.xml b/pengine/test10/guest-host-not-fenceable.xml +new file mode 100755 +index 0000000..a1ccdc8 +--- /dev/null ++++ b/pengine/test10/guest-host-not-fenceable.xml +@@ -0,0 +1,413 @@ ++<cib crm_feature_set="3.0.14" validate-with="pacemaker-2.10" epoch="71" num_updates="166" admin_epoch="0" cib-last-written="Wed Aug 21 12:00:45 2019" update-origin="node1" update-client="crm_resource" update-user="root" have-quorum="0" dc-uuid="1" no-quorum-panic="1" execution-date="1566795162"> ++ <configuration> ++ <crm_config> ++ <cluster_property_set id="cib-bootstrap-options"> ++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/> ++ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.19-8.el7_6.4-c3c624ea3d"/> ++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/> ++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/> ++ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/> ++ <nvpair id="cib-bootstrap-options-cluster-recheck-interval" name="cluster-recheck-interval" value="600s"/> ++ </cluster_property_set> ++ </crm_config> ++ <nodes> ++ <node id="1" uname="node1"> ++ <instance_attributes id="nodes-1"> ++ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/> ++ <nvpair id="nodes-1-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@node1"/> ++ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/> ++ </instance_attributes> ++ </node> ++ <node id="2" uname="node2"> ++ <instance_attributes id="nodes-2"> ++ <nvpair id="nodes-2-rabbitmq-role" name="rabbitmq-role" value="true"/> ++ <nvpair id="nodes-2-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@node2"/> ++ <nvpair id="nodes-2-galera-role" name="galera-role" value="true"/> ++ </instance_attributes> ++ </node> ++ <node id="3" uname="node3"> ++ <instance_attributes id="nodes-3"> ++ <nvpair id="nodes-3-rabbitmq-role" name="rabbitmq-role" value="true"/> ++ <nvpair id="nodes-3-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@node3"/> ++ <nvpair id="nodes-3-galera-role" name="galera-role" value="true"/> ++ </instance_attributes> ++ </node> ++ </nodes> ++ <resources> ++ <bundle id="rabbitmq-bundle"> ++ <docker image="192.168.122.139:8787/rhosp13/openstack-rabbitmq:pcmklatest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="3" run-command="/bin/bash /usr/local/bin/kolla_start"/> ++ <network control-port="3122"/> ++ <storage> ++ <storage-mapping id="rabbitmq-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/rabbitmq.json" target-dir="/var/lib/kolla/config_files/config.json"/> ++ <storage-mapping id="rabbitmq-cfg-data" options="ro" source-dir="/var/lib/config-data/puppet-generated/rabbitmq/" target-dir="/var/lib/kolla/config_files/src"/> ++ <storage-mapping id="rabbitmq-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/> ++ <storage-mapping id="rabbitmq-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/> ++ <storage-mapping id="rabbitmq-lib" options="rw" source-dir="/var/lib/rabbitmq" target-dir="/var/lib/rabbitmq"/> ++ <storage-mapping id="rabbitmq-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/> ++ <storage-mapping id="rabbitmq-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/> ++ <storage-mapping id="rabbitmq-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/> ++ <storage-mapping id="rabbitmq-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/> ++ <storage-mapping id="rabbitmq-log" options="rw" source-dir="/var/log/containers/rabbitmq" target-dir="/var/log/rabbitmq"/> ++ <storage-mapping id="rabbitmq-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/> ++ </storage> ++ <primitive class="ocf" id="rabbitmq" provider="heartbeat" type="rabbitmq-cluster"> ++ <instance_attributes id="rabbitmq-instance_attributes"> ++ <nvpair id="rabbitmq-instance_attributes-set_policy" name="set_policy" value="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/> ++ </instance_attributes> ++ <meta_attributes id="rabbitmq-meta_attributes"> ++ <nvpair id="rabbitmq-meta_attributes-container-attribute-target" name="container-attribute-target" value="host"/> ++ <nvpair id="rabbitmq-meta_attributes-notify" name="notify" value="true"/> ++ </meta_attributes> ++ <operations> ++ <op id="rabbitmq-monitor-interval-10s" interval="10s" name="monitor" timeout="40s"/> ++ <op id="rabbitmq-start-interval-0s" interval="0s" name="start" timeout="200s"/> ++ <op id="rabbitmq-stop-interval-0s" interval="0s" name="stop" timeout="200s"/> ++ </operations> ++ </primitive> ++ <meta_attributes id="rabbitmq-bundle-meta_attributes"/> ++ </bundle> ++ <bundle id="galera-bundle"> ++ <docker image="192.168.122.139:8787/rhosp13/openstack-mariadb:pcmklatest" masters="3" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="3" run-command="/bin/bash /usr/local/bin/kolla_start"/> ++ <network control-port="3123"/> ++ <storage> ++ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/> ++ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/puppet-generated/mysql/" target-dir="/var/lib/kolla/config_files/src"/> ++ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/> ++ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/> ++ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/> ++ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/> ++ <storage-mapping id="mysql-log" options="rw" source-dir="/var/log/containers/mysql" target-dir="/var/log/mysql"/> ++ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/> ++ </storage> ++ <primitive class="ocf" id="galera" provider="heartbeat" type="galera"> ++ <instance_attributes id="galera-instance_attributes"> ++ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/> ++ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="node1:node1.internalapi.localdomain;node2:node2.internalapi.localdomain;node3:node3.internalapi.localdomain"/> ++ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/> ++ <nvpair id="galera-instance_attributes-log" name="log" value="/var/log/mysql/mysqld.log"/> ++ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://node1.internalapi.localdomain,node2.internalapi.localdomain,node3.internalapi.localdomain"/> ++ </instance_attributes> ++ <meta_attributes id="galera-meta_attributes"> ++ <nvpair id="galera-meta_attributes-container-attribute-target" name="container-attribute-target" value="host"/> ++ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="3"/> ++ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/> ++ </meta_attributes> ++ <operations> ++ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120s"/> ++ <op id="galera-monitor-interval-20s" interval="20s" name="monitor" timeout="30s"/> ++ <op id="galera-monitor-interval-10s" interval="10s" name="monitor" role="Master" timeout="30s"/> ++ <op id="galera-monitor-interval-30s" interval="30s" name="monitor" role="Slave" timeout="30s"/> ++ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/> ++ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120s"/> ++ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120s"/> ++ </operations> ++ </primitive> ++ </bundle> ++ <primitive class="stonith" id="stonith-fence_ipmilan-node1" type="fence_ipmilan"> ++ <instance_attributes id="stonith-fence_ipmilan-node1-instance_attributes"> ++ <nvpair id="stonith-fence_ipmilan-node1-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="node1"/> ++ </instance_attributes> ++ <meta_attributes id="stonith-fence_ipmilan-node1-meta_attributes"> ++ <nvpair id="stonith-fence_ipmilan-node1-meta_attributes-failure-timeout" name="failure-timeout" value="600s"/> ++ <nvpair id="stonith-fence_ipmilan-node1-meta_attributes-migration-threshold" name="migration-threshold" value="5"/> ++ </meta_attributes> ++ <operations> ++ <op id="stonith-fence_ipmilan-node1-monitor-interval-60s" interval="60s" name="monitor"/> ++ </operations> ++ </primitive> ++ <primitive class="stonith" id="stonith-fence_ipmilan-node3" type="fence_ipmilan"> ++ <instance_attributes id="stonith-fence_ipmilan-node3-instance_attributes"> ++ <nvpair id="stonith-fence_ipmilan-node3-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="node3"/> ++ </instance_attributes> ++ <meta_attributes id="stonith-fence_ipmilan-node3-meta_attributes"> ++ <nvpair id="stonith-fence_ipmilan-node3-meta_attributes-failure-timeout" name="failure-timeout" value="600s"/> ++ <nvpair id="stonith-fence_ipmilan-node3-meta_attributes-migration-threshold" name="migration-threshold" value="5"/> ++ </meta_attributes> ++ <operations> ++ <op id="stonith-fence_ipmilan-node3-monitor-interval-60s" interval="60s" name="monitor"/> ++ </operations> ++ </primitive> ++ <primitive class="stonith" id="stonith-fence_ipmilan-node2" type="fence_ipmilan"> ++ <instance_attributes id="stonith-fence_ipmilan-node2-instance_attributes"> ++ <nvpair id="stonith-fence_ipmilan-node2-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="node2"/> ++ </instance_attributes> ++ <meta_attributes id="stonith-fence_ipmilan-node2-meta_attributes"> ++ <nvpair id="stonith-fence_ipmilan-node2-meta_attributes-failure-timeout" name="failure-timeout" value="600s"/> ++ <nvpair id="stonith-fence_ipmilan-node2-meta_attributes-migration-threshold" name="migration-threshold" value="5"/> ++ </meta_attributes> ++ <operations> ++ <op id="stonith-fence_ipmilan-node2-monitor-interval-60s" interval="60s" name="monitor"/> ++ </operations> ++ </primitive> ++ </resources> ++ <constraints> ++ <rsc_location id="location-rabbitmq-bundle" resource-discovery="exclusive" rsc="rabbitmq-bundle"> ++ <rule id="location-rabbitmq-bundle-rule" score="0"> ++ <expression attribute="rabbitmq-role" id="location-rabbitmq-bundle-rule-expr" operation="eq" value="true"/> ++ </rule> ++ </rsc_location> ++ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle"> ++ <rule id="location-galera-bundle-rule" score="0"> ++ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/> ++ </rule> ++ </rsc_location> ++ <rsc_location id="location-stonith-fence_ipmilan-node1-node1--INFINITY" node="node1" rsc="stonith-fence_ipmilan-node1" score="-INFINITY"/> ++ <rsc_location id="location-stonith-fence_ipmilan-node3-node3--INFINITY" node="node3" rsc="stonith-fence_ipmilan-node3" score="-INFINITY"/> ++ <rsc_location id="location-stonith-fence_ipmilan-node2-node2--INFINITY" node="node2" rsc="stonith-fence_ipmilan-node2" score="-INFINITY"/> ++ </constraints> ++ <rsc_defaults> ++ <meta_attributes id="rsc_defaults-options"> ++ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/> ++ </meta_attributes> ++ </rsc_defaults> ++ </configuration> ++ <status> ++ <node_state id="1" uname="node1" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member"> ++ <transient_attributes id="1"> ++ <instance_attributes id="status-1"> ++ <nvpair id="status-1-master-galera" name="master-galera" value="100"/> ++ <nvpair id="status-1-rmq-node-attr-rabbitmq" name="rmq-node-attr-rabbitmq" value="rabbit@node1"/> ++ </instance_attributes> ++ </transient_attributes> ++ <lrm id="1"> ++ <lrm_resources> ++ <lrm_resource id="rabbitmq-bundle-1" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-1"> ++ <lrm_rsc_op id="rabbitmq-bundle-1_last_0" operation_key="rabbitmq-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="26:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;26:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="0" queue-time="0" op-digest="4e9836c1fe6ca784363329f38f1a6bab" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-2" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-2"> ++ <lrm_rsc_op id="rabbitmq-bundle-2_last_0" operation_key="rabbitmq-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="28:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;28:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="0" queue-time="0" op-digest="6c8e8fc40a3a8bc0990cb4086a91cb5a" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="stonith-fence_ipmilan-node1" type="fence_ipmilan" class="stonith"> ++ <lrm_rsc_op id="stonith-fence_ipmilan-node1_last_0" operation_key="stonith-fence_ipmilan-node1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="42:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;42:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1566435865" last-rc-change="1566435865" exec-time="139" queue-time="0" op-digest="40ba273e494269d4d7bded2368059e8d" op-secure-params=" password passwd " op-secure-digest="3ffa7c4b1189e29b87161a1808ba7d9f"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="46:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;46:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="75" rc-code="0" op-status="0" interval="0" last-run="1566435865" last-rc-change="1566435865" exec-time="2053" queue-time="0" op-digest="14ba0643d7f91ddb1789cd6c29f9ac3f"/> ++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_monitor_60000" operation_key="rabbitmq-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="47:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;47:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="80" rc-code="0" op-status="0" interval="60000" last-rc-change="1566435867" exec-time="161" queue-time="0" op-digest="f18bc65b8186d36c889d9d67beeef1b3"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-docker-1" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="rabbitmq-bundle-docker-1_last_0" operation_key="rabbitmq-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="25:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;25:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="9" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="196" queue-time="0" op-digest="6611df38a8c054188fb3906bac2a4ac6"/> ++ </lrm_resource> ++ <lrm_resource id="stonith-fence_ipmilan-node2" type="fence_ipmilan" class="stonith"> ++ <lrm_rsc_op id="stonith-fence_ipmilan-node2_last_0" operation_key="stonith-fence_ipmilan-node2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="44:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;44:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="69" rc-code="7" op-status="0" interval="0" last-run="1566435865" last-rc-change="1566435865" exec-time="0" queue-time="0" op-digest="7252381349b2722d90a4ce9fba7ba3b5" op-secure-params=" password passwd " op-secure-digest="5265a046a980e16fb93d35f07f19a6d4"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-docker-0" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="80:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;80:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="74" rc-code="0" op-status="0" interval="0" last-run="1566435865" last-rc-change="1566435865" exec-time="1784" queue-time="0" op-digest="c9e60e09c1b2adcdac0169e69cbeb76e"/> ++ <lrm_rsc_op id="galera-bundle-docker-0_monitor_60000" operation_key="galera-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="81:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;81:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="78" rc-code="0" op-status="0" interval="60000" last-rc-change="1566435867" exec-time="235" queue-time="0" op-digest="5826a0cfeef03a64959f74a25e81bbc4"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-1" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-1"> ++ <lrm_rsc_op id="galera-bundle-1_last_0" operation_key="galera-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="32:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;32:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="0" queue-time="0" op-digest="5902e2bf7cfd38a4e959ce335356dc1f" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-2" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-2"> ++ <lrm_rsc_op id="galera-bundle-2_last_0" operation_key="galera-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="34:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;34:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="0" queue-time="0" op-digest="f724287411b756a9cac4dba67d3bddf6" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-docker-2" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="rabbitmq-bundle-docker-2_last_0" operation_key="rabbitmq-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="27:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;27:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="14" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="166" queue-time="0" op-digest="7ade29d4352e8007c537badef8a2e0b0"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-docker-1" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="galera-bundle-docker-1_last_0" operation_key="galera-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="31:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;31:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="23" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="155" queue-time="0" op-digest="c6f303ef421236554d0dea3151b97cb3"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-docker-2" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="galera-bundle-docker-2_last_0" operation_key="galera-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="33:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;33:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="28" rc-code="7" op-status="0" interval="0" last-run="1566435864" last-rc-change="1566435864" exec-time="174" queue-time="0" op-digest="bb5745d5e3a6dedaf2449265ce05625b"/> ++ </lrm_resource> ++ <lrm_resource id="stonith-fence_ipmilan-node3" type="fence_ipmilan" class="stonith"> ++ <lrm_rsc_op id="stonith-fence_ipmilan-node3_last_0" operation_key="stonith-fence_ipmilan-node3_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="43:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:7;43:889:7:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="65" rc-code="7" op-status="0" interval="0" last-run="1566435865" last-rc-change="1566435865" exec-time="0" queue-time="0" op-digest="094b99e257ba32c2b718b61813e80256" op-secure-params=" password passwd " op-secure-digest="765c80b4e1ed30f2ef368a8296f7481c"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-0" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-0"> ++ <lrm_rsc_op id="galera-bundle-0_last_0" operation_key="galera-bundle-0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="82:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;82:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1566435867" last-rc-change="1566435867" exec-time="0" queue-time="0" op-digest="8193f24828986601946578a7993cbb74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ <lrm_rsc_op id="galera-bundle-0_monitor_60000" operation_key="galera-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="67:890:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;67:890:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="10" rc-code="0" op-status="0" interval="60000" last-rc-change="1566435871" exec-time="0" queue-time="0" op-digest="aa427f07df30a17686d41e30145bf557"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0"> ++ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="48:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;48:889:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="8" rc-code="0" op-status="0" interval="0" last-run="1566435867" last-rc-change="1566435867" exec-time="0" queue-time="0" op-digest="2c6afa7ac6b48957892b7238a3270a5d" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ <lrm_rsc_op id="rabbitmq-bundle-0_monitor_60000" operation_key="rabbitmq-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="32:890:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;32:890:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="9" rc-code="0" op-status="0" interval="60000" last-rc-change="1566435871" exec-time="0" queue-time="0" op-digest="deffe42b0e1f26d58665d8c3a846a8eb"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state id="2" uname="node2" in_ccm="false" crmd="offline" crm-debug-origin="do_state_transition" join="down" expected="down"> ++ <transient_attributes id="2"> ++ <instance_attributes id="status-2"> ++ <nvpair id="status-2-master-galera" name="master-galera" value="100"/> ++ <nvpair id="status-2-rmq-node-attr-rabbitmq" name="rmq-node-attr-rabbitmq" value="rabbit@node2"/> ++ </instance_attributes> ++ </transient_attributes> ++ <lrm id="2"> ++ <lrm_resources> ++ <lrm_resource id="rabbitmq-bundle-1" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-1"> ++ <lrm_rsc_op id="rabbitmq-bundle-1_last_0" operation_key="rabbitmq-bundle-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="27:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;27:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="12" rc-code="0" op-status="0" interval="0" last-run="1566356445" last-rc-change="1566356445" exec-time="0" queue-time="0" op-digest="4e9836c1fe6ca784363329f38f1a6bab" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ <lrm_rsc_op id="rabbitmq-bundle-1_monitor_60000" operation_key="rabbitmq-bundle-1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="36:754:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;36:754:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="13" rc-code="0" op-status="0" interval="60000" last-rc-change="1566356446" exec-time="0" queue-time="0" op-digest="7fe12a754b9debfddd5a529c95d4ea6c"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-2" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-2"> ++ <lrm_rsc_op id="rabbitmq-bundle-2_last_0" operation_key="rabbitmq-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="28:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;28:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="0" queue-time="0" op-digest="6c8e8fc40a3a8bc0990cb4086a91cb5a" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-docker-0" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="29:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;29:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="19" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="204" queue-time="0" op-digest="c9e60e09c1b2adcdac0169e69cbeb76e"/> ++ </lrm_resource> ++ <lrm_resource id="stonith-fence_ipmilan-node1" type="fence_ipmilan" class="stonith"> ++ <lrm_rsc_op id="stonith-fence_ipmilan-node1_last_0" operation_key="stonith-fence_ipmilan-node1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="125:17:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;125:17:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="82" rc-code="0" op-status="0" interval="0" last-run="1565266314" last-rc-change="1565266314" exec-time="322" queue-time="0" op-digest="40ba273e494269d4d7bded2368059e8d" op-secure-params=" password passwd " op-secure-digest="3ffa7c4b1189e29b87161a1808ba7d9f"/> ++ <lrm_rsc_op id="stonith-fence_ipmilan-node1_monitor_60000" operation_key="stonith-fence_ipmilan-node1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="126:17:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;126:17:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="83" rc-code="0" op-status="0" interval="60000" last-rc-change="1565266315" exec-time="239" queue-time="0" op-digest="865620c5b61a242892a200a6a0e1cd35" op-secure-params=" password passwd " op-secure-digest="3ffa7c4b1189e29b87161a1808ba7d9f"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="23:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;23:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="215" queue-time="0" op-digest="14ba0643d7f91ddb1789cd6c29f9ac3f"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-docker-1" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="rabbitmq-bundle-docker-1_last_0" operation_key="rabbitmq-bundle-docker-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="25:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;25:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="94" rc-code="0" op-status="0" interval="0" last-run="1566356445" last-rc-change="1566356445" exec-time="598" queue-time="0" op-digest="6611df38a8c054188fb3906bac2a4ac6"/> ++ <lrm_rsc_op id="rabbitmq-bundle-docker-1_monitor_60000" operation_key="rabbitmq-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="26:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;26:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="95" rc-code="0" op-status="0" interval="60000" last-rc-change="1566356445" exec-time="163" queue-time="0" op-digest="256b8a21685f8d8237dfff64197a60b0"/> ++ </lrm_resource> ++ <lrm_resource id="stonith-fence_ipmilan-node2" type="fence_ipmilan" class="stonith"> ++ <lrm_rsc_op id="stonith-fence_ipmilan-node2_last_0" operation_key="stonith-fence_ipmilan-node2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="44:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;44:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="69" rc-code="7" op-status="0" interval="0" last-run="1565265997" last-rc-change="1565265997" exec-time="0" queue-time="0" op-digest="7252381349b2722d90a4ce9fba7ba3b5" op-secure-params=" password passwd " op-secure-digest="5265a046a980e16fb93d35f07f19a6d4"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-0" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-0"> ++ <lrm_rsc_op id="galera-bundle-0_last_0" operation_key="galera-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="30:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;30:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="0" queue-time="0" op-digest="8193f24828986601946578a7993cbb74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-docker-1" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="galera-bundle-docker-1_last_0" operation_key="galera-bundle-docker-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="84:12:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;84:12:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="76" rc-code="0" op-status="0" interval="0" last-run="1565265997" last-rc-change="1565265997" exec-time="2055" queue-time="0" op-digest="c6f303ef421236554d0dea3151b97cb3"/> ++ <lrm_rsc_op id="galera-bundle-docker-1_monitor_60000" operation_key="galera-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="85:12:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;85:12:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="80" rc-code="0" op-status="0" interval="60000" last-rc-change="1565265999" exec-time="240" queue-time="0" op-digest="b6b5570d0c79fe1f2c57f97e38d0c454"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-2" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-2"> ++ <lrm_rsc_op id="galera-bundle-2_last_0" operation_key="galera-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="34:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;34:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="0" queue-time="0" op-digest="f724287411b756a9cac4dba67d3bddf6" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-docker-2" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="galera-bundle-docker-2_last_0" operation_key="galera-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="33:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;33:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="28" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="161" queue-time="0" op-digest="bb5745d5e3a6dedaf2449265ce05625b"/> ++ </lrm_resource> ++ <lrm_resource id="stonith-fence_ipmilan-node3" type="fence_ipmilan" class="stonith"> ++ <lrm_rsc_op id="stonith-fence_ipmilan-node3_last_0" operation_key="stonith-fence_ipmilan-node3_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="117:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;117:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="88" rc-code="0" op-status="0" interval="0" last-run="1565925136" last-rc-change="1565925136" exec-time="196" queue-time="0" op-digest="094b99e257ba32c2b718b61813e80256" op-secure-params=" password passwd " op-secure-digest="765c80b4e1ed30f2ef368a8296f7481c"/> ++ <lrm_rsc_op id="stonith-fence_ipmilan-node3_monitor_60000" operation_key="stonith-fence_ipmilan-node3_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="118:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;118:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="89" rc-code="0" op-status="0" interval="60000" last-rc-change="1565925136" exec-time="197" queue-time="0" op-digest="14dda17b1320f5f4ce99f062334a12da" op-secure-params=" password passwd " op-secure-digest="765c80b4e1ed30f2ef368a8296f7481c"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-docker-2" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="rabbitmq-bundle-docker-2_last_0" operation_key="rabbitmq-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="27:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;27:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="14" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="171" queue-time="0" op-digest="7ade29d4352e8007c537badef8a2e0b0"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-1" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-1"> ++ <lrm_rsc_op id="galera-bundle-1_last_0" operation_key="galera-bundle-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="86:12:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;86:12:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="8" rc-code="0" op-status="0" interval="0" last-run="1565265999" last-rc-change="1565265999" exec-time="0" queue-time="0" op-digest="5902e2bf7cfd38a4e959ce335356dc1f" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ <lrm_rsc_op id="galera-bundle-1_monitor_60000" operation_key="galera-bundle-1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="71:13:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;71:13:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="10" rc-code="0" op-status="0" interval="60000" last-rc-change="1565266003" exec-time="0" queue-time="0" op-digest="9cd0e6ae49720131ba1b94817d9df3c7"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0"> ++ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="24:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:7;24:12:7:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1565265995" last-rc-change="1565265995" exec-time="0" queue-time="0" op-digest="2c6afa7ac6b48957892b7238a3270a5d" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state id="3" uname="node3" in_ccm="false" crmd="offline" crm-debug-origin="do_state_transition" join="down" expected="member"> ++ <lrm id="3"> ++ <lrm_resources> ++ <lrm_resource id="rabbitmq-bundle-1" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-1"> ++ <lrm_rsc_op id="rabbitmq-bundle-1_last_0" operation_key="rabbitmq-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="26:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;26:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="0" queue-time="0" op-digest="4e9836c1fe6ca784363329f38f1a6bab" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-2" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-2"> ++ <lrm_rsc_op id="rabbitmq-bundle-2_last_0" operation_key="rabbitmq-bundle-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="31:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;31:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="12" rc-code="0" op-status="0" interval="0" last-run="1566356445" last-rc-change="1566356445" exec-time="0" queue-time="0" op-digest="6c8e8fc40a3a8bc0990cb4086a91cb5a" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ <lrm_rsc_op id="rabbitmq-bundle-2_monitor_60000" operation_key="rabbitmq-bundle-2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="41:754:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;41:754:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="13" rc-code="0" op-status="0" interval="60000" last-rc-change="1566356446" exec-time="0" queue-time="0" op-digest="2633a5ceafe55cf96d15c4984e9cb635"/> ++ </lrm_resource> ++ <lrm_resource id="stonith-fence_ipmilan-node1" type="fence_ipmilan" class="stonith"> ++ <lrm_rsc_op id="stonith-fence_ipmilan-node1_last_0" operation_key="stonith-fence_ipmilan-node1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="42:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;42:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1565267341" last-rc-change="1565267341" exec-time="143" queue-time="1" op-digest="40ba273e494269d4d7bded2368059e8d" op-secure-params=" password passwd " op-secure-digest="3ffa7c4b1189e29b87161a1808ba7d9f"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="23:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;23:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="195" queue-time="0" op-digest="14ba0643d7f91ddb1789cd6c29f9ac3f"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-docker-1" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="rabbitmq-bundle-docker-1_last_0" operation_key="rabbitmq-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="25:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;25:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="10" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="176" queue-time="0" op-digest="6611df38a8c054188fb3906bac2a4ac6"/> ++ </lrm_resource> ++ <lrm_resource id="stonith-fence_ipmilan-node2" type="fence_ipmilan" class="stonith"> ++ <lrm_rsc_op id="stonith-fence_ipmilan-node2_last_0" operation_key="stonith-fence_ipmilan-node2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="120:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;120:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="82" rc-code="0" op-status="0" interval="0" last-run="1565925136" last-rc-change="1565925136" exec-time="251" queue-time="1" op-digest="7252381349b2722d90a4ce9fba7ba3b5" op-secure-params=" password passwd " op-secure-digest="5265a046a980e16fb93d35f07f19a6d4"/> ++ <lrm_rsc_op id="stonith-fence_ipmilan-node2_monitor_60000" operation_key="stonith-fence_ipmilan-node2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="121:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;121:1:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="83" rc-code="0" op-status="0" interval="60000" last-rc-change="1565925136" exec-time="194" queue-time="0" op-digest="8fa93fafac88b73db2fde1123f0fac16" op-secure-params=" password passwd " op-secure-digest="5265a046a980e16fb93d35f07f19a6d4"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-0" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-0"> ++ <lrm_rsc_op id="galera-bundle-0_last_0" operation_key="galera-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="30:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;30:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="0" queue-time="0" op-digest="8193f24828986601946578a7993cbb74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-1" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-1"> ++ <lrm_rsc_op id="galera-bundle-1_last_0" operation_key="galera-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="32:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;32:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="0" queue-time="0" op-digest="5902e2bf7cfd38a4e959ce335356dc1f" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-docker-2" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="galera-bundle-docker-2_last_0" operation_key="galera-bundle-docker-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="88:12:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:0;88:12:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="76" rc-code="0" op-status="0" interval="0" last-run="1565267341" last-rc-change="1565267341" exec-time="1505" queue-time="0" op-digest="bb5745d5e3a6dedaf2449265ce05625b"/> ++ <lrm_rsc_op id="galera-bundle-docker-2_monitor_60000" operation_key="galera-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="89:12:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:0;89:12:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="78" rc-code="0" op-status="0" interval="60000" last-rc-change="1565267343" exec-time="255" queue-time="0" op-digest="382bc0d634f7dc834b2e7106a1f740f4"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-docker-0" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="29:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;29:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="19" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="163" queue-time="0" op-digest="c9e60e09c1b2adcdac0169e69cbeb76e"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-docker-1" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="galera-bundle-docker-1_last_0" operation_key="galera-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="31:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;31:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="24" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="205" queue-time="0" op-digest="c6f303ef421236554d0dea3151b97cb3"/> ++ </lrm_resource> ++ <lrm_resource id="galera-bundle-2" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-2"> ++ <lrm_rsc_op id="galera-bundle-2_last_0" operation_key="galera-bundle-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="90:12:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:0;90:12:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="7" rc-code="0" op-status="0" interval="0" last-run="1565267343" last-rc-change="1565267343" exec-time="0" queue-time="0" op-digest="f724287411b756a9cac4dba67d3bddf6" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ <lrm_rsc_op id="galera-bundle-2_monitor_60000" operation_key="galera-bundle-2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="75:13:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:0;75:13:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="10" rc-code="0" op-status="0" interval="60000" last-rc-change="1565267347" exec-time="0" queue-time="0" op-digest="d7f8ff67dde73741c39fc99235612a69"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-docker-2" type="docker" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="rabbitmq-bundle-docker-2_last_0" operation_key="rabbitmq-bundle-docker-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="29:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;29:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="90" rc-code="0" op-status="0" interval="0" last-run="1566356445" last-rc-change="1566356445" exec-time="589" queue-time="0" op-digest="7ade29d4352e8007c537badef8a2e0b0"/> ++ <lrm_rsc_op id="rabbitmq-bundle-docker-2_monitor_60000" operation_key="rabbitmq-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="30:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;30:753:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="91" rc-code="0" op-status="0" interval="60000" last-rc-change="1566356445" exec-time="159" queue-time="0" op-digest="43e4544c25234613205b2a31701bbaa5"/> ++ </lrm_resource> ++ <lrm_resource id="stonith-fence_ipmilan-node3" type="fence_ipmilan" class="stonith"> ++ <lrm_rsc_op id="stonith-fence_ipmilan-node3_last_0" operation_key="stonith-fence_ipmilan-node3_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="43:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;43:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="65" rc-code="7" op-status="0" interval="0" last-run="1565267341" last-rc-change="1565267341" exec-time="0" queue-time="0" op-digest="094b99e257ba32c2b718b61813e80256" op-secure-params=" password passwd " op-secure-digest="765c80b4e1ed30f2ef368a8296f7481c"/> ++ </lrm_resource> ++ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0"> ++ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="24:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:7;24:12:7:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1565267340" last-rc-change="1565267340" exec-time="0" queue-time="0" op-digest="2c6afa7ac6b48957892b7238a3270a5d" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state remote_node="true" id="rabbitmq-bundle-2" uname="rabbitmq-bundle-2" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0"> ++ <lrm id="rabbitmq-bundle-2"> ++ <lrm_resources> ++ <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="48:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;48:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1566356483" last-rc-change="1566356483" exec-time="21171" queue-time="0" op-digest="780d433233eb4f94c1a151623d002e84"/> ++ <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="49:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;49:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node3" call-id="32" rc-code="0" op-status="0" interval="10000" last-rc-change="1566356505" exec-time="6280" queue-time="1" op-digest="6b46cdf9111345cbd0460b2540d3b2c7"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state remote_node="true" id="rabbitmq-bundle-1" uname="rabbitmq-bundle-1" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0"> ++ <lrm id="rabbitmq-bundle-1"> ++ <lrm_resources> ++ <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="46:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;46:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1566356462" last-rc-change="1566356462" exec-time="20783" queue-time="0" op-digest="780d433233eb4f94c1a151623d002e84"/> ++ <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="47:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;47:755:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node2" call-id="32" rc-code="0" op-status="0" interval="10000" last-rc-change="1566356505" exec-time="6246" queue-time="0" op-digest="6b46cdf9111345cbd0460b2540d3b2c7"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state remote_node="true" id="rabbitmq-bundle-0" uname="rabbitmq-bundle-0" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0"> ++ <lrm id="rabbitmq-bundle-0"> ++ <lrm_resources> ++ <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="49:890:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;49:890:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1566435880" last-rc-change="1566435880" exec-time="24518" queue-time="0" op-digest="780d433233eb4f94c1a151623d002e84"/> ++ <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="46:891:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;46:891:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="32" rc-code="0" op-status="0" interval="10000" last-rc-change="1566435907" exec-time="6252" queue-time="0" op-digest="6b46cdf9111345cbd0460b2540d3b2c7"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state remote_node="true" id="galera-bundle-2" uname="galera-bundle-2" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0"> ++ <lrm id="galera-bundle-2"> ++ <lrm_resources> ++ <lrm_resource id="galera" type="galera" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="galera_last_0" operation_key="galera_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="88:14:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:0;88:14:0:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1565267357" last-rc-change="1565267357" exec-time="38152" queue-time="0" op-digest="1b6366d05865bce625517a6aaed95684" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> ++ <lrm_rsc_op id="galera_monitor_10000" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="91:15:8:0bfc4f89-f653-4db4-9512-87ae7f0afc87" transition-magic="0:8;91:15:8:0bfc4f89-f653-4db4-9512-87ae7f0afc87" exit-reason="" on_node="node3" call-id="66" rc-code="8" op-status="0" interval="10000" last-rc-change="1565267396" exec-time="1009" queue-time="0" op-digest="29e39b115884dbdd932fbce6501ece1b" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ <transient_attributes id="galera-bundle-2"> ++ <instance_attributes id="status-galera-bundle-2"/> ++ </transient_attributes> ++ </node_state> ++ <node_state remote_node="true" id="galera-bundle-1" uname="galera-bundle-1" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0"> ++ <lrm id="galera-bundle-1"> ++ <lrm_resources> ++ <lrm_resource id="galera" type="galera" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="galera_last_0" operation_key="galera_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="88:14:0:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:0;88:14:0:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1565266015" last-rc-change="1565266015" exec-time="27052" queue-time="0" op-digest="1b6366d05865bce625517a6aaed95684" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> ++ <lrm_rsc_op id="galera_monitor_10000" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="86:15:8:e9b2624a-a468-4541-8cb4-526ee5466e26" transition-magic="0:8;86:15:8:e9b2624a-a468-4541-8cb4-526ee5466e26" exit-reason="" on_node="node2" call-id="66" rc-code="8" op-status="0" interval="10000" last-rc-change="1565266043" exec-time="961" queue-time="0" op-digest="29e39b115884dbdd932fbce6501ece1b" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ <transient_attributes id="galera-bundle-1"> ++ <instance_attributes id="status-galera-bundle-1"/> ++ </transient_attributes> ++ </node_state> ++ <node_state remote_node="true" id="galera-bundle-0" uname="galera-bundle-0" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0"> ++ <lrm id="galera-bundle-0"> ++ <lrm_resources> ++ <lrm_resource id="galera" type="galera" class="ocf" provider="heartbeat"> ++ <lrm_rsc_op id="galera_last_0" operation_key="galera_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="89:891:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:0;89:891:0:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1566435907" last-rc-change="1566435907" exec-time="10647" queue-time="0" op-digest="1b6366d05865bce625517a6aaed95684" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> ++ <lrm_rsc_op id="galera_monitor_10000" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="83:892:8:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:8;83:892:8:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="" on_node="node1" call-id="66" rc-code="8" op-status="0" interval="10000" last-rc-change="1566435919" exec-time="1059" queue-time="0" op-digest="29e39b115884dbdd932fbce6501ece1b" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> ++ <lrm_rsc_op id="galera_last_failure_0" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="83:892:8:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" transition-magic="0:1;83:892:8:5da9c0f3-4bc1-4a60-b177-1bca293a6d60" exit-reason="local node <node1> is started, but not in primary mode. Unknown state." on_node="node1" call-id="66" rc-code="1" op-status="0" interval="10000" last-rc-change="1566795150" exec-time="0" queue-time="0" op-digest="29e39b115884dbdd932fbce6501ece1b" op-secure-params=" user " op-secure-digest="1b6366d05865bce625517a6aaed95684"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ </status> ++</cib> +-- +1.8.3.1 + diff --git a/SOURCES/05-status-deletion.patch b/SOURCES/05-status-deletion.patch new file mode 100644 index 0000000..8ba7c83 --- /dev/null +++ b/SOURCES/05-status-deletion.patch @@ -0,0 +1,1856 @@ +From 11e8a3b9c8e35301b197724658ec2d243aec3336 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 22 Nov 2019 16:39:54 -0600 +Subject: [PATCH 01/11] Refactor: controller: rename struct recurring_op_s to + active_op_t + +... because it holds both recurring and pending non-recurring actions, +and the name was confusing +--- + crmd/crmd_lrm.h | 8 ++++---- + crmd/lrm.c | 18 +++++++++--------- + crmd/lrm_state.c | 4 ++-- + 3 files changed, 15 insertions(+), 15 deletions(-) + +diff --git a/crmd/crmd_lrm.h b/crmd/crmd_lrm.h +index 0870817..7c6a3c4 100644 +--- a/crmd/crmd_lrm.h ++++ b/crmd/crmd_lrm.h +@@ -42,8 +42,8 @@ typedef struct resource_history_s { + + void history_free(gpointer data); + +-/* TODO - Replace this with lrmd_event_data_t */ +-struct recurring_op_s { ++// In-flight action (recurring or pending) ++typedef struct active_op_s { + int call_id; + int interval; + gboolean remove; +@@ -54,7 +54,7 @@ struct recurring_op_s { + char *op_key; + char *user_data; + GHashTable *params; +-}; ++} active_op_t; + + typedef struct lrm_state_s { + const char *node_name; +@@ -171,4 +171,4 @@ void remote_ra_process_maintenance_nodes(xmlNode *xml); + gboolean remote_ra_controlling_guest(lrm_state_t * lrm_state); + + void process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, +- struct recurring_op_s *pending, xmlNode *action_xml); ++ active_op_t *pending, xmlNode *action_xml); +diff --git a/crmd/lrm.c b/crmd/lrm.c +index 437840f..e459465 100644 +--- a/crmd/lrm.c ++++ b/crmd/lrm.c +@@ -400,7 +400,7 @@ lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, + GHashTableIter gIter; + const char *key = NULL; + rsc_history_t *entry = NULL; +- struct recurring_op_s *pending = NULL; ++ active_op_t *pending = NULL; + + crm_debug("Checking for active resources before exit"); + +@@ -917,7 +917,7 @@ static gboolean + lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data) + { + const char *rsc = user_data; +- struct recurring_op_s *pending = value; ++ active_op_t *pending = value; + + if (crm_str_eq(rsc, pending->rsc_id, TRUE)) { + crm_info("Removing op %s:%d for deleted resource %s", +@@ -1148,7 +1148,7 @@ cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, + { + int rc = pcmk_ok; + char *local_key = NULL; +- struct recurring_op_s *pending = NULL; ++ active_op_t *pending = NULL; + + CRM_CHECK(op != 0, return FALSE); + CRM_CHECK(rsc_id != NULL, return FALSE); +@@ -1213,7 +1213,7 @@ cancel_action_by_key(gpointer key, gpointer value, gpointer user_data) + { + gboolean remove = FALSE; + struct cancel_data *data = user_data; +- struct recurring_op_s *op = (struct recurring_op_s *)value; ++ active_op_t *op = value; + + if (crm_str_eq(op->op_key, data->key, TRUE)) { + data->done = TRUE; +@@ -2104,7 +2104,7 @@ stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data) + { + gboolean remove = FALSE; + struct stop_recurring_action_s *event = user_data; +- struct recurring_op_s *op = (struct recurring_op_s *)value; ++ active_op_t *op = value; + + if (op->interval != 0 && crm_str_eq(op->rsc_id, event->rsc->id, TRUE)) { + crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key); +@@ -2119,7 +2119,7 @@ stop_recurring_actions(gpointer key, gpointer value, gpointer user_data) + { + gboolean remove = FALSE; + lrm_state_t *lrm_state = user_data; +- struct recurring_op_s *op = (struct recurring_op_s *)value; ++ active_op_t *op = value; + + if (op->interval != 0) { + crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, key); +@@ -2294,9 +2294,9 @@ do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *operat + * for them to complete during shutdown + */ + char *call_id_s = make_stop_id(rsc->id, call_id); +- struct recurring_op_s *pending = NULL; ++ active_op_t *pending = NULL; + +- pending = calloc(1, sizeof(struct recurring_op_s)); ++ pending = calloc(1, sizeof(active_op_t)); + crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s); + + pending->call_id = call_id; +@@ -2517,7 +2517,7 @@ did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id, + + void + process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, +- struct recurring_op_s *pending, xmlNode *action_xml) ++ active_op_t *pending, xmlNode *action_xml) + { + char *op_id = NULL; + char *op_key = NULL; +diff --git a/crmd/lrm_state.c b/crmd/lrm_state.c +index 8d07ef5..0f39c3d 100644 +--- a/crmd/lrm_state.c ++++ b/crmd/lrm_state.c +@@ -58,7 +58,7 @@ free_deletion_op(gpointer value) + static void + free_recurring_op(gpointer value) + { +- struct recurring_op_s *op = (struct recurring_op_s *)value; ++ active_op_t *op = value; + + free(op->user_data); + free(op->rsc_id); +@@ -75,7 +75,7 @@ fail_pending_op(gpointer key, gpointer value, gpointer user_data) + { + lrmd_event_data_t event = { 0, }; + lrm_state_t *lrm_state = user_data; +- struct recurring_op_s *op = (struct recurring_op_s *)value; ++ active_op_t *op = value; + + crm_trace("Pre-emptively failing %s_%s_%d on %s (call=%s, %s)", + op->rsc_id, op->op_type, op->interval, +-- +1.8.3.1 + + +From 9795f5401957563de2307f94c393dc83ce41d3d1 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 22 Nov 2019 16:45:31 -0600 +Subject: [PATCH 02/11] Refactor: controller: convert active_op_t booleans to + bitmask + +--- + crmd/crmd_lrm.h | 8 ++++++-- + crmd/lrm.c | 11 +++++------ + 2 files changed, 11 insertions(+), 8 deletions(-) + +diff --git a/crmd/crmd_lrm.h b/crmd/crmd_lrm.h +index 7c6a3c4..eb27d84 100644 +--- a/crmd/crmd_lrm.h ++++ b/crmd/crmd_lrm.h +@@ -42,12 +42,16 @@ typedef struct resource_history_s { + + void history_free(gpointer data); + ++enum active_op_e { ++ active_op_remove = (1 << 0), ++ active_op_cancelled = (1 << 1), ++}; ++ + // In-flight action (recurring or pending) + typedef struct active_op_s { + int call_id; + int interval; +- gboolean remove; +- gboolean cancelled; ++ uint32_t flags; // bitmask of active_op_e + unsigned int start_time; + char *rsc_id; + char *op_type; +diff --git a/crmd/lrm.c b/crmd/lrm.c +index e459465..2ae0d85 100644 +--- a/crmd/lrm.c ++++ b/crmd/lrm.c +@@ -1159,18 +1159,17 @@ cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, + pending = g_hash_table_lookup(lrm_state->pending_ops, key); + + if (pending) { +- if (remove && pending->remove == FALSE) { +- pending->remove = TRUE; ++ if (remove && is_not_set(pending->flags, active_op_remove)) { ++ set_bit(pending->flags, active_op_remove); + crm_debug("Scheduling %s for removal", key); + } + +- if (pending->cancelled) { ++ if (is_set(pending->flags, active_op_cancelled)) { + crm_debug("Operation %s already cancelled", key); + free(local_key); + return FALSE; + } +- +- pending->cancelled = TRUE; ++ set_bit(pending->flags, active_op_cancelled); + + } else { + crm_info("No pending op found for %s", key); +@@ -2636,7 +2635,7 @@ process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, + crm_err("Recurring operation %s was cancelled without transition information", + op_key); + +- } else if (pending->remove) { ++ } else if (is_set(pending->flags, active_op_remove)) { + /* This recurring operation was cancelled (by us) and pending, and we + * have been waiting for it to finish. + */ +-- +1.8.3.1 + + +From 620ff6cb923e4fe6c3d1a9e3345a26ff5180d56d Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 22 Nov 2019 16:58:25 -0600 +Subject: [PATCH 03/11] Refactor: controller: remove unused argument + +--- + crmd/lrm.c | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/crmd/lrm.c b/crmd/lrm.c +index 2ae0d85..80fcd69 100644 +--- a/crmd/lrm.c ++++ b/crmd/lrm.c +@@ -44,8 +44,8 @@ static int delete_rsc_status(lrm_state_t * lrm_state, const char *rsc_id, int ca + + static lrmd_event_data_t *construct_op(lrm_state_t * lrm_state, xmlNode * rsc_op, + const char *rsc_id, const char *operation); +-static void do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *operation, +- xmlNode * msg, xmlNode * request); ++static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, ++ const char *operation, xmlNode *msg); + + void send_direct_ack(const char *to_host, const char *to_sys, + lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id); +@@ -1851,7 +1851,7 @@ do_lrm_invoke(long long action, + crm_rsc_delete, user_name); + + } else { +- do_lrm_rsc_op(lrm_state, rsc, operation, input->xml, input->msg); ++ do_lrm_rsc_op(lrm_state, rsc, operation, input->xml); + } + + lrmd_free_rsc_info(rsc); +@@ -2167,8 +2167,8 @@ record_pending_op(const char *node_name, lrmd_rsc_info_t *rsc, lrmd_event_data_t + } + + static void +-do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *operation, xmlNode * msg, +- xmlNode * request) ++do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, ++ const char *operation, xmlNode *msg) + { + int call_id = 0; + char *op_id = NULL; +-- +1.8.3.1 + + +From 2716f2f4c334b927f9e253979dce27b56bbff46a Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 6 Dec 2019 12:15:05 -0600 +Subject: [PATCH 04/11] Refactor: scheduler: combine two "if" statements + +... for readability, and ease of adding another block later +--- + pengine/graph.c | 120 ++++++++++++++++++++++++++++---------------------------- + 1 file changed, 60 insertions(+), 60 deletions(-) + +diff --git a/pengine/graph.c b/pengine/graph.c +index 9edd1a1..cba30d0 100644 +--- a/pengine/graph.c ++++ b/pengine/graph.c +@@ -1095,71 +1095,71 @@ action2xml(action_t * action, gboolean as_input, pe_working_set_t *data_set) + return action_xml; + } + +- /* List affected resource */ +- if (action->rsc) { +- if (is_set(action->flags, pe_action_pseudo) == FALSE) { +- int lpc = 0; +- +- xmlNode *rsc_xml = create_xml_node(action_xml, crm_element_name(action->rsc->xml)); +- +- const char *attr_list[] = { +- XML_AGENT_ATTR_CLASS, +- XML_AGENT_ATTR_PROVIDER, +- XML_ATTR_TYPE +- }; +- +- if (is_set(action->rsc->flags, pe_rsc_orphan) && action->rsc->clone_name) { +- /* Do not use the 'instance free' name here as that +- * might interfere with the instance we plan to keep. +- * Ie. if there are more than two named /anonymous/ +- * instances on a given node, we need to make sure the +- * command goes to the right one. +- * +- * Keep this block, even when everyone is using +- * 'instance free' anonymous clone names - it means +- * we'll do the right thing if anyone toggles the +- * unique flag to 'off' +- */ +- crm_debug("Using orphan clone name %s instead of %s", action->rsc->id, +- action->rsc->clone_name); +- crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->clone_name); +- crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id); ++ if (action->rsc && is_not_set(action->flags, pe_action_pseudo)) { ++ int lpc = 0; ++ xmlNode *rsc_xml = NULL; ++ const char *attr_list[] = { ++ XML_AGENT_ATTR_CLASS, ++ XML_AGENT_ATTR_PROVIDER, ++ XML_ATTR_TYPE ++ }; ++ ++ // List affected resource ++ ++ rsc_xml = create_xml_node(action_xml, ++ crm_element_name(action->rsc->xml)); ++ if (is_set(action->rsc->flags, pe_rsc_orphan) ++ && action->rsc->clone_name) { ++ /* Do not use the 'instance free' name here as that ++ * might interfere with the instance we plan to keep. ++ * Ie. if there are more than two named /anonymous/ ++ * instances on a given node, we need to make sure the ++ * command goes to the right one. ++ * ++ * Keep this block, even when everyone is using ++ * 'instance free' anonymous clone names - it means ++ * we'll do the right thing if anyone toggles the ++ * unique flag to 'off' ++ */ ++ crm_debug("Using orphan clone name %s instead of %s", action->rsc->id, ++ action->rsc->clone_name); ++ crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->clone_name); ++ crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id); + +- } else if (is_not_set(action->rsc->flags, pe_rsc_unique)) { +- const char *xml_id = ID(action->rsc->xml); +- +- crm_debug("Using anonymous clone name %s for %s (aka. %s)", xml_id, action->rsc->id, +- action->rsc->clone_name); +- +- /* ID is what we'd like client to use +- * ID_LONG is what they might know it as instead +- * +- * ID_LONG is only strictly needed /here/ during the +- * transition period until all nodes in the cluster +- * are running the new software /and/ have rebooted +- * once (meaning that they've only ever spoken to a DC +- * supporting this feature). +- * +- * If anyone toggles the unique flag to 'on', the +- * 'instance free' name will correspond to an orphan +- * and fall into the clause above instead +- */ +- crm_xml_add(rsc_xml, XML_ATTR_ID, xml_id); +- if (action->rsc->clone_name && safe_str_neq(xml_id, action->rsc->clone_name)) { +- crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->clone_name); +- } else { +- crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id); +- } ++ } else if (is_not_set(action->rsc->flags, pe_rsc_unique)) { ++ const char *xml_id = ID(action->rsc->xml); ++ ++ crm_debug("Using anonymous clone name %s for %s (aka. %s)", xml_id, action->rsc->id, ++ action->rsc->clone_name); + ++ /* ID is what we'd like client to use ++ * ID_LONG is what they might know it as instead ++ * ++ * ID_LONG is only strictly needed /here/ during the ++ * transition period until all nodes in the cluster ++ * are running the new software /and/ have rebooted ++ * once (meaning that they've only ever spoken to a DC ++ * supporting this feature). ++ * ++ * If anyone toggles the unique flag to 'on', the ++ * 'instance free' name will correspond to an orphan ++ * and fall into the clause above instead ++ */ ++ crm_xml_add(rsc_xml, XML_ATTR_ID, xml_id); ++ if (action->rsc->clone_name && safe_str_neq(xml_id, action->rsc->clone_name)) { ++ crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->clone_name); + } else { +- CRM_ASSERT(action->rsc->clone_name == NULL); +- crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->id); ++ crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id); + } + +- for (lpc = 0; lpc < DIMOF(attr_list); lpc++) { +- crm_xml_add(rsc_xml, attr_list[lpc], +- g_hash_table_lookup(action->rsc->meta, attr_list[lpc])); +- } ++ } else { ++ CRM_ASSERT(action->rsc->clone_name == NULL); ++ crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->id); ++ } ++ ++ for (lpc = 0; lpc < DIMOF(attr_list); lpc++) { ++ crm_xml_add(rsc_xml, attr_list[lpc], ++ g_hash_table_lookup(action->rsc->meta, attr_list[lpc])); + } + } + +-- +1.8.3.1 + + +From dc73907eeae769d70b04e4a8feae208c12018d83 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 19 Dec 2019 17:18:41 -0600 +Subject: [PATCH 05/11] Log: scheduler: drop redundant trace messages + +We logged "applying placement constraints" three times. +--- + pengine/allocate.c | 17 ++++------------- + 1 file changed, 4 insertions(+), 13 deletions(-) + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index b819af3..30d29e1 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -652,21 +652,15 @@ check_actions(pe_working_set_t * data_set) + } + } + +-static gboolean ++static void + apply_placement_constraints(pe_working_set_t * data_set) + { +- GListPtr gIter = NULL; +- +- crm_trace("Applying constraints..."); +- +- for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) { ++ for (GList *gIter = data_set->placement_constraints; ++ gIter != NULL; gIter = gIter->next) { + pe__location_t *cons = gIter->data; + + cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons); + } +- +- return TRUE; +- + } + + static gboolean +@@ -1026,10 +1020,7 @@ stage2(pe_working_set_t * data_set) + { + GListPtr gIter = NULL; + +- crm_trace("Applying placement constraints"); +- +- gIter = data_set->nodes; +- for (; gIter != NULL; gIter = gIter->next) { ++ for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { + node_t *node = (node_t *) gIter->data; + + if (node == NULL) { +-- +1.8.3.1 + + +From ae404a4e04b50e6177942256c5b8ff57a36af309 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Sat, 7 Dec 2019 12:13:11 -0600 +Subject: [PATCH 06/11] Refactor: libcrmcommon: convenience functions for list + length comparisons + +... for efficiency and readability +--- + include/crm/common/internal.h | 14 ++++++++++++++ + 1 file changed, 14 insertions(+) + +diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h +index 0d225f5..0f7158c 100644 +--- a/include/crm/common/internal.h ++++ b/include/crm/common/internal.h +@@ -84,6 +84,20 @@ crm_getpid_s() + return crm_strdup_printf("%lu", (unsigned long) getpid()); + } + ++// More efficient than g_list_length(list) == 1 ++static inline bool ++pcmk__list_of_1(GList *list) ++{ ++ return list && (list->next == NULL); ++} ++ ++// More efficient than g_list_length(list) > 1 ++static inline bool ++pcmk__list_of_multiple(GList *list) ++{ ++ return list && (list->next != NULL); ++} ++ + /* convenience functions for failure-related node attributes */ + + #define CRM_FAIL_COUNT_PREFIX "fail-count" +-- +1.8.3.1 + + +From 78a1d57ae4028e3068b13d3526840f24c7798140 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Mon, 16 Dec 2019 14:13:30 -0600 +Subject: [PATCH 07/11] Refactor: libcrmcommon: add convenience macros for + plurals + +I've avoided making s_if_plural() an official API due to its hackiness, but +it really is the best solution for now. Promote it to pcmk__plural_s(), along +with a companion macro pcmk__plural_alt() for more complicated plurals. +--- + include/crm/common/internal.h | 23 +++++++++++++++++++++++ + 1 file changed, 23 insertions(+) + +diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h +index 0f7158c..5bbf01b 100644 +--- a/include/crm/common/internal.h ++++ b/include/crm/common/internal.h +@@ -72,6 +72,29 @@ bool crm_compress_string(const char *data, int length, int max, char **result, + unsigned int *result_len); + gint crm_alpha_sort(gconstpointer a, gconstpointer b); + ++/* Correctly displaying singular or plural is complicated; consider "1 node has" ++ * vs. "2 nodes have". A flexible solution is to pluralize entire strings, e.g. ++ * ++ * if (a == 1) { ++ * crm_info("singular message"): ++ * } else { ++ * crm_info("plural message"); ++ * } ++ * ++ * though even that's not sufficient for all languages besides English (if we ++ * ever desire to do translations of output and log messages). But the following ++ * convenience macros are "good enough" and more concise for many cases. ++ */ ++ ++/* Example: ++ * crm_info("Found %d %s", nentries, ++ * pcmk__plural_alt(nentries, "entry", "entries")); ++ */ ++#define pcmk__plural_alt(i, s1, s2) (((i) == 1)? (s1) : (s2)) ++ ++// Example: crm_info("Found %d node%s", nnodes, pcmk__plural_s(nnodes)); ++#define pcmk__plural_s(i) pcmk__plural_alt(i, "", "s") ++ + static inline int + crm_strlen_zero(const char *s) + { +-- +1.8.3.1 + + +From e33b5e297c24182c15ace7142efef1add7664643 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 12 Dec 2019 20:50:50 -0600 +Subject: [PATCH 08/11] Log: controller: improve join messages + +--- + crmd/fsa.c | 79 +++++++----- + crmd/join_dc.c | 375 +++++++++++++++++++++++++++++++++------------------------ + 2 files changed, 263 insertions(+), 191 deletions(-) + +diff --git a/crmd/fsa.c b/crmd/fsa.c +index 9b1189a..7f6a3ac 100644 +--- a/crmd/fsa.c ++++ b/crmd/fsa.c +@@ -469,12 +469,53 @@ log_fsa_input(fsa_data_t * stored_msg) + } + } + ++static void ++check_join_counts(fsa_data_t *msg_data) ++{ ++ int count; ++ guint npeers; ++ ++ count = crmd_join_phase_count(crm_join_finalized); ++ if (count > 0) { ++ crm_err("%d cluster node%s failed to confirm join", ++ count, pcmk__plural_s(count)); ++ crmd_join_phase_log(LOG_NOTICE); ++ return; ++ } ++ ++ npeers = crm_active_peers(); ++ count = crmd_join_phase_count(crm_join_confirmed); ++ if (count == npeers) { ++ if (npeers == 1) { ++ crm_debug("Sole active cluster node is fully joined"); ++ } else { ++ crm_debug("All %d active cluster nodes are fully joined", count); ++ } ++ ++ } else if (count > npeers) { ++ crm_err("New election needed because more nodes confirmed join " ++ "than are in membership (%d > %u)", count, npeers); ++ register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); ++ ++ } else if (saved_ccm_membership_id != crm_peer_seq) { ++ crm_info("New join needed because membership changed (%llu -> %llu)", ++ saved_ccm_membership_id, crm_peer_seq); ++ register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL); ++ ++ } else { ++ crm_warn("Only %d of %u active cluster nodes fully joined " ++ "(%d did not respond to offer)", ++ count, npeers, crmd_join_phase_count(crm_join_welcomed)); ++ } ++} ++ + long long + do_state_transition(long long actions, + enum crmd_fsa_state cur_state, + enum crmd_fsa_state next_state, fsa_data_t * msg_data) + { + int level = LOG_INFO; ++ int count = 0; + long long tmp = actions; + gboolean clear_recovery_bit = TRUE; + +@@ -572,13 +613,14 @@ do_state_transition(long long actions, + crm_warn("Progressed to state %s after %s", + fsa_state2string(next_state), fsa_cause2string(cause)); + } +- if (crmd_join_phase_count(crm_join_welcomed) > 0) { +- crm_warn("%u cluster nodes failed to respond" +- " to the join offer.", crmd_join_phase_count(crm_join_welcomed)); ++ count = crmd_join_phase_count(crm_join_welcomed); ++ if (count > 0) { ++ crm_warn("%d cluster node%s failed to respond to join offer", ++ count, pcmk__plural_s(count)); + crmd_join_phase_log(LOG_NOTICE); + + } else { +- crm_debug("All %d cluster nodes responded to the join offer.", ++ crm_debug("All cluster nodes (%d) responded to join offer", + crmd_join_phase_count(crm_join_integrated)); + } + break; +@@ -590,34 +632,7 @@ do_state_transition(long long actions, + crm_info("Progressed to state %s after %s", + fsa_state2string(next_state), fsa_cause2string(cause)); + } +- +- if (crmd_join_phase_count(crm_join_finalized) > 0) { +- crm_err("%u cluster nodes failed to confirm their join.", +- crmd_join_phase_count(crm_join_finalized)); +- crmd_join_phase_log(LOG_NOTICE); +- +- } else if (crmd_join_phase_count(crm_join_confirmed) +- == crm_active_peers()) { +- crm_debug("All %u cluster nodes are" +- " eligible to run resources.", crm_active_peers()); +- +- } else if (crmd_join_phase_count(crm_join_confirmed) > crm_active_peers()) { +- crm_err("We have more confirmed nodes than our membership does: %d vs. %d", +- crmd_join_phase_count(crm_join_confirmed), crm_active_peers()); +- register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); +- +- } else if (saved_ccm_membership_id != crm_peer_seq) { +- crm_info("Membership changed: %llu -> %llu - join restart", +- saved_ccm_membership_id, crm_peer_seq); +- register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL); +- +- } else { +- crm_warn("Only %u of %u cluster " +- "nodes are eligible to run resources - continue %d", +- crmd_join_phase_count(crm_join_confirmed), +- crm_active_peers(), crmd_join_phase_count(crm_join_welcomed)); +- } +-/* initialize_join(FALSE); */ ++ check_join_counts(msg_data); + break; + + case S_STOPPING: +diff --git a/crmd/join_dc.c b/crmd/join_dc.c +index 857e760..cdb3f77 100644 +--- a/crmd/join_dc.c ++++ b/crmd/join_dc.c +@@ -36,7 +36,11 @@ void finalize_join_for(gpointer key, gpointer value, gpointer user_data); + void finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data); + gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source); + ++/* Numeric counter used to identify join rounds (an unsigned int would be ++ * appropriate, except we get and set it in XML as int) ++ */ + static int current_join_id = 0; ++ + unsigned long long saved_ccm_membership_id = 0; + + void +@@ -44,12 +48,7 @@ crm_update_peer_join(const char *source, crm_node_t * node, enum crm_join_phase + { + enum crm_join_phase last = 0; + +- if(node == NULL) { +- crm_err("Could not update join because node not specified" +- CRM_XS " join-%u source=%s phase=%s", +- current_join_id, source, crm_join_phase_str(phase)); +- return; +- } ++ CRM_CHECK(node != NULL, return); + + /* Remote nodes do not participate in joins */ + if (is_set(node->flags, crm_remote_node)) { +@@ -59,21 +58,23 @@ crm_update_peer_join(const char *source, crm_node_t * node, enum crm_join_phase + last = node->join; + + if(phase == last) { +- crm_trace("%s: Node %s[%u] - join-%u phase still %s", +- source, node->uname, node->id, current_join_id, +- crm_join_phase_str(last)); ++ crm_trace("Node %s join-%d phase is still %s " ++ CRM_XS " nodeid=%u source=%s", ++ node->uname, current_join_id, crm_join_phase_str(last), ++ node->id, source); + + } else if ((phase <= crm_join_none) || (phase == (last + 1))) { + node->join = phase; +- crm_info("%s: Node %s[%u] - join-%u phase %s -> %s", +- source, node->uname, node->id, current_join_id, +- crm_join_phase_str(last), crm_join_phase_str(phase)); ++ crm_trace("Node %s join-%d phase is now %s (was %s) " ++ CRM_XS " nodeid=%u source=%s", ++ node->uname, current_join_id, crm_join_phase_str(phase), ++ crm_join_phase_str(last), node->id, source); + + } else { +- crm_err("Could not update join for node %s because phase transition invalid " +- CRM_XS " join-%u source=%s node_id=%u last=%s new=%s", +- node->uname, current_join_id, source, node->id, +- crm_join_phase_str(last), crm_join_phase_str(phase)); ++ crm_warn("Rejecting join-%d phase update for node %s because " ++ "can't go from %s to %s " CRM_XS " nodeid=%u source=%s", ++ current_join_id, node->uname, crm_join_phase_str(last), ++ crm_join_phase_str(phase), node->id, source); + } + } + +@@ -83,9 +84,7 @@ initialize_join(gboolean before) + GHashTableIter iter; + crm_node_t *peer = NULL; + +- /* clear out/reset a bunch of stuff */ +- crm_debug("join-%d: Initializing join data (flag=%s)", +- current_join_id, before ? "true" : "false"); ++ crm_debug("Starting new join round join-%d", current_join_id); + + g_hash_table_iter_init(&iter, crm_peer_cache); + while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) { +@@ -138,7 +137,9 @@ join_make_offer(gpointer key, gpointer value, gpointer user_data) + + CRM_ASSERT(member != NULL); + if (crm_is_peer_active(member) == FALSE) { +- crm_info("Not making an offer to %s: not active (%s)", member->uname, member->state); ++ crm_info("Not making join-%d offer to inactive node %s", ++ current_join_id, ++ (member->uname? member->uname : "with unknown name")); + if(member->expected == NULL && safe_str_eq(member->state, CRM_NODE_LOST)) { + /* You would think this unsafe, but in fact this plus an + * active resource is what causes it to be fenced. +@@ -155,17 +156,21 @@ join_make_offer(gpointer key, gpointer value, gpointer user_data) + } + + if (member->uname == NULL) { +- crm_info("No recipient for welcome message.(Node uuid:%s)", member->uuid); ++ crm_info("Not making join-%d offer to node uuid %s with unknown name", ++ current_join_id, member->uuid); + return; + } + + if (saved_ccm_membership_id != crm_peer_seq) { + saved_ccm_membership_id = crm_peer_seq; +- crm_info("Making join offers based on membership %llu", crm_peer_seq); ++ crm_info("Making join-%d offers based on membership event %llu", ++ current_join_id, crm_peer_seq); + } + + if(user_data && member->join > crm_join_none) { +- crm_info("Skipping %s: already known %d", member->uname, member->join); ++ crm_info("Not making join-%d offer to already known node %s (%s)", ++ current_join_id, member->uname, ++ crm_join_phase_str(member->join)); + return; + } + +@@ -173,14 +178,11 @@ join_make_offer(gpointer key, gpointer value, gpointer user_data) + + offer = create_dc_message(CRM_OP_JOIN_OFFER, member->uname); + +- /* send the welcome */ +- crm_info("join-%d: Sending offer to %s", current_join_id, member->uname); +- ++ crm_info("Sending join-%d offer to %s", current_join_id, member->uname); + send_cluster_message(member, crm_msg_crmd, offer, TRUE); + free_xml(offer); + + crm_update_peer_join(__FUNCTION__, member, crm_join_welcomed); +- /* crm_update_peer_expected(__FUNCTION__, member, CRMD_JOINSTATE_PENDING); */ + } + + /* A_DC_JOIN_OFFER_ALL */ +@@ -190,6 +192,8 @@ do_dc_join_offer_all(long long action, + enum crmd_fsa_state cur_state, + enum crmd_fsa_input current_input, fsa_data_t * msg_data) + { ++ int count; ++ + /* reset everyone's status back to down or in_ccm in the CIB + * + * any nodes that are active in the CIB but not in the CCM list +@@ -205,9 +209,11 @@ do_dc_join_offer_all(long long action, + } + g_hash_table_foreach(crm_peer_cache, join_make_offer, NULL); + ++ count = crmd_join_phase_count(crm_join_welcomed); ++ crm_info("Waiting on join-%d requests from %d outstanding node%s", ++ current_join_id, count, pcmk__plural_s(count)); ++ + /* don't waste time by invoking the PE yet; */ +- crm_info("join-%d: Waiting on %d outstanding join acks", +- current_join_id, crmd_join_phase_count(crm_join_welcomed)); + } + + /* A_DC_JOIN_OFFER_ONE */ +@@ -219,50 +225,40 @@ do_dc_join_offer_one(long long action, + { + crm_node_t *member; + ha_msg_input_t *welcome = NULL; +- +- const char *op = NULL; ++ int count; + const char *join_to = NULL; + +- if (msg_data->data) { +- welcome = fsa_typed_data(fsa_dt_ha_msg); +- +- } else { +- crm_info("An unknown node joined - (re-)offer to any unconfirmed nodes"); ++ if (msg_data->data == NULL) { ++ crm_info("Making join-%d offers to any unconfirmed nodes " ++ "because an unknown node joined", current_join_id); + g_hash_table_foreach(crm_peer_cache, join_make_offer, &member); + check_join_state(cur_state, __FUNCTION__); + return; + } + ++ welcome = fsa_typed_data(fsa_dt_ha_msg); + if (welcome == NULL) { +- crm_err("Attempt to send welcome message without a message to reply to!"); ++ // fsa_typed_data() already logged an error + return; + } + + join_to = crm_element_value(welcome->msg, F_CRM_HOST_FROM); + if (join_to == NULL) { +- crm_err("Attempt to send welcome message without a host to reply to!"); ++ crm_err("Can't make join-%d offer to unknown node", current_join_id); + return; + } +- + member = crm_get_peer(0, join_to); +- op = crm_element_value(welcome->msg, F_CRM_TASK); +- if (join_to != NULL && (cur_state == S_INTEGRATION || cur_state == S_FINALIZE_JOIN)) { +- /* note: it _is_ possible that a node will have been +- * sick or starting up when the original offer was made. +- * however, it will either re-announce itself in due course +- * _or_ we can re-store the original offer on the client. +- */ +- crm_trace("(Re-)offering membership to %s...", join_to); +- } + +- crm_info("join-%d: Processing %s request from %s in state %s", +- current_join_id, op, join_to, fsa_state2string(cur_state)); ++ /* It is possible that a node will have been sick or starting up when the ++ * original offer was made. However, it will either re-announce itself in ++ * due course, or we can re-store the original offer on the client. ++ */ + + crm_update_peer_join(__FUNCTION__, member, crm_join_none); + join_make_offer(NULL, member, NULL); + +- /* always offer to the DC (ourselves) +- * this ensures the correct value for max_generation_from ++ /* If the offer isn't to the local node, make an offer to the local node as ++ * well, to ensure the correct value for max_generation_from. + */ + if (strcmp(join_to, fsa_our_uname) != 0) { + member = crm_get_peer(0, fsa_our_uname); +@@ -274,9 +270,11 @@ do_dc_join_offer_one(long long action, + */ + abort_transition(INFINITY, tg_restart, "Node join", NULL); + ++ count = crmd_join_phase_count(crm_join_welcomed); ++ crm_info("Waiting on join-%d requests from %d outstanding node%s", ++ current_join_id, count, pcmk__plural_s(count)); ++ + /* don't waste time by invoking the PE yet; */ +- crm_debug("Waiting on %d outstanding join acks for join-%d", +- crmd_join_phase_count(crm_join_welcomed), current_join_id); + } + + static int +@@ -309,20 +307,29 @@ do_dc_join_filter_offer(long long action, + + int cmp = 0; + int join_id = -1; ++ int count = 0; + gboolean ack_nack_bool = TRUE; +- const char *ack_nack = CRMD_JOINSTATE_MEMBER; + ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg); + + const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM); + const char *ref = crm_element_value(join_ack->msg, F_CRM_REFERENCE); ++ crm_node_t *join_node = NULL; + +- crm_node_t *join_node = crm_get_peer(0, join_from); +- +- crm_debug("Processing req from %s", join_from); ++ if (join_from == NULL) { ++ crm_err("Ignoring invalid join request without node name"); ++ return; ++ } ++ join_node = crm_get_peer(0, join_from); + +- generation = join_ack->xml; + crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id); ++ if (join_id != current_join_id) { ++ crm_debug("Ignoring join-%d request from %s because we are on join-%d", ++ join_id, join_from, current_join_id); ++ check_join_state(cur_state, __FUNCTION__); ++ return; ++ } + ++ generation = join_ack->xml; + if (max_generation_xml != NULL && generation != NULL) { + int lpc = 0; + +@@ -337,61 +344,63 @@ do_dc_join_filter_offer(long long action, + } + } + +- if (join_id != current_join_id) { +- crm_debug("Invalid response from %s: join-%d vs. join-%d", +- join_from, join_id, current_join_id); +- check_join_state(cur_state, __FUNCTION__); +- return; ++ if (ref == NULL) { ++ ref = "none"; // for logging only ++ } + +- } else if (join_node == NULL || crm_is_peer_active(join_node) == FALSE) { +- crm_err("Node %s is not a member", join_from); ++ if (crm_is_peer_active(join_node) == FALSE) { ++ crm_err("Rejecting join-%d request from inactive node %s " ++ CRM_XS " ref=%s", join_id, join_from, ref); + ack_nack_bool = FALSE; + + } else if (generation == NULL) { +- crm_err("Generation was NULL"); ++ crm_err("Rejecting invalid join-%d request from node %s " ++ "missing CIB generation " CRM_XS " ref=%s", ++ join_id, join_from, ref); + ack_nack_bool = FALSE; + + } else if (max_generation_xml == NULL) { ++ crm_debug("Accepting join-%d request from %s " ++ "(with first CIB generation) " CRM_XS " ref=%s", ++ join_id, join_from, ref); + max_generation_xml = copy_xml(generation); + max_generation_from = strdup(join_from); + + } else if (cmp < 0 || (cmp == 0 && safe_str_eq(join_from, fsa_our_uname))) { +- crm_debug("%s has a better generation number than" +- " the current max %s", join_from, max_generation_from); +- if (max_generation_xml) { +- crm_log_xml_debug(max_generation_xml, "Max generation"); +- } +- crm_log_xml_debug(generation, "Their generation"); ++ crm_debug("Accepting join-%d request from %s (with better " ++ "CIB generation than current best from %s) " CRM_XS " ref=%s", ++ join_id, join_from, max_generation_from, ref); ++ crm_log_xml_debug(max_generation_xml, "Old max generation"); ++ crm_log_xml_debug(generation, "New max generation"); + + free(max_generation_from); + free_xml(max_generation_xml); + + max_generation_from = strdup(join_from); + max_generation_xml = copy_xml(join_ack->xml); ++ ++ } else { ++ crm_debug("Accepting join-%d request from %s " CRM_XS " ref=%s", ++ join_id, join_from, ref); + } + + if (ack_nack_bool == FALSE) { +- /* NACK this client */ +- ack_nack = CRMD_JOINSTATE_NACK; + crm_update_peer_join(__FUNCTION__, join_node, crm_join_nack); +- crm_err("Rejecting cluster join request from %s " CRM_XS +- " NACK join-%d ref=%s", join_from, join_id, ref); +- ++ crm_update_peer_expected(__FUNCTION__, join_node, CRMD_JOINSTATE_NACK); + } else { +- crm_debug("join-%d: Welcoming node %s (ref %s)", join_id, join_from, ref); + crm_update_peer_join(__FUNCTION__, join_node, crm_join_integrated); ++ crm_update_peer_expected(__FUNCTION__, join_node, CRMD_JOINSTATE_MEMBER); + } + +- crm_update_peer_expected(__FUNCTION__, join_node, ack_nack); +- +- crm_debug("%u nodes have been integrated into join-%d", +- crmd_join_phase_count(crm_join_integrated), join_id); +- ++ count = crmd_join_phase_count(crm_join_integrated); ++ crm_debug("%d node%s currently integrated in join-%d", ++ count, pcmk__plural_s(count), join_id); + + if (check_join_state(cur_state, __FUNCTION__) == FALSE) { + /* don't waste time by invoking the PE yet; */ +- crm_debug("join-%d: Still waiting on %d outstanding offers", +- join_id, crmd_join_phase_count(crm_join_welcomed)); ++ count = crmd_join_phase_count(crm_join_welcomed); ++ crm_debug("Waiting on join-%d requests from %d outstanding node%s", ++ join_id, count, pcmk__plural_s(count)); + } + } + +@@ -404,21 +413,24 @@ do_dc_join_finalize(long long action, + { + char *sync_from = NULL; + int rc = pcmk_ok; ++ int count_welcomed = crmd_join_phase_count(crm_join_welcomed); ++ int count_integrated = crmd_join_phase_count(crm_join_integrated); + + /* This we can do straight away and avoid clients timing us out + * while we compute the latest CIB + */ +- crm_debug("Finalizing join-%d for %d clients", +- current_join_id, crmd_join_phase_count(crm_join_integrated)); +- +- crmd_join_phase_log(LOG_INFO); +- if (crmd_join_phase_count(crm_join_welcomed) != 0) { +- crm_info("Waiting for %d more nodes", crmd_join_phase_count(crm_join_welcomed)); ++ if (count_welcomed != 0) { ++ crm_debug("Waiting on join-%d requests from %d outstanding node%s " ++ "before finalizing join", current_join_id, count_welcomed, ++ pcmk__plural_s(count_welcomed)); ++ crmd_join_phase_log(LOG_DEBUG); + /* crmd_fsa_stall(FALSE); Needed? */ + return; + +- } else if (crmd_join_phase_count(crm_join_integrated) == 0) { +- /* Nothing to do */ ++ } else if (count_integrated == 0) { ++ crm_debug("Finalization not needed for join-%d at the current time", ++ current_join_id); ++ crmd_join_phase_log(LOG_DEBUG); + check_join_state(fsa_state, __FUNCTION__); + return; + } +@@ -429,8 +441,9 @@ do_dc_join_finalize(long long action, + } + + if (is_set(fsa_input_register, R_IN_TRANSITION)) { +- crm_warn("Delaying response to cluster join offer while transition in progress " +- CRM_XS " join-%d", current_join_id); ++ crm_warn("Delaying join-%d finalization while transition in progress", ++ current_join_id); ++ crmd_join_phase_log(LOG_DEBUG); + crmd_fsa_stall(FALSE); + return; + } +@@ -439,18 +452,20 @@ do_dc_join_finalize(long long action, + /* ask for the agreed best CIB */ + sync_from = strdup(max_generation_from); + set_bit(fsa_input_register, R_CIB_ASKED); +- crm_notice("Syncing the Cluster Information Base from %s to rest of cluster " +- CRM_XS " join-%d", sync_from, current_join_id); +- crm_log_xml_notice(max_generation_xml, "Requested version"); ++ crm_notice("Finalizing join-%d for %d node%s (sync'ing CIB from %s)", ++ current_join_id, count_integrated, ++ pcmk__plural_s(count_integrated), sync_from); ++ crm_log_xml_notice(max_generation_xml, "Requested CIB version"); + + } else { + /* Send _our_ CIB out to everyone */ + sync_from = strdup(fsa_our_uname); +- crm_info("join-%d: Syncing our CIB to the rest of the cluster", +- current_join_id); +- crm_log_xml_debug(max_generation_xml, "Requested version"); ++ crm_debug("Finalizing join-%d for %d node%s (sync'ing from local CIB)", ++ current_join_id, count_integrated, ++ pcmk__plural_s(count_integrated)); ++ crm_log_xml_debug(max_generation_xml, "Requested CIB version"); + } +- ++ crmd_join_phase_log(LOG_DEBUG); + + rc = fsa_cib_conn->cmds->sync_from(fsa_cib_conn, sync_from, NULL, cib_quorum_override); + fsa_register_cib_callback(rc, FALSE, sync_from, finalize_sync_callback); +@@ -462,26 +477,33 @@ finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, voi + CRM_LOG_ASSERT(-EPERM != rc); + clear_bit(fsa_input_register, R_CIB_ASKED); + if (rc != pcmk_ok) { +- do_crm_log((rc == -pcmk_err_old_data ? LOG_WARNING : LOG_ERR), +- "Sync from %s failed: %s", (char *)user_data, pcmk_strerror(rc)); ++ do_crm_log(((rc == -pcmk_err_old_data)? LOG_WARNING : LOG_ERR), ++ "Could not sync CIB from %s in join-%d: %s", ++ (char *) user_data, current_join_id, pcmk_strerror(rc)); + + /* restart the whole join process */ + register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION_DC, NULL, NULL, __FUNCTION__); + +- } else if (AM_I_DC && fsa_state == S_FINALIZE_JOIN) { ++ } else if (!AM_I_DC) { ++ crm_debug("Sync'ed CIB for join-%d but no longer DC", current_join_id); ++ ++ } else if (fsa_state != S_FINALIZE_JOIN) { ++ crm_debug("Sync'ed CIB for join-%d but no longer in S_FINALIZE_JOIN (%s)", ++ current_join_id, fsa_state2string(fsa_state)); ++ ++ } else { + set_bit(fsa_input_register, R_HAVE_CIB); + clear_bit(fsa_input_register, R_CIB_ASKED); + + /* make sure dc_uuid is re-set to us */ + if (check_join_state(fsa_state, __FUNCTION__) == FALSE) { +- crm_debug("Notifying %d clients of join-%d results", +- crmd_join_phase_count(crm_join_integrated), current_join_id); ++ int count_integrated = crmd_join_phase_count(crm_join_integrated); ++ ++ crm_debug("Notifying %d node%s of join-%d results", ++ count_integrated, pcmk__plural_s(count_integrated), ++ current_join_id); + g_hash_table_foreach(crm_peer_cache, finalize_join_for, NULL); + } +- +- } else { +- crm_debug("No longer the DC in S_FINALIZE_JOIN: %s/%s", +- AM_I_DC ? "DC" : "CRMd", fsa_state2string(fsa_state)); + } + } + +@@ -491,11 +513,14 @@ join_update_complete_callback(xmlNode * msg, int call_id, int rc, xmlNode * outp + fsa_data_t *msg_data = NULL; + + if (rc == pcmk_ok) { +- crm_debug("Join update %d complete", call_id); ++ crm_debug("join-%d node history update (via CIB call %d) complete", ++ current_join_id, call_id); + check_join_state(fsa_state, __FUNCTION__); + + } else { +- crm_err("Join update %d failed", call_id); ++ crm_err("join-%d node history update (via CIB call %d) failed: %s " ++ "(next transition may determine resource status incorrectly)", ++ current_join_id, call_id, pcmk_strerror(rc)); + crm_log_xml_debug(msg, "failed"); + register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); + } +@@ -514,60 +539,75 @@ do_dc_join_ack(long long action, + + const char *op = crm_element_value(join_ack->msg, F_CRM_TASK); + const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM); +- crm_node_t *peer = crm_get_peer(0, join_from); ++ crm_node_t *peer = NULL; + +- if (safe_str_neq(op, CRM_OP_JOIN_CONFIRM) || peer == NULL) { +- crm_debug("Ignoring op=%s message from %s", op, join_from); ++ // Sanity checks ++ if (join_from == NULL) { ++ crm_warn("Ignoring message received without node identification"); ++ return; ++ } ++ if (op == NULL) { ++ crm_warn("Ignoring message received from %s without task", join_from); + return; + } + +- crm_trace("Processing ack from %s", join_from); +- crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id); ++ if (strcmp(op, CRM_OP_JOIN_CONFIRM)) { ++ crm_debug("Ignoring '%s' message from %s while waiting for '%s'", ++ op, join_from, CRM_OP_JOIN_CONFIRM); ++ return; ++ } + ++ if (crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id) != 0) { ++ crm_warn("Ignoring join confirmation from %s without valid join ID", ++ join_from); ++ return; ++ } ++ ++ peer = crm_get_peer(0, join_from); + if (peer->join != crm_join_finalized) { +- crm_info("Join not in progress: ignoring join-%d from %s (phase = %d)", +- join_id, join_from, peer->join); ++ crm_info("Ignoring out-of-sequence join-%d confirmation from %s " ++ "(currently %s not %s)", ++ join_id, join_from, crm_join_phase_str(peer->join), ++ crm_join_phase_str(crm_join_finalized)); + return; ++ } + +- } else if (join_id != current_join_id) { +- crm_err("Invalid response from %s: join-%d vs. join-%d", +- join_from, join_id, current_join_id); ++ if (join_id != current_join_id) { ++ crm_err("Rejecting join-%d confirmation from %s " ++ "because currently on join-%d", ++ join_id, join_from, current_join_id); + crm_update_peer_join(__FUNCTION__, peer, crm_join_nack); + return; + } + + crm_update_peer_join(__FUNCTION__, peer, crm_join_confirmed); + +- crm_info("join-%d: Updating node state to %s for %s", +- join_id, CRMD_JOINSTATE_MEMBER, join_from); +- +- /* update CIB with the current LRM status from the node +- * We don't need to notify the TE of these updates, a transition will +- * be started in due time ++ /* Update CIB with node's current LRM state. A new transition will be ++ * triggered later, when the CIB notifies us of the change. + */ + erase_status_tag(join_from, XML_CIB_TAG_LRM, cib_scope_local); +- + if (safe_str_eq(join_from, fsa_our_uname)) { + xmlNode *now_dc_lrmd_state = do_lrm_query(TRUE, fsa_our_uname); + + if (now_dc_lrmd_state != NULL) { +- crm_debug("LRM state is updated from do_lrm_query.(%s)", join_from); + fsa_cib_update(XML_CIB_TAG_STATUS, now_dc_lrmd_state, + cib_scope_local | cib_quorum_override | cib_can_create, call_id, NULL); + free_xml(now_dc_lrmd_state); ++ crm_debug("Updating local node history for join-%d " ++ "from query result (via CIB call %d)", join_id, call_id); + } else { +- crm_warn("Could not get our LRM state. LRM state is updated from join_ack->xml.(%s)", join_from); + fsa_cib_update(XML_CIB_TAG_STATUS, join_ack->xml, + cib_scope_local | cib_quorum_override | cib_can_create, call_id, NULL); ++ crm_warn("Updating local node history from join-%d confirmation " ++ "because query failed (via CIB call %d)", join_id, call_id); + } + } else { +- crm_debug("LRM state is updated from join_ack->xml.(%s)", join_from); + fsa_cib_update(XML_CIB_TAG_STATUS, join_ack->xml, + cib_scope_local | cib_quorum_override | cib_can_create, call_id, NULL); ++ crm_debug("Updating node history for %s from join-%d confirmation " ++ "(via CIB call %d)", join_from, join_id, call_id); + } +- + fsa_register_cib_callback(call_id, FALSE, NULL, join_update_complete_callback); +- crm_debug("join-%d: Registered callback for LRM update %d", join_id, call_id); + } + + void +@@ -579,17 +619,16 @@ finalize_join_for(gpointer key, gpointer value, gpointer user_data) + const char *join_to = join_node->uname; + + if(join_node->join != crm_join_integrated) { +- crm_trace("Skipping %s in state %d", join_to, join_node->join); ++ crm_trace("Not updating non-integrated node %s (%s) for join-%d", ++ join_to, crm_join_phase_str(join_node->join), ++ current_join_id); + return; + } + +- /* make sure a node entry exists for the new node */ +- crm_trace("Creating node entry for %s", join_to); +- ++ crm_trace("Updating node state for %s", join_to); + tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE); + set_uuid(tmp1, XML_ATTR_UUID, join_node); + crm_xml_add(tmp1, XML_ATTR_UNAME, join_to); +- + fsa_cib_anon_update(XML_CIB_TAG_NODES, tmp1); + free_xml(tmp1); + +@@ -608,11 +647,10 @@ finalize_join_for(gpointer key, gpointer value, gpointer user_data) + return; + } + +- /* send the ack/nack to the node */ +- acknak = create_dc_message(CRM_OP_JOIN_ACKNAK, join_to); +- +- crm_debug("join-%d: ACK'ing join request from %s", ++ // Acknowledge node's join request ++ crm_debug("Acknowledging join-%d request from %s", + current_join_id, join_to); ++ acknak = create_dc_message(CRM_OP_JOIN_ACKNAK, join_to); + crm_xml_add(acknak, CRM_OP_JOIN_ACKNAK, XML_BOOLEAN_TRUE); + crm_update_peer_join(__FUNCTION__, join_node, crm_join_finalized); + crm_update_peer_expected(__FUNCTION__, join_node, CRMD_JOINSTATE_MEMBER); +@@ -629,11 +667,11 @@ check_join_state(enum crmd_fsa_state cur_state, const char *source) + { + static unsigned long long highest_seq = 0; + +- crm_debug("Invoked by %s in state: %s", source, fsa_state2string(cur_state)); +- + if (saved_ccm_membership_id != crm_peer_seq) { +- crm_debug("%s: Membership changed since join started: %llu -> %llu (%llu)", +- source, saved_ccm_membership_id, crm_peer_seq, highest_seq); ++ crm_debug("join-%d: Membership changed from %llu to %llu " ++ CRM_XS " highest=%llu state=%s for=%s", ++ current_join_id, saved_ccm_membership_id, crm_peer_seq, highest_seq, ++ fsa_state2string(cur_state), source); + if(highest_seq < crm_peer_seq) { + /* Don't spam the FSA with duplicates */ + highest_seq = crm_peer_seq; +@@ -642,34 +680,53 @@ check_join_state(enum crmd_fsa_state cur_state, const char *source) + + } else if (cur_state == S_INTEGRATION) { + if (crmd_join_phase_count(crm_join_welcomed) == 0) { +- crm_debug("join-%d: Integration of %d peers complete: %s", +- current_join_id, crmd_join_phase_count(crm_join_integrated), source); ++ int count = crmd_join_phase_count(crm_join_integrated); ++ ++ crm_debug("join-%d: Integration of %d peer%s complete " ++ CRM_XS " state=%s for=%s", ++ current_join_id, count, pcmk__plural_s(count), ++ fsa_state2string(cur_state), source); + register_fsa_input_before(C_FSA_INTERNAL, I_INTEGRATED, NULL); + return TRUE; + } + + } else if (cur_state == S_FINALIZE_JOIN) { + if (is_set(fsa_input_register, R_HAVE_CIB) == FALSE) { +- crm_debug("join-%d: Delaying I_FINALIZED until we have the CIB", current_join_id); ++ crm_debug("join-%d: Delaying finalization until we have CIB " ++ CRM_XS " state=%s for=%s", ++ current_join_id, fsa_state2string(cur_state), source); + return TRUE; + + } else if (crmd_join_phase_count(crm_join_welcomed) != 0) { +- crm_debug("join-%d: Still waiting on %d welcomed nodes", +- current_join_id, crmd_join_phase_count(crm_join_welcomed)); ++ int count = crmd_join_phase_count(crm_join_welcomed); ++ ++ crm_debug("join-%d: Still waiting on %d welcomed node%s " ++ CRM_XS " state=%s for=%s", ++ current_join_id, count, pcmk__plural_s(count), ++ fsa_state2string(cur_state), source); + crmd_join_phase_log(LOG_DEBUG); + + } else if (crmd_join_phase_count(crm_join_integrated) != 0) { +- crm_debug("join-%d: Still waiting on %d integrated nodes", +- current_join_id, crmd_join_phase_count(crm_join_integrated)); ++ int count = crmd_join_phase_count(crm_join_integrated); ++ ++ crm_debug("join-%d: Still waiting on %d integrated node%s " ++ CRM_XS " state=%s for=%s", ++ current_join_id, count, pcmk__plural_s(count), ++ fsa_state2string(cur_state), source); + crmd_join_phase_log(LOG_DEBUG); + + } else if (crmd_join_phase_count(crm_join_finalized) != 0) { +- crm_debug("join-%d: Still waiting on %d finalized nodes", +- current_join_id, crmd_join_phase_count(crm_join_finalized)); ++ int count = crmd_join_phase_count(crm_join_finalized); ++ ++ crm_debug("join-%d: Still waiting on %d finalized node%s " ++ CRM_XS " state=%s for=%s", ++ current_join_id, count, pcmk__plural_s(count), ++ fsa_state2string(cur_state), source); + crmd_join_phase_log(LOG_DEBUG); + + } else { +- crm_debug("join-%d complete: %s", current_join_id, source); ++ crm_debug("join-%d: Complete " CRM_XS " state=%s for=%s", ++ current_join_id, fsa_state2string(cur_state), source); + register_fsa_input_later(C_FSA_INTERNAL, I_FINALIZED, NULL); + return TRUE; + } +-- +1.8.3.1 + + +From 770cdaa5c477bfc75a6a38afc1d34f0d4f521ee1 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 13 Dec 2019 10:39:34 -0600 +Subject: [PATCH 09/11] Log: controller: improve CIB status deletion messages + +--- + crmd/utils.c | 25 +++++++++++++++++-------- + 1 file changed, 17 insertions(+), 8 deletions(-) + +diff --git a/crmd/utils.c b/crmd/utils.c +index 761f5a7..47aa6f0 100644 +--- a/crmd/utils.c ++++ b/crmd/utils.c +@@ -983,14 +983,18 @@ update_dc(xmlNode * msg) + return TRUE; + } + +-#define STATUS_PATH_MAX 512 + static void + erase_xpath_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) + { + char *xpath = user_data; + +- do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE, +- "Deletion of \"%s\": %s (rc=%d)", xpath, pcmk_strerror(rc), rc); ++ if (rc == 0) { ++ crm_debug("Deletion of '%s' from CIB (via CIB call %d) succeeded", ++ xpath, call_id); ++ } else { ++ crm_warn("Deletion of '%s' from CIB (via CIB call %d) failed: %s " ++ CRM_XS " rc=%d", xpath, call_id, pcmk_strerror(rc), rc); ++ } + } + + #define XPATH_STATUS_TAG "//node_state[@uname='%s']/%s" +@@ -998,14 +1002,19 @@ erase_xpath_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void + void + erase_status_tag(const char *uname, const char *tag, int options) + { +- if (fsa_cib_conn && uname) { ++ CRM_CHECK(uname != NULL, return); ++ ++ if (fsa_cib_conn == NULL) { ++ crm_warn("Unable to delete CIB '%s' section for node %s: " ++ "no CIB connection", tag, uname); ++ } else { + int call_id; + char *xpath = crm_strdup_printf(XPATH_STATUS_TAG, uname, tag); + +- crm_info("Deleting %s status entries for %s " CRM_XS " xpath=%s", +- tag, uname, xpath); +- call_id = fsa_cib_conn->cmds->delete(fsa_cib_conn, xpath, NULL, +- cib_quorum_override | cib_xpath | options); ++ options |= cib_quorum_override|cib_xpath; ++ call_id = fsa_cib_conn->cmds->delete(fsa_cib_conn, xpath, NULL, options); ++ crm_info("Deleting CIB '%s' section for node %s (via CIB call %d) " ++ CRM_XS " xpath=%s", tag, uname, call_id, xpath); + fsa_register_cib_callback(call_id, FALSE, xpath, erase_xpath_callback); + // CIB library handles freeing xpath + } +-- +1.8.3.1 + + +From 19d74b26b9b30157499c2fc4bd2da55c408fc638 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 13 Dec 2019 10:36:56 -0600 +Subject: [PATCH 10/11] Refactor: controller: move erase_status_tag() to + controld_based.c + +--- + crmd/cib.c | 38 ++++++++++++++++++++++++++++++++++++++ + crmd/utils.c | 37 ------------------------------------- + 2 files changed, 38 insertions(+), 37 deletions(-) + +diff --git a/crmd/cib.c b/crmd/cib.c +index a8a097a..ff489b4 100644 +--- a/crmd/cib.c ++++ b/crmd/cib.c +@@ -224,3 +224,41 @@ controld_action_is_recordable(const char *action) + } + return TRUE; + } ++ ++static void ++erase_xpath_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, ++ void *user_data) ++{ ++ char *xpath = user_data; ++ ++ if (rc == 0) { ++ crm_debug("Deletion of '%s' from CIB (via CIB call %d) succeeded", ++ xpath, call_id); ++ } else { ++ crm_warn("Deletion of '%s' from CIB (via CIB call %d) failed: %s " ++ CRM_XS " rc=%d", xpath, call_id, pcmk_strerror(rc), rc); ++ } ++} ++ ++#define XPATH_STATUS_TAG "//node_state[@uname='%s']/%s" ++ ++void ++erase_status_tag(const char *uname, const char *tag, int options) ++{ ++ CRM_CHECK(uname != NULL, return); ++ ++ if (fsa_cib_conn == NULL) { ++ crm_warn("Unable to delete CIB '%s' section for node %s: " ++ "no CIB connection", tag, uname); ++ } else { ++ int call_id; ++ char *xpath = crm_strdup_printf(XPATH_STATUS_TAG, uname, tag); ++ ++ options |= cib_quorum_override|cib_xpath; ++ call_id = fsa_cib_conn->cmds->delete(fsa_cib_conn, xpath, NULL, options); ++ crm_info("Deleting CIB '%s' section for node %s (via CIB call %d) " ++ CRM_XS " xpath=%s", tag, uname, call_id, xpath); ++ fsa_register_cib_callback(call_id, FALSE, xpath, erase_xpath_callback); ++ // CIB library handles freeing xpath ++ } ++} +diff --git a/crmd/utils.c b/crmd/utils.c +index 47aa6f0..b2a0b7d 100644 +--- a/crmd/utils.c ++++ b/crmd/utils.c +@@ -983,43 +983,6 @@ update_dc(xmlNode * msg) + return TRUE; + } + +-static void +-erase_xpath_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) +-{ +- char *xpath = user_data; +- +- if (rc == 0) { +- crm_debug("Deletion of '%s' from CIB (via CIB call %d) succeeded", +- xpath, call_id); +- } else { +- crm_warn("Deletion of '%s' from CIB (via CIB call %d) failed: %s " +- CRM_XS " rc=%d", xpath, call_id, pcmk_strerror(rc), rc); +- } +-} +- +-#define XPATH_STATUS_TAG "//node_state[@uname='%s']/%s" +- +-void +-erase_status_tag(const char *uname, const char *tag, int options) +-{ +- CRM_CHECK(uname != NULL, return); +- +- if (fsa_cib_conn == NULL) { +- crm_warn("Unable to delete CIB '%s' section for node %s: " +- "no CIB connection", tag, uname); +- } else { +- int call_id; +- char *xpath = crm_strdup_printf(XPATH_STATUS_TAG, uname, tag); +- +- options |= cib_quorum_override|cib_xpath; +- call_id = fsa_cib_conn->cmds->delete(fsa_cib_conn, xpath, NULL, options); +- crm_info("Deleting CIB '%s' section for node %s (via CIB call %d) " +- CRM_XS " xpath=%s", tag, uname, call_id, xpath); +- fsa_register_cib_callback(call_id, FALSE, xpath, erase_xpath_callback); +- // CIB library handles freeing xpath +- } +-} +- + void crmd_peer_down(crm_node_t *peer, bool full) + { + if(full && peer->state == NULL) { +-- +1.8.3.1 + + +From 8ba2bfa5aca514dcd2ad6c8a4f88ffedd028d206 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 13 Dec 2019 11:16:25 -0600 +Subject: [PATCH 11/11] Refactor: controller: improve efficiency when deleting + node state + +Rename erase_status_xpath() to controld_delete_node_state() to follow current +naming practice. + +Instead of passing it a node_state subsection name, pass a new enum value +indicating what to erase (resource history, transient node attributes, or +both). This allows us to improve the log messages further, as well as improving +efficiency when both need to be cleared. +--- + crmd/callbacks.c | 15 +++++------ + crmd/cib.c | 69 +++++++++++++++++++++++++++++++++++++++------------ + crmd/crmd_utils.h | 11 +++++++- + crmd/join_client.c | 3 ++- + crmd/join_dc.c | 3 ++- + crmd/lrm.c | 3 ++- + crmd/remote_lrmd_ra.c | 24 +++++++++--------- + crmd/te_actions.c | 5 ++-- + 8 files changed, 91 insertions(+), 42 deletions(-) + +diff --git a/crmd/callbacks.c b/crmd/callbacks.c +index 7560470..419b154 100644 +--- a/crmd/callbacks.c ++++ b/crmd/callbacks.c +@@ -202,17 +202,18 @@ peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *d + * transient attributes intact until it rejoins. + */ + if (compare_version(fsa_our_dc_version, "3.0.9") > 0) { +- erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local); ++ controld_delete_node_state(node->uname, ++ controld_section_attrs, ++ cib_scope_local); + } + + } else if(AM_I_DC) { +- if (appeared == FALSE) { +- crm_info("Peer %s left us", node->uname); +- erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local); +- } else { +- crm_info("New peer %s we want to sync fence history with", +- node->uname); ++ if (appeared) { + te_trigger_stonith_history_sync(FALSE); ++ } else { ++ controld_delete_node_state(node->uname, ++ controld_section_attrs, ++ cib_scope_local); + } + } + break; +diff --git a/crmd/cib.c b/crmd/cib.c +index ff489b4..c602130 100644 +--- a/crmd/cib.c ++++ b/crmd/cib.c +@@ -226,39 +226,76 @@ controld_action_is_recordable(const char *action) + } + + static void +-erase_xpath_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, +- void *user_data) ++cib_delete_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, ++ void *user_data) + { +- char *xpath = user_data; ++ char *desc = user_data; + + if (rc == 0) { +- crm_debug("Deletion of '%s' from CIB (via CIB call %d) succeeded", +- xpath, call_id); ++ crm_debug("Deletion of %s (via CIB call %d) succeeded", desc, call_id); + } else { +- crm_warn("Deletion of '%s' from CIB (via CIB call %d) failed: %s " +- CRM_XS " rc=%d", xpath, call_id, pcmk_strerror(rc), rc); ++ crm_warn("Deletion of %s (via CIB call %d) failed: %s " CRM_XS " rc=%d", ++ desc, call_id, pcmk_strerror(rc), rc); + } + } + +-#define XPATH_STATUS_TAG "//node_state[@uname='%s']/%s" ++// Searches for various portions of node_state to delete + ++// Match a particular node's node_state (takes node name 1x) ++#define XPATH_NODE_STATE "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']" ++ ++// Node's lrm section (name 1x) ++#define XPATH_NODE_LRM XPATH_NODE_STATE "/" XML_CIB_TAG_LRM ++ ++// Node's transient_attributes section (name 1x) ++#define XPATH_NODE_ATTRS XPATH_NODE_STATE "/" XML_TAG_TRANSIENT_NODEATTRS ++ ++// Everything under node_state (name 1x) ++#define XPATH_NODE_ALL XPATH_NODE_STATE "/*" ++ ++/*! ++ * \internal ++ * \brief Delete subsection of a node's CIB node_state ++ * ++ * \param[in] uname Desired node ++ * \param[in] section Subsection of node_state to delete ++ * \param[in] options CIB call options to use ++ */ + void +-erase_status_tag(const char *uname, const char *tag, int options) ++controld_delete_node_state(const char *uname, enum controld_section_e section, ++ int options) + { ++ char *xpath = NULL; ++ char *desc = NULL; ++ + CRM_CHECK(uname != NULL, return); ++ switch (section) { ++ case controld_section_lrm: ++ xpath = crm_strdup_printf(XPATH_NODE_LRM, uname); ++ desc = crm_strdup_printf("resource history for node %s", uname); ++ break; ++ case controld_section_attrs: ++ xpath = crm_strdup_printf(XPATH_NODE_ATTRS, uname); ++ desc = crm_strdup_printf("transient attributes for node %s", uname); ++ break; ++ case controld_section_all: ++ xpath = crm_strdup_printf(XPATH_NODE_ALL, uname); ++ desc = crm_strdup_printf("all state for node %s", uname); ++ break; ++ } + + if (fsa_cib_conn == NULL) { +- crm_warn("Unable to delete CIB '%s' section for node %s: " +- "no CIB connection", tag, uname); ++ crm_warn("Unable to delete %s: no CIB connection", desc); ++ free(desc); + } else { + int call_id; +- char *xpath = crm_strdup_printf(XPATH_STATUS_TAG, uname, tag); + + options |= cib_quorum_override|cib_xpath; + call_id = fsa_cib_conn->cmds->delete(fsa_cib_conn, xpath, NULL, options); +- crm_info("Deleting CIB '%s' section for node %s (via CIB call %d) " +- CRM_XS " xpath=%s", tag, uname, call_id, xpath); +- fsa_register_cib_callback(call_id, FALSE, xpath, erase_xpath_callback); +- // CIB library handles freeing xpath ++ crm_info("Deleting %s (via CIB call %d) " CRM_XS " xpath=%s", ++ desc, call_id, xpath); ++ fsa_register_cib_callback(call_id, FALSE, desc, cib_delete_callback); ++ // CIB library handles freeing desc + } ++ free(xpath); + } +diff --git a/crmd/crmd_utils.h b/crmd/crmd_utils.h +index 955d859..9afa2ca 100644 +--- a/crmd/crmd_utils.h ++++ b/crmd/crmd_utils.h +@@ -96,7 +96,6 @@ xmlNode *create_node_state_update(crm_node_t *node, int flags, + xmlNode *parent, const char *source); + void populate_cib_nodes(enum node_update_flags flags, const char *source); + void crm_update_quorum(gboolean quorum, gboolean force_update); +-void erase_status_tag(const char *uname, const char *tag, int options); + void update_attrd(const char *host, const char *name, const char *value, const char *user_name, gboolean is_remote_node); + void update_attrd_remote_node_removed(const char *host, const char *user_name); + void update_attrd_clear_failures(const char *host, const char *rsc, +@@ -115,6 +114,16 @@ void crmd_peer_down(crm_node_t *peer, bool full); + unsigned int cib_op_timeout(void); + bool controld_action_is_recordable(const char *action); + ++// Subsections of node_state ++enum controld_section_e { ++ controld_section_lrm, ++ controld_section_attrs, ++ controld_section_all, ++}; ++ ++void controld_delete_node_state(const char *uname, ++ enum controld_section_e section, int options); ++ + const char *get_node_id(xmlNode *lrm_rsc_op); + + /* Convenience macro for registering a CIB callback +diff --git a/crmd/join_client.c b/crmd/join_client.c +index 2142d21..9f572ad 100644 +--- a/crmd/join_client.c ++++ b/crmd/join_client.c +@@ -298,7 +298,8 @@ do_cl_join_finalize_respond(long long action, + * present for legacy attrd, but given legacy attrd's imminent + * demise, this is preferable to making intrusive changes to it. + */ +- erase_status_tag(fsa_our_uname, XML_TAG_TRANSIENT_NODEATTRS, 0); ++ controld_delete_node_state(fsa_our_uname, controld_section_attrs, ++ cib_scope_local); + update_attrd(fsa_our_uname, "terminate", NULL, NULL, FALSE); + update_attrd(fsa_our_uname, XML_CIB_ATTR_SHUTDOWN, "0", NULL, FALSE); + #endif +diff --git a/crmd/join_dc.c b/crmd/join_dc.c +index cdb3f77..6705022 100644 +--- a/crmd/join_dc.c ++++ b/crmd/join_dc.c +@@ -585,7 +585,8 @@ do_dc_join_ack(long long action, + /* Update CIB with node's current LRM state. A new transition will be + * triggered later, when the CIB notifies us of the change. + */ +- erase_status_tag(join_from, XML_CIB_TAG_LRM, cib_scope_local); ++ controld_delete_node_state(join_from, controld_section_lrm, ++ cib_scope_local); + if (safe_str_eq(join_from, fsa_our_uname)) { + xmlNode *now_dc_lrmd_state = do_lrm_query(TRUE, fsa_our_uname); + +diff --git a/crmd/lrm.c b/crmd/lrm.c +index 80fcd69..2c9e475 100644 +--- a/crmd/lrm.c ++++ b/crmd/lrm.c +@@ -1421,7 +1421,8 @@ force_reprobe(lrm_state_t *lrm_state, const char *from_sys, + } + + /* Now delete the copy in the CIB */ +- erase_status_tag(lrm_state->node_name, XML_CIB_TAG_LRM, cib_scope_local); ++ controld_delete_node_state(lrm_state->node_name, controld_section_lrm, ++ cib_scope_local); + + /* And finally, _delete_ the value in attrd + * Setting it to FALSE results in the PE sending us back here again +diff --git a/crmd/remote_lrmd_ra.c b/crmd/remote_lrmd_ra.c +index 1214814..c4f58d6 100644 +--- a/crmd/remote_lrmd_ra.c ++++ b/crmd/remote_lrmd_ra.c +@@ -195,13 +195,13 @@ remote_node_up(const char *node_name) + CRM_CHECK(node_name != NULL, return); + crm_info("Announcing pacemaker_remote node %s", node_name); + +- /* Clear node's operation history. The node's transient attributes should +- * and normally will be cleared when the node leaves, but since remote node +- * state has a number of corner cases, clear them here as well, to be sure. ++ /* Clear node's entire state (resource history and transient attributes). ++ * The transient attributes should and normally will be cleared when the ++ * node leaves, but since remote node state has a number of corner cases, ++ * clear them here as well, to be sure. + */ + call_opt = crmd_cib_smart_opt(); +- erase_status_tag(node_name, XML_CIB_TAG_LRM, call_opt); +- erase_status_tag(node_name, XML_TAG_TRANSIENT_NODEATTRS, call_opt); ++ controld_delete_node_state(node_name, controld_section_all, call_opt); + + /* Clear node's probed attribute */ + update_attrd(node_name, CRM_OP_PROBED, NULL, NULL, TRUE); +@@ -266,15 +266,15 @@ remote_node_down(const char *node_name, const enum down_opts opts) + /* Purge node from attrd's memory */ + update_attrd_remote_node_removed(node_name, NULL); + +- /* Purge node's transient attributes */ +- erase_status_tag(node_name, XML_TAG_TRANSIENT_NODEATTRS, call_opt); +- +- /* Normally, the LRM operation history should be kept until the node comes +- * back up. However, after a successful fence, we want to clear it, so we +- * don't think resources are still running on the node. ++ /* Normally, only node attributes should be erased, and the resource history ++ * should be kept until the node comes back up. However, after a successful ++ * fence, we want to clear the history as well, so we don't think resources ++ * are still running on the node. + */ + if (opts == DOWN_ERASE_LRM) { +- erase_status_tag(node_name, XML_CIB_TAG_LRM, call_opt); ++ controld_delete_node_state(node_name, controld_section_all, call_opt); ++ } else { ++ controld_delete_node_state(node_name, controld_section_attrs, call_opt); + } + + /* Ensure node is in the remote peer cache with lost state */ +diff --git a/crmd/te_actions.c b/crmd/te_actions.c +index 14097ab..19bb199 100644 +--- a/crmd/te_actions.c ++++ b/crmd/te_actions.c +@@ -150,9 +150,8 @@ send_stonith_update(crm_action_t * action, const char *target, const char *uuid) + /* Make sure it sticks */ + /* fsa_cib_conn->cmds->bump_epoch(fsa_cib_conn, cib_quorum_override|cib_scope_local); */ + +- erase_status_tag(peer->uname, XML_CIB_TAG_LRM, cib_scope_local); +- erase_status_tag(peer->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local); +- ++ controld_delete_node_state(peer->uname, controld_section_all, ++ cib_scope_local); + free_xml(node_state); + return; + } +-- +1.8.3.1 + diff --git a/SOURCES/06-refactors.patch b/SOURCES/06-refactors.patch new file mode 100644 index 0000000..c81e72a --- /dev/null +++ b/SOURCES/06-refactors.patch @@ -0,0 +1,2163 @@ +From 1ffb476ceb3be0522d3a344bc14614edca0e289e Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 16 Jan 2020 09:46:38 -0600 +Subject: [PATCH 01/13] Refactor: libcrmcommon: add more error codes + +This backports some minor changes from the master branch to make the file +easier to patch with later backports. +--- + include/crm/error.h | 40 +++++++++++++++++++--------------------- + lib/common/logging.c | 16 +++++++++++++++- + 2 files changed, 34 insertions(+), 22 deletions(-) + +diff --git a/include/crm/error.h b/include/crm/error.h +index cfa0cc5..7dad8ab 100644 +--- a/include/crm/error.h ++++ b/include/crm/error.h +@@ -1,37 +1,22 @@ + /* +- * Copyright (C) 2012 Andrew Beekhof <andrew@beekhof.net> ++ * Copyright 2012-2019 the Pacemaker project contributors + * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of the GNU Lesser General Public +- * License as published by the Free Software Foundation; either +- * version 2 of the License, or (at your option) any later version. ++ * The version control history for this file may have further details. + * +- * This software is distributed in the hope that it will be useful, +- * but WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * General Public License for more details. +- * +- * You should have received a copy of the GNU Lesser General Public +- * License along with this library; if not, write to the Free Software +- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA ++ * This source code is licensed under the GNU Lesser General Public License ++ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ + #ifndef CRM_ERROR__H + # define CRM_ERROR__H + # include <crm_config.h> + # include <assert.h> + +-/** ++/*! + * \file +- * \brief Error codes and asserts ++ * \brief Function and executable result codes + * \ingroup core + */ + +-/* +- System error codes +- - /usr/include/asm-generic/errno.h +- - /usr/include/asm-generic/errno-base.h +-*/ +- + # define CRM_ASSERT(expr) do { \ + if(__unlikely((expr) == FALSE)) { \ + crm_abort(__FILE__, __FUNCTION__, __LINE__, #expr, TRUE, FALSE); \ +@@ -39,6 +24,14 @@ + } \ + } while(0) + ++/* ++ * Function return codes ++ * ++ * For system error codes, see: ++ * - /usr/include/asm-generic/errno.h ++ * - /usr/include/asm-generic/errno-base.h ++ */ ++ + # define pcmk_ok 0 + # define PCMK_ERROR_OFFSET 190 /* Replacements on non-linux systems, see include/portability.h */ + # define PCMK_CUSTOM_OFFSET 200 /* Purely custom codes */ +@@ -54,6 +47,11 @@ + # define pcmk_err_cib_save 210 + # define pcmk_err_schema_unchanged 211 + # define pcmk_err_cib_corrupt 212 ++# define pcmk_err_multiple 213 ++# define pcmk_err_node_unknown 214 ++# define pcmk_err_already 215 ++# define pcmk_err_bad_nvpair 216 ++# define pcmk_err_unknown_format 217 + # define pcmk_err_panic 255 + + const char *pcmk_strerror(int rc); +diff --git a/lib/common/logging.c b/lib/common/logging.c +index c392468..3c8957c 100644 +--- a/lib/common/logging.c ++++ b/lib/common/logging.c +@@ -1137,6 +1137,11 @@ pcmk_errorname(int rc) + case pcmk_err_cib_backup: return "pcmk_err_cib_backup"; + case pcmk_err_cib_save: return "pcmk_err_cib_save"; + case pcmk_err_cib_corrupt: return "pcmk_err_cib_corrupt"; ++ case pcmk_err_multiple: return "pcmk_err_multiple"; ++ case pcmk_err_node_unknown: return "pcmk_err_node_unknown"; ++ case pcmk_err_already: return "pcmk_err_already"; ++ case pcmk_err_bad_nvpair: return "pcmk_err_bad_nvpair"; ++ case pcmk_err_unknown_format: return "pcmk_err_unknown_format"; + } + return "Unknown"; + } +@@ -1178,9 +1183,18 @@ pcmk_strerror(int rc) + return "Could not save the new configuration to disk"; + case pcmk_err_cib_corrupt: + return "Could not parse on-disk configuration"; +- ++ case pcmk_err_multiple: ++ return "Resource active on multiple nodes"; ++ case pcmk_err_node_unknown: ++ return "Node not found"; ++ case pcmk_err_already: ++ return "Situation already as requested"; ++ case pcmk_err_bad_nvpair: ++ return "Bad name/value pair given"; + case pcmk_err_schema_unchanged: + return "Schema is already the latest available"; ++ case pcmk_err_unknown_format: ++ return "Unknown output format"; + + /* The following cases will only be hit on systems for which they are non-standard */ + /* coverity[dead_error_condition] False positive on non-Linux */ +-- +1.8.3.1 + + +From 5da6f2708ed194ca0e3ed48b93efd85896a6760d Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 13 Dec 2019 16:05:05 -0600 +Subject: [PATCH 02/13] Refactor: libcrmcommon: introduce new set of return + codes + +Since we plan to introduce a high-level public API, it's a good time to +introduce some best practices. + +Most Pacemaker API functions currently return an integer return code, such that +its absolute value is either a system error number or a custom pcmk_err_* +number. This is less than ideal because system error numbers are constrained +only to the positive int range, so there's the possibility (though not noticed +in the wild) that system errors and custom errors could collide. + +The new method being introduced here still uses an integer return code, +but negative values are from a new enumeration, and positive values are +system error numbers. 0 still represents success. + +It is expected that the new method will be used with new functions, and +existing internal functions will be gradually refactored to use it as well. +Existing public API functions can be addressed at the next backward +compatibility break (2.1.0). +--- + include/crm/error.h | 58 ++++++- + lib/common/logging.c | 467 ++++++++++++++++++++++++++++++++++----------------- + tools/crm_error.c | 91 ++++++---- + 3 files changed, 433 insertions(+), 183 deletions(-) + +diff --git a/include/crm/error.h b/include/crm/error.h +index 7dad8ab..b632152 100644 +--- a/include/crm/error.h ++++ b/include/crm/error.h +@@ -1,5 +1,5 @@ + /* +- * Copyright 2012-2019 the Pacemaker project contributors ++ * Copyright 2012-2020 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -27,11 +27,21 @@ + /* + * Function return codes + * ++ * Most Pacemaker API functions return an integer return code. There are two ++ * alternative interpretations. The legacy interpration is that the absolute ++ * value of the return code is either a system error number or a custom ++ * pcmk_err_* number. This is less than ideal because system error numbers are ++ * constrained only to the positive int range, so there's the possibility ++ * (though not noticed in the wild) that system errors and custom errors could ++ * collide. The new intepretation is that negative values are from the pcmk_rc_e ++ * enum, and positive values are system error numbers. Both use 0 for success. ++ * + * For system error codes, see: + * - /usr/include/asm-generic/errno.h + * - /usr/include/asm-generic/errno-base.h + */ + ++// Legacy custom return codes for Pacemaker API functions + # define pcmk_ok 0 + # define PCMK_ERROR_OFFSET 190 /* Replacements on non-linux systems, see include/portability.h */ + # define PCMK_CUSTOM_OFFSET 200 /* Purely custom codes */ +@@ -54,6 +64,52 @@ + # define pcmk_err_unknown_format 217 + # define pcmk_err_panic 255 + ++/*! ++ * \enum pcmk_rc_e ++ * \brief Return codes for Pacemaker API functions ++ * ++ * Any Pacemaker API function documented as returning a "standard Pacemaker ++ * return code" will return pcmk_rc_ok (0) on success, and one of this ++ * enumeration's other (negative) values or a (positive) system error number ++ * otherwise. The custom codes are at -1001 and lower, so that the caller may ++ * use -1 through -1000 for their own custom values if desired. While generally ++ * referred to as "errors", nonzero values simply indicate a result, which might ++ * or might not be an error depending on the calling context. ++ */ ++enum pcmk_rc_e { ++ /* When adding new values, use consecutively lower numbers, update the array ++ * in lib/common/logging.c and test with crm_error. ++ */ ++ pcmk_rc_no_quorum = -1017, ++ pcmk_rc_schema_validation = -1016, ++ pcmk_rc_schema_unchanged = -1015, ++ pcmk_rc_transform_failed = -1014, ++ pcmk_rc_old_data = -1013, ++ pcmk_rc_diff_failed = -1012, ++ pcmk_rc_diff_resync = -1011, ++ pcmk_rc_cib_modified = -1010, ++ pcmk_rc_cib_backup = -1009, ++ pcmk_rc_cib_save = -1008, ++ pcmk_rc_cib_corrupt = -1007, ++ pcmk_rc_multiple = -1006, ++ pcmk_rc_node_unknown = -1005, ++ pcmk_rc_already = -1004, ++ pcmk_rc_bad_nvpair = -1003, ++ pcmk_rc_unknown_format = -1002, ++ // Developers: Use a more specific code than pcmk_rc_error whenever possible ++ pcmk_rc_error = -1001, ++ ++ // Values -1 through -1000 reserved for caller use ++ ++ pcmk_rc_ok = 0 ++ ++ // Positive values reserved for system error numbers ++}; ++ ++const char *pcmk_rc_name(int rc); ++const char *pcmk_rc_str(int rc); ++int pcmk_rc2legacy(int rc); ++int pcmk_legacy2rc(int legacy_rc); + const char *pcmk_strerror(int rc); + const char *pcmk_errorname(int rc); + const char *bz2_strerror(int rc); +diff --git a/lib/common/logging.c b/lib/common/logging.c +index 3c8957c..2357c1f 100644 +--- a/lib/common/logging.c ++++ b/lib/common/logging.c +@@ -1,5 +1,7 @@ + /* +- * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net> ++ * Copyright 2004-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public +@@ -984,148 +986,13 @@ crm_log_args(int argc, char **argv) + free(arg_string); + } + ++// @COMPAT Legacy function return codes ++ + const char * +-pcmk_errorname(int rc) ++pcmk_errorname(int rc) + { +- int error = ABS(rc); +- +- switch (error) { +- case E2BIG: return "E2BIG"; +- case EACCES: return "EACCES"; +- case EADDRINUSE: return "EADDRINUSE"; +- case EADDRNOTAVAIL: return "EADDRNOTAVAIL"; +- case EAFNOSUPPORT: return "EAFNOSUPPORT"; +- case EAGAIN: return "EAGAIN"; +- case EALREADY: return "EALREADY"; +- case EBADF: return "EBADF"; +- case EBADMSG: return "EBADMSG"; +- case EBUSY: return "EBUSY"; +- case ECANCELED: return "ECANCELED"; +- case ECHILD: return "ECHILD"; +- case ECOMM: return "ECOMM"; +- case ECONNABORTED: return "ECONNABORTED"; +- case ECONNREFUSED: return "ECONNREFUSED"; +- case ECONNRESET: return "ECONNRESET"; +- /* case EDEADLK: return "EDEADLK"; */ +- case EDESTADDRREQ: return "EDESTADDRREQ"; +- case EDOM: return "EDOM"; +- case EDQUOT: return "EDQUOT"; +- case EEXIST: return "EEXIST"; +- case EFAULT: return "EFAULT"; +- case EFBIG: return "EFBIG"; +- case EHOSTDOWN: return "EHOSTDOWN"; +- case EHOSTUNREACH: return "EHOSTUNREACH"; +- case EIDRM: return "EIDRM"; +- case EILSEQ: return "EILSEQ"; +- case EINPROGRESS: return "EINPROGRESS"; +- case EINTR: return "EINTR"; +- case EINVAL: return "EINVAL"; +- case EIO: return "EIO"; +- case EISCONN: return "EISCONN"; +- case EISDIR: return "EISDIR"; +- case ELIBACC: return "ELIBACC"; +- case ELOOP: return "ELOOP"; +- case EMFILE: return "EMFILE"; +- case EMLINK: return "EMLINK"; +- case EMSGSIZE: return "EMSGSIZE"; +-#ifdef EMULTIHOP // Not available on OpenBSD +- case EMULTIHOP: return "EMULTIHOP"; +-#endif +- case ENAMETOOLONG: return "ENAMETOOLONG"; +- case ENETDOWN: return "ENETDOWN"; +- case ENETRESET: return "ENETRESET"; +- case ENETUNREACH: return "ENETUNREACH"; +- case ENFILE: return "ENFILE"; +- case ENOBUFS: return "ENOBUFS"; +- case ENODATA: return "ENODATA"; +- case ENODEV: return "ENODEV"; +- case ENOENT: return "ENOENT"; +- case ENOEXEC: return "ENOEXEC"; +- case ENOKEY: return "ENOKEY"; +- case ENOLCK: return "ENOLCK"; +-#ifdef ENOLINK // Not available on OpenBSD +- case ENOLINK: return "ENOLINK"; +-#endif +- case ENOMEM: return "ENOMEM"; +- case ENOMSG: return "ENOMSG"; +- case ENOPROTOOPT: return "ENOPROTOOPT"; +- case ENOSPC: return "ENOSPC"; +- case ENOSR: return "ENOSR"; +- case ENOSTR: return "ENOSTR"; +- case ENOSYS: return "ENOSYS"; +- case ENOTBLK: return "ENOTBLK"; +- case ENOTCONN: return "ENOTCONN"; +- case ENOTDIR: return "ENOTDIR"; +- case ENOTEMPTY: return "ENOTEMPTY"; +- case ENOTSOCK: return "ENOTSOCK"; +- /* case ENOTSUP: return "ENOTSUP"; */ +- case ENOTTY: return "ENOTTY"; +- case ENOTUNIQ: return "ENOTUNIQ"; +- case ENXIO: return "ENXIO"; +- case EOPNOTSUPP: return "EOPNOTSUPP"; +- case EOVERFLOW: return "EOVERFLOW"; +- case EPERM: return "EPERM"; +- case EPFNOSUPPORT: return "EPFNOSUPPORT"; +- case EPIPE: return "EPIPE"; +- case EPROTO: return "EPROTO"; +- case EPROTONOSUPPORT: return "EPROTONOSUPPORT"; +- case EPROTOTYPE: return "EPROTOTYPE"; +- case ERANGE: return "ERANGE"; +- case EREMOTE: return "EREMOTE"; +- case EREMOTEIO: return "EREMOTEIO"; +- +- case EROFS: return "EROFS"; +- case ESHUTDOWN: return "ESHUTDOWN"; +- case ESPIPE: return "ESPIPE"; +- case ESOCKTNOSUPPORT: return "ESOCKTNOSUPPORT"; +- case ESRCH: return "ESRCH"; +- case ESTALE: return "ESTALE"; +- case ETIME: return "ETIME"; +- case ETIMEDOUT: return "ETIMEDOUT"; +- case ETXTBSY: return "ETXTBSY"; +- case EUNATCH: return "EUNATCH"; +- case EUSERS: return "EUSERS"; +- /* case EWOULDBLOCK: return "EWOULDBLOCK"; */ +- case EXDEV: return "EXDEV"; +- +-#ifdef EBADE +- /* Not available on OSX */ +- case EBADE: return "EBADE"; +- case EBADFD: return "EBADFD"; +- case EBADSLT: return "EBADSLT"; +- case EDEADLOCK: return "EDEADLOCK"; +- case EBADR: return "EBADR"; +- case EBADRQC: return "EBADRQC"; +- case ECHRNG: return "ECHRNG"; +-#ifdef EISNAM /* Not available on Illumos/Solaris */ +- case EISNAM: return "EISNAM"; +- case EKEYEXPIRED: return "EKEYEXPIRED"; +- case EKEYREJECTED: return "EKEYREJECTED"; +- case EKEYREVOKED: return "EKEYREVOKED"; +-#endif +- case EL2HLT: return "EL2HLT"; +- case EL2NSYNC: return "EL2NSYNC"; +- case EL3HLT: return "EL3HLT"; +- case EL3RST: return "EL3RST"; +- case ELIBBAD: return "ELIBBAD"; +- case ELIBMAX: return "ELIBMAX"; +- case ELIBSCN: return "ELIBSCN"; +- case ELIBEXEC: return "ELIBEXEC"; +-#ifdef ENOMEDIUM /* Not available on Illumos/Solaris */ +- case ENOMEDIUM: return "ENOMEDIUM"; +- case EMEDIUMTYPE: return "EMEDIUMTYPE"; +-#endif +- case ENONET: return "ENONET"; +- case ENOPKG: return "ENOPKG"; +- case EREMCHG: return "EREMCHG"; +- case ERESTART: return "ERESTART"; +- case ESTRPIPE: return "ESTRPIPE"; +-#ifdef EUCLEAN /* Not available on Illumos/Solaris */ +- case EUCLEAN: return "EUCLEAN"; +-#endif +- case EXFULL: return "EXFULL"; +-#endif +- ++ rc = abs(rc); ++ switch (rc) { + case pcmk_err_generic: return "pcmk_err_generic"; + case pcmk_err_no_quorum: return "pcmk_err_no_quorum"; + case pcmk_err_schema_validation: return "pcmk_err_schema_validation"; +@@ -1142,25 +1009,25 @@ pcmk_errorname(int rc) + case pcmk_err_already: return "pcmk_err_already"; + case pcmk_err_bad_nvpair: return "pcmk_err_bad_nvpair"; + case pcmk_err_unknown_format: return "pcmk_err_unknown_format"; ++ default: return pcmk_rc_name(rc); // system errno + } +- return "Unknown"; + } + +- + const char * + pcmk_strerror(int rc) + { +- int error = abs(rc); +- +- if (error == 0) { ++ if (rc == 0) { + return "OK"; ++ } + +- // Of course error > 0 ... unless someone passed INT_MIN as rc +- } else if ((error > 0) && (error < PCMK_ERROR_OFFSET)) { +- return strerror(error); ++ rc = abs(rc); ++ ++ // Of course rc > 0 ... unless someone passed INT_MIN as rc ++ if ((rc > 0) && (rc < PCMK_ERROR_OFFSET)) { ++ return strerror(rc); + } + +- switch (error) { ++ switch (rc) { + case pcmk_err_generic: + return "Generic Pacemaker error"; + case pcmk_err_no_quorum: +@@ -1216,11 +1083,309 @@ pcmk_strerror(int rc) + case ENOKEY: + return "Required key not available"; + } +- + crm_err("Unknown error code: %d", rc); + return "Unknown error"; + } + ++// Standard Pacemaker API return codes ++ ++/* This array is used only for nonzero values of pcmk_rc_e. Its values must be ++ * kept in the exact reverse order of the enum value numbering (i.e. add new ++ * values to the end of the array). ++ */ ++static struct pcmk__rc_info { ++ const char *name; ++ const char *desc; ++ int legacy_rc; ++} pcmk__rcs[] = { ++ { "pcmk_rc_error", ++ "Error", ++ -pcmk_err_generic, ++ }, ++ { "pcmk_rc_unknown_format", ++ "Unknown output format", ++ -pcmk_err_unknown_format, ++ }, ++ { "pcmk_rc_bad_nvpair", ++ "Bad name/value pair given", ++ -pcmk_err_bad_nvpair, ++ }, ++ { "pcmk_rc_already", ++ "Already in requested state", ++ -pcmk_err_already, ++ }, ++ { "pcmk_rc_node_unknown", ++ "Node not found", ++ -pcmk_err_node_unknown, ++ }, ++ { "pcmk_rc_multiple", ++ "Resource active on multiple nodes", ++ -pcmk_err_multiple, ++ }, ++ { "pcmk_rc_cib_corrupt", ++ "Could not parse on-disk configuration", ++ -pcmk_err_cib_corrupt, ++ }, ++ { "pcmk_rc_cib_save", ++ "Could not save new configuration to disk", ++ -pcmk_err_cib_save, ++ }, ++ { "pcmk_rc_cib_backup", ++ "Could not archive previous configuration", ++ -pcmk_err_cib_backup, ++ }, ++ { "pcmk_rc_cib_modified", ++ "On-disk configuration was manually modified", ++ -pcmk_err_cib_modified, ++ }, ++ { "pcmk_rc_diff_resync", ++ "Application of update diff failed, requesting full refresh", ++ -pcmk_err_diff_resync, ++ }, ++ { "pcmk_rc_diff_failed", ++ "Application of update diff failed", ++ -pcmk_err_diff_failed, ++ }, ++ { "pcmk_rc_old_data", ++ "Update was older than existing configuration", ++ -pcmk_err_old_data, ++ }, ++ { "pcmk_rc_transform_failed", ++ "Schema transform failed", ++ -pcmk_err_transform_failed, ++ }, ++ { "pcmk_rc_schema_unchanged", ++ "Schema is already the latest available", ++ -pcmk_err_schema_unchanged, ++ }, ++ { "pcmk_rc_schema_validation", ++ "Update does not conform to the configured schema", ++ -pcmk_err_schema_validation, ++ }, ++ { "pcmk_rc_no_quorum", ++ "Operation requires quorum", ++ -pcmk_err_no_quorum, ++ }, ++}; ++ ++#define PCMK__N_RC (sizeof(pcmk__rcs) / sizeof(struct pcmk__rc_info)) ++ ++/*! ++ * \brief Get a return code constant name as a string ++ * ++ * \param[in] rc Integer return code to convert ++ * ++ * \return String of constant name corresponding to rc ++ */ ++const char * ++pcmk_rc_name(int rc) ++{ ++ if ((rc <= pcmk_rc_error) && ((pcmk_rc_error - rc) < PCMK__N_RC)) { ++ return pcmk__rcs[pcmk_rc_error - rc].name; ++ } ++ switch (rc) { ++ case pcmk_rc_ok: return "pcmk_rc_ok"; ++ case E2BIG: return "E2BIG"; ++ case EACCES: return "EACCES"; ++ case EADDRINUSE: return "EADDRINUSE"; ++ case EADDRNOTAVAIL: return "EADDRNOTAVAIL"; ++ case EAFNOSUPPORT: return "EAFNOSUPPORT"; ++ case EAGAIN: return "EAGAIN"; ++ case EALREADY: return "EALREADY"; ++ case EBADF: return "EBADF"; ++ case EBADMSG: return "EBADMSG"; ++ case EBUSY: return "EBUSY"; ++ case ECANCELED: return "ECANCELED"; ++ case ECHILD: return "ECHILD"; ++ case ECOMM: return "ECOMM"; ++ case ECONNABORTED: return "ECONNABORTED"; ++ case ECONNREFUSED: return "ECONNREFUSED"; ++ case ECONNRESET: return "ECONNRESET"; ++ /* case EDEADLK: return "EDEADLK"; */ ++ case EDESTADDRREQ: return "EDESTADDRREQ"; ++ case EDOM: return "EDOM"; ++ case EDQUOT: return "EDQUOT"; ++ case EEXIST: return "EEXIST"; ++ case EFAULT: return "EFAULT"; ++ case EFBIG: return "EFBIG"; ++ case EHOSTDOWN: return "EHOSTDOWN"; ++ case EHOSTUNREACH: return "EHOSTUNREACH"; ++ case EIDRM: return "EIDRM"; ++ case EILSEQ: return "EILSEQ"; ++ case EINPROGRESS: return "EINPROGRESS"; ++ case EINTR: return "EINTR"; ++ case EINVAL: return "EINVAL"; ++ case EIO: return "EIO"; ++ case EISCONN: return "EISCONN"; ++ case EISDIR: return "EISDIR"; ++ case ELIBACC: return "ELIBACC"; ++ case ELOOP: return "ELOOP"; ++ case EMFILE: return "EMFILE"; ++ case EMLINK: return "EMLINK"; ++ case EMSGSIZE: return "EMSGSIZE"; ++#ifdef EMULTIHOP // Not available on OpenBSD ++ case EMULTIHOP: return "EMULTIHOP"; ++#endif ++ case ENAMETOOLONG: return "ENAMETOOLONG"; ++ case ENETDOWN: return "ENETDOWN"; ++ case ENETRESET: return "ENETRESET"; ++ case ENETUNREACH: return "ENETUNREACH"; ++ case ENFILE: return "ENFILE"; ++ case ENOBUFS: return "ENOBUFS"; ++ case ENODATA: return "ENODATA"; ++ case ENODEV: return "ENODEV"; ++ case ENOENT: return "ENOENT"; ++ case ENOEXEC: return "ENOEXEC"; ++ case ENOKEY: return "ENOKEY"; ++ case ENOLCK: return "ENOLCK"; ++#ifdef ENOLINK // Not available on OpenBSD ++ case ENOLINK: return "ENOLINK"; ++#endif ++ case ENOMEM: return "ENOMEM"; ++ case ENOMSG: return "ENOMSG"; ++ case ENOPROTOOPT: return "ENOPROTOOPT"; ++ case ENOSPC: return "ENOSPC"; ++ case ENOSR: return "ENOSR"; ++ case ENOSTR: return "ENOSTR"; ++ case ENOSYS: return "ENOSYS"; ++ case ENOTBLK: return "ENOTBLK"; ++ case ENOTCONN: return "ENOTCONN"; ++ case ENOTDIR: return "ENOTDIR"; ++ case ENOTEMPTY: return "ENOTEMPTY"; ++ case ENOTSOCK: return "ENOTSOCK"; ++#if ENOTSUP != EOPNOTSUPP ++ case ENOTSUP: return "ENOTSUP"; ++#endif ++ case ENOTTY: return "ENOTTY"; ++ case ENOTUNIQ: return "ENOTUNIQ"; ++ case ENXIO: return "ENXIO"; ++ case EOPNOTSUPP: return "EOPNOTSUPP"; ++ case EOVERFLOW: return "EOVERFLOW"; ++ case EPERM: return "EPERM"; ++ case EPFNOSUPPORT: return "EPFNOSUPPORT"; ++ case EPIPE: return "EPIPE"; ++ case EPROTO: return "EPROTO"; ++ case EPROTONOSUPPORT: return "EPROTONOSUPPORT"; ++ case EPROTOTYPE: return "EPROTOTYPE"; ++ case ERANGE: return "ERANGE"; ++ case EREMOTE: return "EREMOTE"; ++ case EREMOTEIO: return "EREMOTEIO"; ++ case EROFS: return "EROFS"; ++ case ESHUTDOWN: return "ESHUTDOWN"; ++ case ESPIPE: return "ESPIPE"; ++ case ESOCKTNOSUPPORT: return "ESOCKTNOSUPPORT"; ++ case ESRCH: return "ESRCH"; ++ case ESTALE: return "ESTALE"; ++ case ETIME: return "ETIME"; ++ case ETIMEDOUT: return "ETIMEDOUT"; ++ case ETXTBSY: return "ETXTBSY"; ++ case EUNATCH: return "EUNATCH"; ++ case EUSERS: return "EUSERS"; ++ /* case EWOULDBLOCK: return "EWOULDBLOCK"; */ ++ case EXDEV: return "EXDEV"; ++ ++#ifdef EBADE // Not available on OS X ++ case EBADE: return "EBADE"; ++ case EBADFD: return "EBADFD"; ++ case EBADSLT: return "EBADSLT"; ++ case EDEADLOCK: return "EDEADLOCK"; ++ case EBADR: return "EBADR"; ++ case EBADRQC: return "EBADRQC"; ++ case ECHRNG: return "ECHRNG"; ++#ifdef EISNAM // Not available on OS X, Illumos, Solaris ++ case EISNAM: return "EISNAM"; ++ case EKEYEXPIRED: return "EKEYEXPIRED"; ++ case EKEYREJECTED: return "EKEYREJECTED"; ++ case EKEYREVOKED: return "EKEYREVOKED"; ++#endif ++ case EL2HLT: return "EL2HLT"; ++ case EL2NSYNC: return "EL2NSYNC"; ++ case EL3HLT: return "EL3HLT"; ++ case EL3RST: return "EL3RST"; ++ case ELIBBAD: return "ELIBBAD"; ++ case ELIBMAX: return "ELIBMAX"; ++ case ELIBSCN: return "ELIBSCN"; ++ case ELIBEXEC: return "ELIBEXEC"; ++#ifdef ENOMEDIUM // Not available on OS X, Illumos, Solaris ++ case ENOMEDIUM: return "ENOMEDIUM"; ++ case EMEDIUMTYPE: return "EMEDIUMTYPE"; ++#endif ++ case ENONET: return "ENONET"; ++ case ENOPKG: return "ENOPKG"; ++ case EREMCHG: return "EREMCHG"; ++ case ERESTART: return "ERESTART"; ++ case ESTRPIPE: return "ESTRPIPE"; ++#ifdef EUCLEAN // Not available on OS X, Illumos, Solaris ++ case EUCLEAN: return "EUCLEAN"; ++#endif ++ case EXFULL: return "EXFULL"; ++#endif // EBADE ++ default: return "Unknown"; ++ } ++} ++ ++/*! ++ * \brief Get a user-friendly description of a return code ++ * ++ * \param[in] rc Integer return code to convert ++ * ++ * \return String description of rc ++ */ ++const char * ++pcmk_rc_str(int rc) ++{ ++ if (rc == pcmk_rc_ok) { ++ return "OK"; ++ } ++ if ((rc <= pcmk_rc_error) && ((pcmk_rc_error - rc) < PCMK__N_RC)) { ++ return pcmk__rcs[pcmk_rc_error - rc].desc; ++ } ++ if (rc < 0) { ++ return "Unknown error"; ++ } ++ return strerror(rc); ++} ++ ++// This returns negative values for errors ++int ++pcmk_rc2legacy(int rc) ++{ ++ if (rc >= 0) { ++ return -rc; // OK or system errno ++ } ++ if ((rc <= pcmk_rc_error) && ((pcmk_rc_error - rc) < PCMK__N_RC)) { ++ return pcmk__rcs[pcmk_rc_error - rc].legacy_rc; ++ } ++ return -pcmk_err_generic; ++} ++ ++int ++pcmk_legacy2rc(int legacy_rc) ++{ ++ legacy_rc = abs(legacy_rc); ++ switch (legacy_rc) { ++ case pcmk_err_no_quorum: return pcmk_rc_no_quorum; ++ case pcmk_err_schema_validation: return pcmk_rc_schema_validation; ++ case pcmk_err_schema_unchanged: return pcmk_rc_schema_unchanged; ++ case pcmk_err_transform_failed: return pcmk_rc_transform_failed; ++ case pcmk_err_old_data: return pcmk_rc_old_data; ++ case pcmk_err_diff_failed: return pcmk_rc_diff_failed; ++ case pcmk_err_diff_resync: return pcmk_rc_diff_resync; ++ case pcmk_err_cib_modified: return pcmk_rc_cib_modified; ++ case pcmk_err_cib_backup: return pcmk_rc_cib_backup; ++ case pcmk_err_cib_save: return pcmk_rc_cib_save; ++ case pcmk_err_cib_corrupt: return pcmk_rc_cib_corrupt; ++ case pcmk_err_multiple: return pcmk_rc_multiple; ++ case pcmk_err_node_unknown: return pcmk_rc_node_unknown; ++ case pcmk_err_already: return pcmk_rc_already; ++ case pcmk_err_bad_nvpair: return pcmk_rc_bad_nvpair; ++ case pcmk_err_unknown_format: return pcmk_rc_unknown_format; ++ case pcmk_err_generic: return pcmk_rc_error; ++ case pcmk_ok: return pcmk_rc_ok; ++ default: return legacy_rc; // system errno ++ } ++} ++ + const char * + bz2_strerror(int rc) + { +diff --git a/tools/crm_error.c b/tools/crm_error.c +index bd75a8f..fddd916 100644 +--- a/tools/crm_error.c ++++ b/tools/crm_error.c +@@ -1,19 +1,10 @@ +-/* +- * Copyright (C) 2012 Andrew Beekhof <andrew@beekhof.net> +- * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of the GNU General Public +- * License as published by the Free Software Foundation; either +- * version 2 of the License, or (at your option) any later version. +- * +- * This software is distributed in the hope that it will be useful, +- * but WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * General Public License for more details. +- * +- * You should have received a copy of the GNU General Public +- * License along with this library; if not, write to the Free Software +- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA ++/* ++ * Copyright 2012-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. ++ * ++ * This source code is licensed under the GNU General Public License version 2 ++ * or later (GPLv2+) WITHOUT ANY WARRANTY. + */ + + #include <crm_internal.h> +@@ -31,11 +22,26 @@ static struct crm_option long_options[] = { + "\n\t\t\tUseful for looking for sources of the error in source code"}, + + {"list", 0, 0, 'l', "\tShow all known errors."}, ++ {"rc", 0, 0, 'r', "\tInterpret as return code rather than legacy function return value"}, + + {0, 0, 0, 0} + }; + /* *INDENT-ON* */ + ++static bool as_rc = false; ++ ++static void ++get_strings(int rc, const char **name, const char **str) ++{ ++ if (as_rc) { ++ *str = pcmk_rc_str(rc); ++ *name = pcmk_rc_name(rc); ++ } else { ++ *str = pcmk_strerror(rc); ++ *name = pcmk_errorname(rc); ++ } ++} ++ + int + main(int argc, char **argv) + { +@@ -47,8 +53,11 @@ main(int argc, char **argv) + bool do_list = FALSE; + bool with_name = FALSE; + ++ const char *name = NULL; ++ const char *desc = NULL; ++ + crm_log_cli_init("crm_error"); +- crm_set_options(NULL, "[options] -- rc", long_options, ++ crm_set_options(NULL, "[options] -- <rc> [...]", long_options, + "Tool for displaying the textual name or description of a reported error code"); + + while (flag >= 0) { +@@ -69,6 +78,9 @@ main(int argc, char **argv) + case 'l': + do_list = TRUE; + break; ++ case 'r': ++ as_rc = true; ++ break; + default: + crm_help(flag, EX_OK); + break; +@@ -76,26 +88,43 @@ main(int argc, char **argv) + } + + if(do_list) { +- for (rc = 0; rc < 256; rc++) { +- const char *name = pcmk_errorname(rc); +- const char *desc = pcmk_strerror(rc); ++ int start, end, width; ++ ++ // 256 is a hacky magic number that "should" be enough ++ if (as_rc) { ++ start = pcmk_rc_error - 256; ++ end = PCMK_CUSTOM_OFFSET; ++ width = 4; ++ } else { ++ start = 0; ++ end = 256; ++ width = 3; ++ } ++ ++ for (rc = start; rc < end; rc++) { ++ if (rc == (pcmk_rc_error + 1)) { ++ // Values in between are reserved for callers, no use iterating ++ rc = pcmk_rc_ok; ++ } ++ get_strings(rc, &name, &desc); + if(name == NULL || strcmp("Unknown", name) == 0) { +- /* Unknown */ ++ // Undefined + } else if(with_name) { +- printf("%.3d: %-25s %s\n", rc, name, desc); ++ printf("% .*d: %-26s %s\n", width, rc, name, desc); + } else { +- printf("%.3d: %s\n", rc, desc); ++ printf("% .*d: %s\n", width, rc, desc); + } + } +- return 0; +- } + +- for (lpc = optind; lpc < argc; lpc++) { +- rc = crm_atoi(argv[lpc], NULL); +- if(with_name) { +- printf("%s - %s\n", pcmk_errorname(rc), pcmk_strerror(rc)); +- } else { +- printf("%s\n", pcmk_strerror(rc)); ++ } else { ++ for (lpc = optind; lpc < argc; lpc++) { ++ rc = crm_atoi(argv[lpc], NULL); ++ get_strings(rc, &name, &desc); ++ if (with_name) { ++ printf("%s - %s\n", name, desc); ++ } else { ++ printf("%s\n", desc); ++ } + } + } + return 0; +-- +1.8.3.1 + + +From b3875680199d116ddfd5ad2c4dac2c8f984866ae Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 2 Jan 2020 09:51:52 -0600 +Subject: [PATCH 03/13] Refactor: attrd: properly declare global variables as + extern in header + +Restores buildability with GCC 10 +--- + attrd/internal.h | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/attrd/internal.h b/attrd/internal.h +index e574060..b6da14c 100644 +--- a/attrd/internal.h ++++ b/attrd/internal.h +@@ -1,5 +1,7 @@ + /* +- * Copyright (C) 2013-2017 Andrew Beekhof <andrew@beekhof.net> ++ * Copyright 2013-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public +@@ -59,8 +61,8 @@ typedef struct attribute_value_s { + gboolean seen; + } attribute_value_t; + +-crm_cluster_t *attrd_cluster; +-GHashTable *attributes; ++extern crm_cluster_t *attrd_cluster; ++extern GHashTable *attributes; + + #define attrd_send_ack(client, id, flags) \ + crm_ipcs_send_ack((client), (id), (flags), "ack", __FUNCTION__, __LINE__) +-- +1.8.3.1 + + +From fc15f7e587fb0bff31ae25851d69f11ea240da9e Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ferenc=20W=C3=A1gner?= <wferi@debian.org> +Date: Tue, 7 Jan 2020 11:58:34 +0100 +Subject: [PATCH 04/13] Avoid clashes with glibc error codes on HPPA + +In principle this is an ABI change, but the affected versions (2.0.2 and +later) were impossible to build due to duplicate case values in +lib/common/results.c. + +https://bugs.clusterlabs.org/show_bug.cgi?id=5419 +--- + include/crm/error.h | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/include/crm/error.h b/include/crm/error.h +index b632152..5b24641 100644 +--- a/include/crm/error.h ++++ b/include/crm/error.h +@@ -60,8 +60,14 @@ + # define pcmk_err_multiple 213 + # define pcmk_err_node_unknown 214 + # define pcmk_err_already 215 ++/* On HPPA 215 is ENOSYM (Unknown error 215), which hopefully never happens. */ ++#ifdef __hppa__ ++# define pcmk_err_bad_nvpair 250 /* 216 is ENOTSOCK */ ++# define pcmk_err_unknown_format 252 /* 217 is EDESTADDRREQ */ ++#else + # define pcmk_err_bad_nvpair 216 + # define pcmk_err_unknown_format 217 ++#endif + # define pcmk_err_panic 255 + + /*! +-- +1.8.3.1 + + +From c4d6c29917bf0c04c1cb0d822d11a6e609c322d7 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Tue, 14 Jan 2020 12:52:21 -0600 +Subject: [PATCH 05/13] Log: controller: improve messages when deleting CIB + resource history + +This also moves delete_rsc_status() to controld_based.c and renames it. +--- + crmd/cib.c | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ + crmd/crmd_utils.h | 6 ++++- + crmd/lrm.c | 50 +++++++-------------------------------- + 3 files changed, 85 insertions(+), 42 deletions(-) + +diff --git a/crmd/cib.c b/crmd/cib.c +index c602130..e8c6376 100644 +--- a/crmd/cib.c ++++ b/crmd/cib.c +@@ -299,3 +299,74 @@ controld_delete_node_state(const char *uname, enum controld_section_e section, + } + free(xpath); + } ++ ++// Takes node name and resource ID ++#define XPATH_RESOURCE_HISTORY "//" XML_CIB_TAG_STATE \ ++ "[@" XML_ATTR_UNAME "='%s'] /" \ ++ XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \ ++ "/" XML_LRM_TAG_RESOURCE \ ++ "[@" XML_ATTR_ID "='%s']" ++// @TODO could add "and @XML_CONFIG_ATTR_SHUTDOWN_LOCK" to limit to locks ++ ++/*! ++ * \internal ++ * \brief Clear resource history from CIB for a given resource and node ++ * ++ * \param[in] rsc_id ID of resource to be cleared ++ * \param[in] node Node whose resource history should be cleared ++ * \param[in] user_name ACL user name to use ++ * \param[in] call_options CIB call options ++ * ++ * \return Standard Pacemaker return code ++ */ ++int ++controld_delete_resource_history(const char *rsc_id, const char *node, ++ const char *user_name, int call_options) ++{ ++ char *desc = NULL; ++ char *xpath = NULL; ++ int rc = pcmk_rc_ok; ++ ++ CRM_CHECK((rsc_id != NULL) && (node != NULL), return EINVAL); ++ ++ desc = crm_strdup_printf("resource history for %s on %s", rsc_id, node); ++ if (fsa_cib_conn == NULL) { ++ crm_err("Unable to clear %s: no CIB connection", desc); ++ free(desc); ++ return ENOTCONN; ++ } ++ ++ // Ask CIB to delete the entry ++ xpath = crm_strdup_printf(XPATH_RESOURCE_HISTORY, node, rsc_id); ++ rc = cib_internal_op(fsa_cib_conn, CIB_OP_DELETE, NULL, xpath, NULL, ++ NULL, call_options|cib_xpath, user_name); ++ ++ if (rc < 0) { ++ rc = pcmk_legacy2rc(rc); ++ crm_err("Could not delete resource status of %s on %s%s%s: %s " ++ CRM_XS " rc=%d", rsc_id, node, ++ (user_name? " for user " : ""), (user_name? user_name : ""), ++ pcmk_rc_str(rc), rc); ++ free(desc); ++ free(xpath); ++ return rc; ++ } ++ ++ if (is_set(call_options, cib_sync_call)) { ++ if (is_set(call_options, cib_dryrun)) { ++ crm_debug("Deletion of %s would succeed", desc); ++ } else { ++ crm_debug("Deletion of %s succeeded", desc); ++ } ++ free(desc); ++ ++ } else { ++ crm_info("Clearing %s (via CIB call %d) " CRM_XS " xpath=%s", ++ desc, rc, xpath); ++ fsa_register_cib_callback(rc, FALSE, desc, cib_delete_callback); ++ // CIB library handles freeing desc ++ } ++ ++ free(xpath); ++ return pcmk_rc_ok; ++} +diff --git a/crmd/crmd_utils.h b/crmd/crmd_utils.h +index 9afa2ca..eeaa8b7 100644 +--- a/crmd/crmd_utils.h ++++ b/crmd/crmd_utils.h +@@ -1,5 +1,7 @@ + /* +- * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net> ++ * Copyright 2004-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public +@@ -123,6 +125,8 @@ enum controld_section_e { + + void controld_delete_node_state(const char *uname, + enum controld_section_e section, int options); ++int controld_delete_resource_history(const char *rsc_id, const char *node, ++ const char *user_name, int call_options); + + const char *get_node_id(xmlNode *lrm_rsc_op); + +diff --git a/crmd/lrm.c b/crmd/lrm.c +index 2c9e475..f9af502 100644 +--- a/crmd/lrm.c ++++ b/crmd/lrm.c +@@ -39,8 +39,6 @@ struct delete_event_s { + static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id); + static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list); + static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data); +-static int delete_rsc_status(lrm_state_t * lrm_state, const char *rsc_id, int call_options, +- const char *user_name); + + static lrmd_event_data_t *construct_op(lrm_state_t * lrm_state, xmlNode * rsc_op, + const char *rsc_id, const char *operation); +@@ -178,7 +176,8 @@ update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_ + + if (op->rsc_deleted) { + crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type); +- delete_rsc_status(lrm_state, op->rsc_id, cib_quorum_override, NULL); ++ controld_delete_resource_history(op->rsc_id, lrm_state->node_name, ++ NULL, crmd_cib_smart_opt()); + return; + } + +@@ -927,34 +926,6 @@ lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data) + return FALSE; + } + +-/* +- * Remove the rsc from the CIB +- * +- * Avoids refreshing the entire LRM section of this host +- */ +-#define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']" +- +-static int +-delete_rsc_status(lrm_state_t * lrm_state, const char *rsc_id, int call_options, +- const char *user_name) +-{ +- char *rsc_xpath = NULL; +- int max = 0; +- int rc = pcmk_ok; +- +- CRM_CHECK(rsc_id != NULL, return -ENXIO); +- +- max = strlen(rsc_template) + strlen(lrm_state->node_name) + strlen(rsc_id) + 1; +- rsc_xpath = calloc(1, max); +- snprintf(rsc_xpath, max, rsc_template, lrm_state->node_name, rsc_id); +- +- rc = cib_internal_op(fsa_cib_conn, CIB_OP_DELETE, NULL, rsc_xpath, +- NULL, NULL, call_options | cib_xpath, user_name); +- +- free(rsc_xpath); +- return rc; +-} +- + static void + delete_rsc_entry(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, + GHashTableIter * rsc_gIter, int rc, const char *user_name) +@@ -971,7 +942,8 @@ delete_rsc_entry(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rs + else + g_hash_table_remove(lrm_state->resource_history, rsc_id_copy); + crm_debug("sync: Sending delete op for %s", rsc_id_copy); +- delete_rsc_status(lrm_state, rsc_id_copy, cib_quorum_override, user_name); ++ controld_delete_resource_history(rsc_id_copy, lrm_state->node_name, ++ user_name, crmd_cib_smart_opt()); + + g_hash_table_foreach_remove(lrm_state->pending_ops, lrm_remove_deleted_op, rsc_id_copy); + free(rsc_id_copy); +@@ -1692,21 +1664,17 @@ do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state, + gboolean unregister = TRUE; + + #if ENABLE_ACL +- int cib_rc = delete_rsc_status(lrm_state, rsc->id, +- cib_dryrun|cib_sync_call, user_name); ++ int cib_rc = controld_delete_resource_history(rsc->id, lrm_state->node_name, ++ user_name, ++ cib_dryrun|cib_sync_call); + +- if (cib_rc != pcmk_ok) { ++ if (cib_rc != pcmk_rc_ok) { + lrmd_event_data_t *op = NULL; + +- crm_err("Could not delete resource status of %s for %s (user %s) on %s: %s" +- CRM_XS " rc=%d", +- rsc->id, from_sys, (user_name? user_name : "unknown"), +- from_host, pcmk_strerror(cib_rc), cib_rc); +- + op = construct_op(lrm_state, input->xml, rsc->id, CRMD_ACTION_DELETE); + op->op_status = PCMK_LRM_OP_ERROR; + +- if (cib_rc == -EACCES) { ++ if (cib_rc == EACCES) { + op->rc = PCMK_OCF_INSUFFICIENT_PRIV; + } else { + op->rc = PCMK_OCF_UNKNOWN_ERROR; +-- +1.8.3.1 + + +From f58eb3eb1939cac5dbff4321900423a8041e07e5 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Mon, 6 Jan 2020 11:01:38 -0600 +Subject: [PATCH 06/13] Refactor: controller: remove unused function arguments + +... and rename affected functions +--- + crmd/crmd_fsa.h | 9 ++++++--- + crmd/fsa.c | 1 - + crmd/fsa_proto.h | 2 -- + crmd/join_client.c | 6 ++++-- + crmd/join_dc.c | 34 ++++++++++++++++------------------ + crmd/lrm.c | 2 +- + 6 files changed, 27 insertions(+), 27 deletions(-) + +diff --git a/crmd/crmd_fsa.h b/crmd/crmd_fsa.h +index 7da9545..53fee91 100644 +--- a/crmd/crmd_fsa.h ++++ b/crmd/crmd_fsa.h +@@ -1,5 +1,7 @@ + /* +- * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net> ++ * Copyright 2004-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public +@@ -112,9 +114,10 @@ extern struct crm_subsystem_s *cib_subsystem; + extern struct crm_subsystem_s *te_subsystem; + extern struct crm_subsystem_s *pe_subsystem; + +-/* these two should be moved elsewhere... */ +-extern void do_update_cib_nodes(gboolean overwrite, const char *caller); ++// These should be moved elsewhere ++void do_update_cib_nodes(gboolean overwrite, const char *caller); + int crmd_cib_smart_opt(void); ++xmlNode *controld_query_executor_state(const char *node_name); + + # define AM_I_DC is_set(fsa_input_register, R_THE_DC) + # define AM_I_OPERATIONAL (is_set(fsa_input_register, R_STARTING)==FALSE) +diff --git a/crmd/fsa.c b/crmd/fsa.c +index 7f6a3ac..90c4577 100644 +--- a/crmd/fsa.c ++++ b/crmd/fsa.c +@@ -67,7 +67,6 @@ volatile enum crmd_fsa_state fsa_state = S_STARTING; + + extern uint highest_born_on; + extern uint num_join_invites; +-extern void initialize_join(gboolean before); + + #define DOT_PREFIX "actions:trace: " + #define do_dot_log(fmt, args...) crm_trace( fmt, ##args) +diff --git a/crmd/fsa_proto.h b/crmd/fsa_proto.h +index 9d55415..c8a202e 100644 +--- a/crmd/fsa_proto.h ++++ b/crmd/fsa_proto.h +@@ -19,8 +19,6 @@ + #ifndef XML_FSA_PROTO__H + # define XML_FSA_PROTO__H + +-extern xmlNode *do_lrm_query(gboolean, const char *node_name); +- + /* A_READCONFIG */ + void + +diff --git a/crmd/join_client.c b/crmd/join_client.c +index 9f572ad..ea10f0f 100644 +--- a/crmd/join_client.c ++++ b/crmd/join_client.c +@@ -1,5 +1,7 @@ + /* +- * Copyright 2004-2018 Andrew Beekhof <andrew@beekhof.net> ++ * Copyright 2004-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. +@@ -268,7 +270,7 @@ do_cl_join_finalize_respond(long long action, + update_dc_expected(input->msg); + + /* send our status section to the DC */ +- tmp1 = do_lrm_query(TRUE, fsa_our_uname); ++ tmp1 = controld_query_executor_state(fsa_our_uname); + if (tmp1 != NULL) { + xmlNode *reply = create_request(CRM_OP_JOIN_CONFIRM, tmp1, fsa_our_dc, + CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL); +diff --git a/crmd/join_dc.c b/crmd/join_dc.c +index 6705022..8284695 100644 +--- a/crmd/join_dc.c ++++ b/crmd/join_dc.c +@@ -1,5 +1,7 @@ + /* +- * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net> ++ * Copyright 2004-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public +@@ -31,7 +33,6 @@ char *max_epoch = NULL; + char *max_generation_from = NULL; + xmlNode *max_generation_xml = NULL; + +-void initialize_join(gboolean before); + void finalize_join_for(gpointer key, gpointer value, gpointer user_data); + void finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data); + gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source); +@@ -78,8 +79,8 @@ crm_update_peer_join(const char *source, crm_node_t * node, enum crm_join_phase + } + } + +-void +-initialize_join(gboolean before) ++static void ++start_join_round() + { + GHashTableIter iter; + crm_node_t *peer = NULL; +@@ -90,19 +91,16 @@ initialize_join(gboolean before) + while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) { + crm_update_peer_join(__FUNCTION__, peer, crm_join_none); + } +- +- if (before) { +- if (max_generation_from != NULL) { +- free(max_generation_from); +- max_generation_from = NULL; +- } +- if (max_generation_xml != NULL) { +- free_xml(max_generation_xml); +- max_generation_xml = NULL; +- } +- clear_bit(fsa_input_register, R_HAVE_CIB); +- clear_bit(fsa_input_register, R_CIB_ASKED); ++ if (max_generation_from != NULL) { ++ free(max_generation_from); ++ max_generation_from = NULL; ++ } ++ if (max_generation_xml != NULL) { ++ free_xml(max_generation_xml); ++ max_generation_xml = NULL; + } ++ clear_bit(fsa_input_register, R_HAVE_CIB); ++ clear_bit(fsa_input_register, R_CIB_ASKED); + } + + /*! +@@ -200,7 +198,7 @@ do_dc_join_offer_all(long long action, + * will be seen as offline by the PE anyway + */ + current_join_id++; +- initialize_join(TRUE); ++ start_join_round(); + /* do_update_cib_nodes(TRUE, __FUNCTION__); */ + + update_dc(NULL); +@@ -588,7 +586,7 @@ do_dc_join_ack(long long action, + controld_delete_node_state(join_from, controld_section_lrm, + cib_scope_local); + if (safe_str_eq(join_from, fsa_our_uname)) { +- xmlNode *now_dc_lrmd_state = do_lrm_query(TRUE, fsa_our_uname); ++ xmlNode *now_dc_lrmd_state = controld_query_executor_state(fsa_our_uname); + + if (now_dc_lrmd_state != NULL) { + fsa_cib_update(XML_CIB_TAG_STATUS, now_dc_lrmd_state, +diff --git a/crmd/lrm.c b/crmd/lrm.c +index f9af502..ea0d1bb 100644 +--- a/crmd/lrm.c ++++ b/crmd/lrm.c +@@ -836,7 +836,7 @@ do_lrm_query_internal(lrm_state_t *lrm_state, int update_flags) + } + + xmlNode * +-do_lrm_query(gboolean is_replace, const char *node_name) ++controld_query_executor_state(const char *node_name) + { + lrm_state_t *lrm_state = lrm_state_find(node_name); + xmlNode *xml_state; +-- +1.8.3.1 + + +From 5edb8a903ccf29c028354c86971f8b1688b011a8 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Wed, 15 Jan 2020 17:56:44 -0600 +Subject: [PATCH 07/13] Refactor: controller: functionize parts of resource + deletion notification + +... for future reuse +--- + crmd/crmd_lrm.h | 26 ++++++------- + crmd/lrm.c | 115 ++++++++++++++++++++++++++++++++++++++------------------ + 2 files changed, 91 insertions(+), 50 deletions(-) + +diff --git a/crmd/crmd_lrm.h b/crmd/crmd_lrm.h +index eb27d84..7d35264 100644 +--- a/crmd/crmd_lrm.h ++++ b/crmd/crmd_lrm.h +@@ -1,20 +1,13 @@ + /* +- * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net> ++ * Copyright 2004-2020 the Pacemaker project contributors + * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of the GNU Lesser General Public +- * License as published by the Free Software Foundation; either +- * version 2 of the License, or (at your option) any later version. ++ * The version control history for this file may have further details. + * +- * This software is distributed in the hope that it will be useful, +- * but WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * General Public License for more details. +- * +- * You should have received a copy of the GNU Lesser General Public +- * License along with this library; if not, write to the Free Software +- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA ++ * This source code is licensed under the GNU Lesser General Public License ++ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ ++#ifndef CONTROLD_LRM__H ++# define CONTROLD_LRM__H + + #include <crmd_messages.h> + #include <crmd_metadata.h> +@@ -176,3 +169,10 @@ gboolean remote_ra_controlling_guest(lrm_state_t * lrm_state); + + void process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, + active_op_t *pending, xmlNode *action_xml); ++void controld_ack_event_directly(const char *to_host, const char *to_sys, ++ lrmd_rsc_info_t *rsc, lrmd_event_data_t *op, ++ const char *rsc_id); ++void controld_rc2event(lrmd_event_data_t *event, int rc); ++void controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id); ++ ++#endif +diff --git a/crmd/lrm.c b/crmd/lrm.c +index ea0d1bb..584ab62 100644 +--- a/crmd/lrm.c ++++ b/crmd/lrm.c +@@ -45,9 +45,6 @@ static lrmd_event_data_t *construct_op(lrm_state_t * lrm_state, xmlNode * rsc_op + static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, + const char *operation, xmlNode *msg); + +-void send_direct_ack(const char *to_host, const char *to_sys, +- lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id); +- + static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, + int log_level); + static int do_update_resource(const char *node_name, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op); +@@ -285,7 +282,7 @@ send_task_ok_ack(lrm_state_t *lrm_state, ha_msg_input_t *input, + + op->rc = PCMK_OCF_OK; + op->op_status = PCMK_LRM_OP_DONE; +- send_direct_ack(ack_host, ack_sys, rsc, op, rsc_id); ++ controld_ack_event_directly(ack_host, ack_sys, rsc, op, rsc_id); + lrmd_free_event(op); + } + +@@ -861,6 +858,57 @@ controld_query_executor_state(const char *node_name) + return xml_state; + } + ++/*! ++ * \internal ++ * \brief Map standard Pacemaker return code to operation status and OCF code ++ * ++ * \param[out] event Executor event whose status and return code should be set ++ * \param[in] rc Standard Pacemaker return code ++ */ ++void ++controld_rc2event(lrmd_event_data_t *event, int rc) ++{ ++ switch (rc) { ++ case pcmk_rc_ok: ++ event->rc = PCMK_OCF_OK; ++ event->op_status = PCMK_LRM_OP_DONE; ++ break; ++ case EACCES: ++ event->rc = PCMK_OCF_INSUFFICIENT_PRIV; ++ event->op_status = PCMK_LRM_OP_ERROR; ++ break; ++ default: ++ event->rc = PCMK_OCF_UNKNOWN_ERROR; ++ event->op_status = PCMK_LRM_OP_ERROR; ++ break; ++ } ++} ++ ++/*! ++ * \internal ++ * \brief Trigger a new transition after CIB status was deleted ++ * ++ * If a CIB status delete was not expected (as part of the transition graph), ++ * trigger a new transition by updating the (arbitrary) "last-lrm-refresh" ++ * cluster property. ++ * ++ * \param[in] from_sys IPC name that requested the delete ++ * \param[in] rsc_id Resource whose status was deleted (for logging only) ++ */ ++void ++controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id) ++{ ++ if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) { ++ char *now_s = crm_strdup_printf("%lld", (long long) time(NULL)); ++ ++ crm_debug("Triggering a refresh after %s cleaned %s", from_sys, rsc_id); ++ update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, ++ NULL, NULL, NULL, NULL, "last-lrm-refresh", now_s, ++ FALSE, NULL, NULL); ++ free(now_s); ++ } ++} ++ + static void + notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc) + { +@@ -871,32 +919,11 @@ notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_ + crm_info("Notifying %s on %s that %s was%s deleted", + from_sys, (from_host? from_host : "localhost"), rsc_id, + ((rc == pcmk_ok)? "" : " not")); +- + op = construct_op(lrm_state, input->xml, rsc_id, CRMD_ACTION_DELETE); +- +- if (rc == pcmk_ok) { +- op->op_status = PCMK_LRM_OP_DONE; +- op->rc = PCMK_OCF_OK; +- } else { +- op->op_status = PCMK_LRM_OP_ERROR; +- op->rc = PCMK_OCF_UNKNOWN_ERROR; +- } +- +- send_direct_ack(from_host, from_sys, NULL, op, rsc_id); ++ controld_rc2event(op, pcmk_legacy2rc(rc)); ++ controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id); + lrmd_free_event(op); +- +- if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) { +- /* this isn't expected - trigger a new transition */ +- time_t now = time(NULL); +- char *now_s = crm_itoa(now); +- +- crm_debug("Triggering a refresh after %s deleted %s from the LRM", from_sys, rsc_id); +- +- update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL, +- "last-lrm-refresh", now_s, FALSE, NULL, NULL); +- +- free(now_s); +- } ++ controld_trigger_delete_refresh(from_sys, rsc_id); + } + + static gboolean +@@ -1490,7 +1517,7 @@ fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name, + #if ENABLE_ACL + if (user_name && is_privileged(user_name) == FALSE) { + crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc)); +- send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc)); ++ controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc)); + lrmd_free_event(op); + return; + } +@@ -1509,7 +1536,7 @@ fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name, + crm_log_xml_warn(xml, "bad input"); + } + +- send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc)); ++ controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc)); + lrmd_free_event(op); + } + +@@ -1679,7 +1706,7 @@ do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state, + } else { + op->rc = PCMK_OCF_UNKNOWN_ERROR; + } +- send_direct_ack(from_host, from_sys, NULL, op, rsc->id); ++ controld_ack_event_directly(from_host, from_sys, NULL, op, rsc->id); + lrmd_free_event(op); + return; + } +@@ -1996,9 +2023,23 @@ construct_op(lrm_state_t * lrm_state, xmlNode * rsc_op, const char *rsc_id, cons + return op; + } + ++/*! ++ * \internal ++ * \brief Send a (synthesized) event result ++ * ++ * Reply with a synthesized event result directly, as opposed to going through ++ * the executor. ++ * ++ * \param[in] to_host Host to send result to ++ * \param[in] to_sys IPC name to send result to (NULL for transition engine) ++ * \param[in] rsc Type information about resource the result is for ++ * \param[in] op Event with result to send ++ * \param[in] rsc_id ID of resource the result is for ++ */ + void +-send_direct_ack(const char *to_host, const char *to_sys, +- lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id) ++controld_ack_event_directly(const char *to_host, const char *to_sys, ++ lrmd_rsc_info_t *rsc, lrmd_event_data_t *op, ++ const char *rsc_id) + { + xmlNode *reply = NULL; + xmlNode *update, *iter; +@@ -2216,7 +2257,7 @@ do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, + + op->rc = CRM_DIRECT_NACK_RC; + op->op_status = PCMK_LRM_OP_ERROR; +- send_direct_ack(NULL, NULL, rsc, op, rsc->id); ++ controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id); + lrmd_free_event(op); + free(op_id); + return; +@@ -2287,7 +2328,7 @@ do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, + + op->rc = target_rc; + op->op_status = PCMK_LRM_OP_DONE; +- send_direct_ack(NULL, NULL, rsc, op, rsc->id); ++ controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id); + } + + pending->params = op->params; +@@ -2387,7 +2428,7 @@ do_update_resource(const char *node_name, lrmd_rsc_info_t * rsc, lrmd_event_data + + } else { + crm_warn("Resource %s no longer exists in the lrmd", op->rsc_id); +- send_direct_ack(NULL, NULL, rsc, op, op->rsc_id); ++ controld_ack_event_directly(NULL, NULL, rsc, op, op->rsc_id); + goto cleanup; + } + +@@ -2643,7 +2684,7 @@ process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, + } + + if (need_direct_ack) { +- send_direct_ack(NULL, NULL, NULL, op, op->rsc_id); ++ controld_ack_event_directly(NULL, NULL, NULL, op, op->rsc_id); + } + + if(remove == FALSE) { +-- +1.8.3.1 + + +From 95d35d3649d52226f1ae906a07a9e4744fae4061 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Tue, 14 Jan 2020 16:00:36 -0600 +Subject: [PATCH 08/13] Low: tools: improve crm_resource "why" messages + +--- + tools/crm_resource_runtime.c | 22 +++++++++++++--------- + 1 file changed, 13 insertions(+), 9 deletions(-) + +diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c +index fc84275..30d7e45 100644 +--- a/tools/crm_resource_runtime.c ++++ b/tools/crm_resource_runtime.c +@@ -860,7 +860,7 @@ cli_cleanup_all(crm_ipc_t *crmd_channel, const char *node_name, + void + cli_resource_check(cib_t * cib_conn, resource_t *rsc) + { +- int need_nl = 0; ++ bool printed = false; + char *role_s = NULL; + char *managed = NULL; + resource_t *parent = uber_parent(rsc); +@@ -877,21 +877,25 @@ cli_resource_check(cib_t * cib_conn, resource_t *rsc) + // Treated as if unset + + } else if(role == RSC_ROLE_STOPPED) { +- printf("\n * The configuration specifies that '%s' should remain stopped\n", parent->id); +- need_nl++; ++ printf("\n * Configuration specifies '%s' should remain stopped\n", ++ parent->id); ++ printed = true; + + } else if(parent->variant == pe_master && role == RSC_ROLE_SLAVE) { +- printf("\n * The configuration specifies that '%s' should not be promoted\n", parent->id); +- need_nl++; ++ printf("\n * Configuration specifies '%s' should not be promoted\n", ++ parent->id); ++ printed = true; + } + } + +- if(managed && crm_is_true(managed) == FALSE) { +- printf("%s * The configuration prevents the cluster from stopping or starting '%s' (unmanaged)\n", need_nl == 0?"\n":"", parent->id); +- need_nl++; ++ if (managed && !crm_is_true(managed)) { ++ printf("%s * Configuration prevents cluster from stopping or starting unmanaged '%s'\n", ++ (printed? "" : "\n"), parent->id); ++ printed = true; + } ++ free(managed); + +- if(need_nl) { ++ if (printed) { + printf("\n"); + } + } +-- +1.8.3.1 + + +From e9f09c7c55982db4d290980f87371ad622fdaf68 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Tue, 14 Jan 2020 16:23:25 -0600 +Subject: [PATCH 09/13] Low: tools: improve error checking for crm_resource + cleanup/fail commands + +Bail earlier for misconfigured resources, and return error (rather than hang) +for unknown or offline node. Also add timeout directly to controller request +rather than rely on the controller using the interval as default timeout. +--- + tools/crm_resource_runtime.c | 54 +++++++++++++++++++++++++++----------------- + 1 file changed, 33 insertions(+), 21 deletions(-) + +diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c +index 30d7e45..9e70db8 100644 +--- a/tools/crm_resource_runtime.c ++++ b/tools/crm_resource_runtime.c +@@ -453,8 +453,9 @@ send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op, + int rc = -ECOMM; + xmlNode *cmd = NULL; + xmlNode *xml_rsc = NULL; +- const char *value = NULL; + const char *router_node = host_uname; ++ const char *rsc_class = NULL; ++ const char *rsc_type = NULL; + xmlNode *params = NULL; + xmlNode *msg_data = NULL; + resource_t *rsc = pe_find_resource(data_set->resources, rsc_id); +@@ -466,26 +467,48 @@ send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op, + } else if (rsc->variant != pe_native) { + CMD_ERR("We can only process primitive resources, not %s", rsc_id); + return -EINVAL; ++ } + +- } else if (host_uname == NULL) { ++ rsc_class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS); ++ rsc_type = crm_element_value(rsc->xml, XML_ATTR_TYPE); ++ if ((rsc_class == NULL) || (rsc_type == NULL)) { ++ CMD_ERR("Resource %s does not have a class and type", rsc_id); ++ return -EINVAL; ++ } ++ ++ if (host_uname == NULL) { + CMD_ERR("Please specify a node name"); + return -EINVAL; ++ + } else { +- node_t *node = pe_find_node(data_set->nodes, host_uname); ++ pe_node_t *node = pe_find_node(data_set->nodes, host_uname); + ++ if (node == NULL) { ++ CMD_ERR("Node %s not found", host_uname); ++ return -pcmk_err_node_unknown; ++ } ++ ++ if (!(node->details->online)) { ++ CMD_ERR("Node %s is not online", host_uname); ++ return -ENOTCONN; ++ } + if (node && is_remote_node(node)) { + node = pe__current_node(node->details->remote_rsc); + if (node == NULL) { + CMD_ERR("No lrmd connection detected to remote node %s", host_uname); +- return -ENXIO; ++ return -ENOTCONN; + } + router_node = node->details->uname; + } + } + +- key = generate_transition_key(0, getpid(), 0, "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx"); +- + msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP); ++ ++ /* The controller logs the transition key from requests, so we need to have ++ * *something* for it. ++ */ ++ key = generate_transition_key(0, getpid(), 0, ++ "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx"); + crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key); + free(key); + +@@ -503,31 +526,20 @@ send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op, + crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->id); + } + +- value = crm_copy_xml_element(rsc->xml, xml_rsc, XML_ATTR_TYPE); +- if (value == NULL) { +- CMD_ERR("%s has no type! Aborting...", rsc_id); +- return -ENXIO; +- } +- +- value = crm_copy_xml_element(rsc->xml, xml_rsc, XML_AGENT_ATTR_CLASS); +- if (value == NULL) { +- CMD_ERR("%s has no class! Aborting...", rsc_id); +- return -ENXIO; +- } +- ++ crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, rsc_class); + crm_copy_xml_element(rsc->xml, xml_rsc, XML_AGENT_ATTR_PROVIDER); ++ crm_xml_add(xml_rsc, XML_ATTR_TYPE, rsc_type); + + params = create_xml_node(msg_data, XML_TAG_ATTRS); + crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET); + +- key = crm_meta_name(XML_LRM_ATTR_INTERVAL); ++ // The controller parses the timeout from the request ++ key = crm_meta_name(XML_ATTR_TIMEOUT); + crm_xml_add(params, key, "60000"); /* 1 minute */ + free(key); + + our_pid = crm_getpid_s(); + cmd = create_request(op, msg_data, router_node, CRM_SYSTEM_CRMD, crm_system_name, our_pid); +- +-/* crm_log_xml_warn(cmd, "send_lrm_rsc_op"); */ + free_xml(msg_data); + + if (crm_ipc_send(crmd_channel, cmd, 0, 0, NULL) > 0) { +-- +1.8.3.1 + + +From 36ecc33e4c9b48d7b02d078a1d00c87254809d0e Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Tue, 14 Jan 2020 18:07:18 -0600 +Subject: [PATCH 10/13] Refactor: liblrmd: new convenience function for + allocating lrmd_event_data_t + +--- + crmd/lrm.c | 7 +------ + include/crm/lrmd.h | 2 ++ + lib/lrmd/lrmd_client.c | 34 +++++++++++++++++++++++++++++++++- + lib/transition/unpack.c | 9 +++------ + tools/fake_transition.c | 7 +------ + 5 files changed, 40 insertions(+), 19 deletions(-) + +diff --git a/crmd/lrm.c b/crmd/lrm.c +index 584ab62..27fdd8b 100644 +--- a/crmd/lrm.c ++++ b/crmd/lrm.c +@@ -1871,15 +1871,10 @@ construct_op(lrm_state_t * lrm_state, xmlNode * rsc_op, const char *rsc_id, cons + + CRM_ASSERT(rsc_id && operation); + +- op = calloc(1, sizeof(lrmd_event_data_t)); +- CRM_ASSERT(op != NULL); +- ++ op = lrmd_new_event(rsc_id, operation, 0); + op->type = lrmd_event_exec_complete; +- op->op_type = strdup(operation); + op->op_status = PCMK_LRM_OP_PENDING; + op->rc = -1; +- op->rsc_id = strdup(rsc_id); +- op->interval = 0; + op->timeout = 0; + op->start_delay = 0; + +diff --git a/include/crm/lrmd.h b/include/crm/lrmd.h +index ecfc984..746257a 100644 +--- a/include/crm/lrmd.h ++++ b/include/crm/lrmd.h +@@ -254,6 +254,8 @@ typedef struct lrmd_event_data_s { + const char *exit_reason; + } lrmd_event_data_t; + ++lrmd_event_data_t *lrmd_new_event(const char *rsc_id, const char *task, ++ int interval_ms); + lrmd_event_data_t *lrmd_copy_event(lrmd_event_data_t * event); + void lrmd_free_event(lrmd_event_data_t * event); + +diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c +index b17d598..487c992 100644 +--- a/lib/lrmd/lrmd_client.c ++++ b/lib/lrmd/lrmd_client.c +@@ -1,5 +1,7 @@ + /* +- * Copyright (c) 2012 David Vossel <davidvossel@gmail.com> ++ * Copyright 2012-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public +@@ -187,6 +189,36 @@ lrmd_key_value_freeall(lrmd_key_value_t * head) + } + } + ++/*! ++ * Create a new lrmd_event_data_t object ++ * ++ * \param[in] rsc_id ID of resource involved in event ++ * \param[in] task Action name ++ * \param[in] interval_ms Action interval ++ * ++ * \return Newly allocated and initialized lrmd_event_data_t ++ * \note This functions asserts on memory errors, so the return value is ++ * guaranteed to be non-NULL. The caller is responsible for freeing the ++ * result with lrmd_free_event(). ++ */ ++lrmd_event_data_t * ++lrmd_new_event(const char *rsc_id, const char *task, int interval_ms) ++{ ++ lrmd_event_data_t *event = calloc(1, sizeof(lrmd_event_data_t)); ++ ++ CRM_ASSERT(event != NULL); ++ if (rsc_id != NULL) { ++ event->rsc_id = strdup(rsc_id); ++ CRM_ASSERT(event->rsc_id != NULL); ++ } ++ if (task != NULL) { ++ event->op_type = strdup(task); ++ CRM_ASSERT(event->op_type != NULL); ++ } ++ event->interval = interval_ms; ++ return event; ++} ++ + lrmd_event_data_t * + lrmd_copy_event(lrmd_event_data_t * event) + { +diff --git a/lib/transition/unpack.c b/lib/transition/unpack.c +index 2552716..d923a02 100644 +--- a/lib/transition/unpack.c ++++ b/lib/transition/unpack.c +@@ -310,12 +310,9 @@ convert_graph_action(xmlNode * resource, crm_action_t * action, int status, int + CRM_CHECK(action_resource != NULL, crm_log_xml_warn(action->xml, "Bad"); + return NULL); + +- op = calloc(1, sizeof(lrmd_event_data_t)); +- +- op->rsc_id = strdup(ID(action_resource)); +- op->interval = action->interval; +- op->op_type = strdup(crm_element_value(action->xml, XML_LRM_ATTR_TASK)); +- ++ op = lrmd_new_event(ID(action_resource), ++ crm_element_value(action->xml, XML_LRM_ATTR_TASK), ++ action->interval); + op->rc = rc; + op->op_status = status; + op->t_run = time(NULL); +diff --git a/tools/fake_transition.c b/tools/fake_transition.c +index 3cdd1f1..9068642 100644 +--- a/tools/fake_transition.c ++++ b/tools/fake_transition.c +@@ -147,12 +147,7 @@ create_op(xmlNode * cib_resource, const char *task, int interval, int outcome) + lrmd_event_data_t *op = NULL; + xmlNode *xop = NULL; + +- op = calloc(1, sizeof(lrmd_event_data_t)); +- +- op->rsc_id = strdup(ID(cib_resource)); +- op->interval = interval; +- op->op_type = strdup(task); +- ++ op = lrmd_new_event(ID(cib_resource), task, interval); + op->rc = outcome; + op->op_status = 0; + op->params = NULL; /* TODO: Fill me in */ +-- +1.8.3.1 + + +From 4d55fe7eb191b8e6f5590b8dd526797e0082109b Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= <jpokorny@redhat.com> +Date: Thu, 16 Jan 2020 15:58:02 +0100 +Subject: [PATCH 11/13] Fix: rectify thinko possibly behind spurious "process + will not die" msg + +Problem of not explicating how exactly the function is supposed to +behave that would allow for a straightforward cross-check, as opposed +to examining the callers into full depth that often gets substituted +with guestimation. +--- + lib/common/mainloop.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c +index 9bdd026..d4a5830 100644 +--- a/lib/common/mainloop.c ++++ b/lib/common/mainloop.c +@@ -1020,7 +1020,7 @@ child_timeout_callback(gpointer p) + } + + rc = child_kill_helper(child); +- if (rc == ESRCH) { ++ if (rc == -ESRCH) { + /* Nothing left to do. pid doesn't exist */ + return FALSE; + } +-- +1.8.3.1 + + +From 180e0991367aa81d219f7cea4c17b7ce3b968b2a Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 13 Sep 2019 13:17:00 -0500 +Subject: [PATCH 12/13] Feature: libcrmcommon: add XML getter and setter for + long long values + +--- + include/crm/common/nvpair.h | 2 ++ + lib/common/nvpair.c | 57 +++++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 59 insertions(+) + +diff --git a/include/crm/common/nvpair.h b/include/crm/common/nvpair.h +index 472c425..bb81b60 100644 +--- a/include/crm/common/nvpair.h ++++ b/include/crm/common/nvpair.h +@@ -45,12 +45,14 @@ GHashTable *xml2list(xmlNode *parent); + const char *crm_xml_add(xmlNode *node, const char *name, const char *value); + const char *crm_xml_replace(xmlNode *node, const char *name, const char *value); + const char *crm_xml_add_int(xmlNode *node, const char *name, int value); ++const char *crm_xml_add_ll(xmlNode *node, const char *name, long long value); + const char *crm_xml_add_ms(xmlNode *node, const char *name, guint ms); + + const char *crm_element_value(const xmlNode *data, const char *name); + int crm_element_value_int(const xmlNode *data, const char *name, int *dest); + int crm_element_value_const_int(const xmlNode *data, const char *name, int *dest); + const char *crm_element_value_const(const xmlNode *data, const char *name); ++int crm_element_value_ll(const xmlNode *data, const char *name, long long *dest); + int crm_element_value_timeval(const xmlNode *data, const char *name_sec, + const char *name_usec, struct timeval *dest); + char *crm_element_value_copy(const xmlNode *data, const char *name); +diff --git a/lib/common/nvpair.c b/lib/common/nvpair.c +index e9fec8a..2a3c7e7 100644 +--- a/lib/common/nvpair.c ++++ b/lib/common/nvpair.c +@@ -352,6 +352,35 @@ crm_xml_add_ms(xmlNode *node, const char *name, guint ms) + return added; + } + ++// Maximum size of null-terminated string representation of 64-bit integer ++// -9223372036854775808 ++#define LLSTRSIZE 21 ++ ++/*! ++ * \brief Create an XML attribute with specified name and long long int value ++ * ++ * This is like \c crm_xml_add() but taking a long long int value. It is a ++ * useful equivalent for defined types like time_t, etc. ++ * ++ * \param[in,out] xml XML node to modify ++ * \param[in] name Attribute name to set ++ * \param[in] value Attribute value to set ++ * ++ * \return New value as string on success, \c NULL otherwise ++ * \note This does nothing if xml or name are \c NULL or empty. ++ * This does not support greater than 64-bit values. ++ */ ++const char * ++crm_xml_add_ll(xmlNode *xml, const char *name, long long value) ++{ ++ char s[LLSTRSIZE] = { '\0', }; ++ ++ if (snprintf(s, LLSTRSIZE, "%lld", (long long) value) == LLSTRSIZE) { ++ return NULL; ++ } ++ return crm_xml_add(xml, name, s); ++} ++ + /*! + * \brief Retrieve the value of an XML attribute + * +@@ -409,6 +438,34 @@ crm_element_value_int(const xmlNode *data, const char *name, int *dest) + return -1; + } + ++/*! ++ * \brief Retrieve the long long integer value of an XML attribute ++ * ++ * This is like \c crm_element_value() but getting the value as a long long int. ++ * ++ * \param[in] data XML node to check ++ * \param[in] name Attribute name to check ++ * \param[in] dest Where to store element value ++ * ++ * \return 0 on success, -1 otherwise ++ */ ++int ++crm_element_value_ll(const xmlNode *data, const char *name, long long *dest) ++{ ++ const char *value = NULL; ++ ++ CRM_CHECK(dest != NULL, return -1); ++ value = crm_element_value(data, name); ++ if (value) { ++ errno = 0; ++ *dest = crm_int_helper(value, NULL); ++ if (errno == 0) { ++ return 0; ++ } ++ } ++ return -1; ++} ++ + int + crm_element_value_const_int(const xmlNode * data, const char *name, int *dest) + { +-- +1.8.3.1 + + +From e19fcaf0ae0d3ef58c93fb6982df7091496b1550 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 13 Sep 2019 16:46:06 -0500 +Subject: [PATCH 13/13] Feature: libcrmcommon: add XML getter for epoch time + values + +This scans epoch time values as long long rather than integer, to avoid +Year 2038 issues. +--- + include/crm/common/nvpair.h | 1 + + lib/common/nvpair.c | 27 +++++++++++++++++++++++++++ + 2 files changed, 28 insertions(+) + +diff --git a/include/crm/common/nvpair.h b/include/crm/common/nvpair.h +index bb81b60..3350a19 100644 +--- a/include/crm/common/nvpair.h ++++ b/include/crm/common/nvpair.h +@@ -53,6 +53,7 @@ int crm_element_value_int(const xmlNode *data, const char *name, int *dest); + int crm_element_value_const_int(const xmlNode *data, const char *name, int *dest); + const char *crm_element_value_const(const xmlNode *data, const char *name); + int crm_element_value_ll(const xmlNode *data, const char *name, long long *dest); ++int crm_element_value_epoch(const xmlNode *xml, const char *name, time_t *dest); + int crm_element_value_timeval(const xmlNode *data, const char *name_sec, + const char *name_usec, struct timeval *dest); + char *crm_element_value_copy(const xmlNode *data, const char *name); +diff --git a/lib/common/nvpair.c b/lib/common/nvpair.c +index 2a3c7e7..3dc4a1b 100644 +--- a/lib/common/nvpair.c ++++ b/lib/common/nvpair.c +@@ -479,6 +479,33 @@ crm_element_value_const(const xmlNode * data, const char *name) + } + + /*! ++ * \brief Retrieve the seconds-since-epoch value of an XML attribute ++ * ++ * This is like \c crm_element_value() but returning the value as a time_t. ++ * ++ * \param[in] xml XML node to check ++ * \param[in] name Attribute name to check ++ * \param[out] dest Where to store attribute value ++ * ++ * \return \c pcmk_ok on success, -1 otherwise ++ */ ++int ++crm_element_value_epoch(const xmlNode *xml, const char *name, time_t *dest) ++{ ++ long long value_ll = 0; ++ ++ if (crm_element_value_ll(xml, name, &value_ll) < 0) { ++ return -1; ++ } ++ ++ /* Unfortunately, we can't do any bounds checking, since time_t has neither ++ * standardized bounds nor constants defined for them. ++ */ ++ *dest = (time_t) value_ll; ++ return pcmk_ok; ++} ++ ++/*! + * \brief Retrieve the value of XML second/microsecond attributes as time + * + * This is like \c crm_element_value() but returning value as a struct timeval. +-- +1.8.3.1 + diff --git a/SOURCES/2.0-cleanup-behavior.patch b/SOURCES/2.0-cleanup-behavior.patch index 4a20eaa..039079e 100644 --- a/SOURCES/2.0-cleanup-behavior.patch +++ b/SOURCES/2.0-cleanup-behavior.patch @@ -11,7 +11,7 @@ diff --git a/tools/crm_resource.c b/tools/crm_resource.c index 128d075..bbdba25 100644 --- a/tools/crm_resource.c +++ b/tools/crm_resource.c -@@ -212,8 +212,6 @@ static struct crm_option long_options[] = { +@@ -214,8 +214,6 @@ static struct crm_option long_options[] = { }, { "cleanup", no_argument, NULL, 'C', @@ -71,47 +71,6 @@ index 128d075..bbdba25 100644 find_flags = pe_find_renamed|pe_find_anon; break; -@@ -1120,21 +1115,25 @@ main(int argc, char **argv) - start_mainloop(); - } - -- } else if ((rsc_cmd == 'R') && rsc) { -- if (do_force == FALSE) { -- rsc = uber_parent(rsc); -- } -- crmd_replies_needed = 0; -- -- crm_debug("Re-checking the state of %s (%s requested) on %s", -- rsc->id, rsc_id, (host_uname? host_uname: "all nodes")); -- rc = cli_resource_delete(crmd_channel, host_uname, rsc, -- NULL, 0, FALSE, data_set); -- -- if ((rc == pcmk_ok) && !BE_QUIET) { -- // Show any reasons why resource might stay stopped -- cli_resource_check(cib_conn, rsc); -- } -+ } else if ((rsc_cmd == 'R') && rsc) { -+ if (do_force == FALSE) { -+ rsc = uber_parent(rsc); -+ } -+ crmd_replies_needed = 0; -+ -+ crm_debug("Re-checking the state of %s (%s requested) on %s", -+ rsc->id, rsc_id, (host_uname? host_uname: "all nodes")); -+ rc = cli_resource_delete(crmd_channel, host_uname, rsc, -+ NULL, 0, FALSE, data_set); -+ -+ if ((rc == pcmk_ok) && !BE_QUIET) { -+ // Show any reasons why resource might stay stopped -+ cli_resource_check(cib_conn, rsc); -+ } -+ -+ if (rc == pcmk_ok) { -+ start_mainloop(); -+ } - - } else if (rsc_cmd == 'R') { - #if HAVE_ATOMIC_ATTRD -- 1.8.3.1 diff --git a/SOURCES/shutdown-lock-01.patch b/SOURCES/shutdown-lock-01.patch new file mode 100644 index 0000000..81ba0ad --- /dev/null +++ b/SOURCES/shutdown-lock-01.patch @@ -0,0 +1,132 @@ +From c3e2a63e08046b25f1773504d8c1ab431d3abf78 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 21 Nov 2019 14:48:02 -0600 +Subject: [PATCH 01/10] Feature: scheduler: add shutdown lock cluster options + +This commit adds shutdown-lock and shutdown-lock-limit options (just the +options, not the feature itself). + +shutdown-lock defaults to false, which preserves current behavior. The intended +purpose of setting it to true is to *prevent* recovery of a node's resources +elsewhere when the node is cleanly shut down, until the node rejoins. If +shutdown-lock-limit is set to a nonzero time duration, the cluster will +be allowed to recover the resources if the node has not rejoined within this +time. + +The use case is when rebooting a node (such as for software updates) is done by +cluster-unaware system administrators during scheduled maintenance windows, +resources prefer specific nodes, and resource recovery time is high. +--- + include/crm/msg_xml.h | 6 +++++- + include/crm/pengine/status.h | 2 ++ + lib/pengine/common.c | 26 ++++++++++++++++++++++++-- + lib/pengine/unpack.c | 10 ++++++++++ + 4 files changed, 41 insertions(+), 3 deletions(-) + +diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h +index de99959..50fdf45 100644 +--- a/include/crm/msg_xml.h ++++ b/include/crm/msg_xml.h +@@ -1,5 +1,7 @@ + /* +- * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net> ++ * Copyright 2004-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public +@@ -378,6 +380,8 @@ + # define XML_CONFIG_ATTR_FORCE_QUIT "shutdown-escalation" + # define XML_CONFIG_ATTR_RECHECK "cluster-recheck-interval" + # define XML_CONFIG_ATTR_FENCE_REACTION "fence-reaction" ++# define XML_CONFIG_ATTR_SHUTDOWN_LOCK "shutdown-lock" ++# define XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT "shutdown-lock-limit" + + # define XML_ALERT_ATTR_PATH "path" + # define XML_ALERT_ATTR_TIMEOUT "timeout" +diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h +index 415f60e..c6d4bdb 100644 +--- a/include/crm/pengine/status.h ++++ b/include/crm/pengine/status.h +@@ -83,6 +83,7 @@ enum pe_find { + # define pe_flag_start_failure_fatal 0x00001000ULL + # define pe_flag_remove_after_stop 0x00002000ULL + # define pe_flag_startup_fencing 0x00004000ULL ++# define pe_flag_shutdown_lock 0x00008000ULL + + # define pe_flag_startup_probes 0x00010000ULL + # define pe_flag_have_status 0x00020000ULL +@@ -148,6 +149,7 @@ typedef struct pe_working_set_s { + + GList *param_check; // History entries that need to be checked + GList *stop_needed; // Containers that need stop actions ++ guint shutdown_lock;// How long (seconds) to lock resources to shutdown node + } pe_working_set_t; + + enum pe_check_parameters { +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index e82434a..fc976d3 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -1,5 +1,7 @@ +-/* +- * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net> ++/* ++ * Copyright 2004-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public +@@ -101,6 +103,26 @@ pe_cluster_option pe_opts[] = { + "When set to TRUE, the cluster will immediately ban a resource from a node if it fails to start there. When FALSE, the cluster will instead check the resource's fail count against its migration-threshold." }, + { "enable-startup-probes", NULL, "boolean", NULL, "true", &check_boolean, + "Should the cluster check for active resources during startup", NULL }, ++ { ++ XML_CONFIG_ATTR_SHUTDOWN_LOCK, ++ NULL, "boolean", NULL, "false", &check_boolean, ++ "Whether to lock resources to a cleanly shut down node", ++ "When true, resources active on a node when it is cleanly shut down " ++ "are kept \"locked\" to that node (not allowed to run elsewhere) " ++ "until they start again on that node after it rejoins (or for at " ++ "most shutdown-lock-limit, if set). Stonith resources and " ++ "Pacemaker Remote connections are never locked. Clone and bundle " ++ "instances and the master role of promotable clones are currently " ++ "never locked, though support could be added in a future release." ++ }, ++ { ++ XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT, ++ NULL, "time", NULL, "0", &check_timer, ++ "Do not lock resources to a cleanly shut down node longer than this", ++ "If shutdown-lock is true and this is set to a nonzero time duration, " ++ "shutdown locks will expire after this much time has passed since " ++ "the shutdown was initiated, even if the node has not rejoined." ++ }, + + /* Stonith Options */ + { "stonith-enabled", "stonith_enabled", "boolean", NULL, "true", &check_boolean, +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 24e56f5..7b0d837 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -340,6 +340,16 @@ unpack_config(xmlNode * config, pe_working_set_t * data_set) + data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy"); + crm_trace("Placement strategy: %s", data_set->placement_strategy); + ++ set_config_flag(data_set, "shutdown-lock", pe_flag_shutdown_lock); ++ crm_trace("Resources will%s be locked to cleanly shut down nodes", ++ (is_set(data_set->flags, pe_flag_shutdown_lock)? "" : " not")); ++ if (is_set(data_set->flags, pe_flag_shutdown_lock)) { ++ value = pe_pref(data_set->config_hash, ++ XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT); ++ data_set->shutdown_lock = crm_get_interval(value) / 1000; ++ crm_trace("Shutdown locks expire after %us", data_set->shutdown_lock); ++ } ++ + return TRUE; + } + +-- +1.8.3.1 + diff --git a/SOURCES/shutdown-lock-02.patch b/SOURCES/shutdown-lock-02.patch new file mode 100644 index 0000000..bd12192 --- /dev/null +++ b/SOURCES/shutdown-lock-02.patch @@ -0,0 +1,143 @@ +From 4e85d3012d61dcf534a51d4f82b91fce9aef8d0b Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 6 Dec 2019 11:57:59 -0600 +Subject: [PATCH 02/10] Low: scheduler: respect shutdown locks when placing + active resources + +Use new pe_resource_t members to indicate that a resource is locked to a +particular node. + +For active resources (i.e. in the transition where the node is scheduled for +shutdown), these are connected by checking each lockable resource for whether +it is running on a single clean node that is shutting down. + +When applying constraints, place -INFINITY location constraints for locked +resources on all nodes other than the lock node. + +(Inactive resources -- i.e. in later transitions after the node is shut down -- +are not yet locked.) +--- + include/crm/pengine/status.h | 2 ++ + pengine/allocate.c | 86 ++++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 88 insertions(+) + +diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h +index c6d4bdb..1e8d5bb 100644 +--- a/include/crm/pengine/status.h ++++ b/include/crm/pengine/status.h +@@ -347,6 +347,8 @@ struct resource_s { + pe_working_set_t *cluster; + + pe_node_t *pending_node; // Node on which pending_task is happening ++ pe_node_t *lock_node; // Resource is shutdown-locked to this node ++ time_t lock_time; // When shutdown lock started + + #if ENABLE_VERSIONED_ATTRS + xmlNode *versioned_parameters; +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 30d29e1..09f9e51 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1009,6 +1009,86 @@ rsc_discover_filter(resource_t *rsc, node_t *node) + } + } + ++static time_t ++shutdown_time(pe_node_t *node, pe_working_set_t *data_set) ++{ ++ const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN); ++ time_t result = 0; ++ ++ if (shutdown) { ++ errno = 0; ++ result = (time_t) crm_int_helper(shutdown, NULL); ++ if (errno != 0) { ++ result = 0; ++ } ++ } ++ return result? result : get_effective_time(data_set); ++} ++ ++static void ++apply_shutdown_lock(pe_resource_t *rsc, pe_working_set_t *data_set) ++{ ++ const char *class; ++ ++ // Only primitives and (uncloned) groups may be locked ++ if (rsc->variant == pe_group) { ++ for (GList *item = rsc->children; item != NULL; ++ item = item->next) { ++ apply_shutdown_lock((pe_resource_t *) item->data, data_set); ++ } ++ } else if (rsc->variant != pe_native) { ++ return; ++ } ++ ++ // Fence devices and remote connections can't be locked ++ class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS); ++ if ((class == NULL) || !strcmp(class, PCMK_RESOURCE_CLASS_STONITH) ++ || is_rsc_baremetal_remote_node(rsc, data_set)) { ++ return; ++ } ++ ++ // Only a resource active on exactly one node can be locked ++ if (pcmk__list_of_1(rsc->running_on)) { ++ pe_node_t *node = rsc->running_on->data; ++ ++ if (node->details->shutdown) { ++ if (node->details->unclean) { ++ pe_rsc_debug(rsc, "Not locking %s to unclean %s for shutdown", ++ rsc->id, node->details->uname); ++ } else { ++ rsc->lock_node = node; ++ rsc->lock_time = shutdown_time(node, data_set); ++ } ++ } ++ } ++ ++ if (rsc->lock_node == NULL) { ++ // No lock needed ++ return; ++ } ++ ++ if (data_set->shutdown_lock > 0) { ++ time_t lock_expiration = rsc->lock_time + data_set->shutdown_lock; ++ ++ pe_rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)", ++ rsc->id, rsc->lock_node->details->uname, ++ (long long) lock_expiration); ++ } else { ++ pe_rsc_info(rsc, "Locking %s to %s due to shutdown", ++ rsc->id, rsc->lock_node->details->uname); ++ } ++ ++ // If resource is locked to one node, ban it from all other nodes ++ for (GList *item = data_set->nodes; item != NULL; item = item->next) { ++ pe_node_t *node = item->data; ++ ++ if (strcmp(node->details->uname, rsc->lock_node->details->uname)) { ++ resource_location(rsc, node, -INFINITY, ++ XML_CONFIG_ATTR_SHUTDOWN_LOCK, data_set); ++ } ++ } ++} ++ + /* + * Count how many valid nodes we have (so we know the maximum number of + * colors we can resolve). +@@ -1020,6 +1100,12 @@ stage2(pe_working_set_t * data_set) + { + GListPtr gIter = NULL; + ++ if (is_set(data_set->flags, pe_flag_shutdown_lock)) { ++ for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) { ++ apply_shutdown_lock((pe_resource_t *) gIter->data, data_set); ++ } ++ } ++ + for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { + node_t *node = (node_t *) gIter->data; + +-- +1.8.3.1 + diff --git a/SOURCES/shutdown-lock-03.patch b/SOURCES/shutdown-lock-03.patch new file mode 100644 index 0000000..154de19 --- /dev/null +++ b/SOURCES/shutdown-lock-03.patch @@ -0,0 +1,204 @@ +From 749f6b256cb2864ce3e862442adc6d219eefeca3 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 6 Dec 2019 12:17:03 -0600 +Subject: [PATCH 03/10] Low: scheduler: respect shutdown locks when placing + inactive resources + +When shutdown-lock is enabled, and we're either scheduling a resource stop +on a node that's cleanly shutting down or scheduling any action for a +previously locked resource, add "shutdown-lock=<shutdown-timestamp>" to the +graph action. The controller will be able to use this to know when to preserve +the lock (by adding the lock time to the resource state entry). + +When the scheduler unpacks a resource state entry with a lock, it will remember +the lock node and lock time, which will trigger existing code for applying +shutdown locks. +--- + lib/pengine/unpack.c | 49 ++++++++++++++++++++++++++++++++++++++++++++----- + pengine/allocate.c | 17 ++++++++++++++++- + pengine/graph.c | 32 +++++++++++++++++++++++++++++++- + 3 files changed, 91 insertions(+), 7 deletions(-) + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 7b0d837..bb5efa4 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -18,6 +18,7 @@ + #include <crm_internal.h> + + #include <glib.h> ++#include <time.h> + + #include <crm/crm.h> + #include <crm/services.h> +@@ -1151,7 +1152,8 @@ unpack_node_loop(xmlNode * status, bool fence, pe_working_set_t * data_set) + crm_trace("Checking node %s/%s/%s status %d/%d/%d", id, rsc->id, rsc->container->id, fence, rsc->role, RSC_ROLE_STARTED); + + } else if (is_container_remote_node(this_node) == FALSE +- && rsc->role == RSC_ROLE_STARTED) { ++ && ((rsc->role == RSC_ROLE_STARTED) ++ || is_set(data_set->flags, pe_flag_shutdown_lock))) { + check = TRUE; + crm_trace("Checking node %s/%s status %d/%d/%d", id, rsc->id, fence, rsc->role, RSC_ROLE_STARTED); + } +@@ -1167,6 +1169,9 @@ unpack_node_loop(xmlNode * status, bool fence, pe_working_set_t * data_set) + + } else if (fence) { + process = TRUE; ++ ++ } else if (is_set(data_set->flags, pe_flag_shutdown_lock)) { ++ process = TRUE; + } + + if(process) { +@@ -2286,6 +2291,28 @@ calculate_active_ops(GListPtr sorted_op_list, int *start_index, int *stop_index) + } + } + ++// If resource history entry has shutdown lock, remember lock node and time ++static void ++unpack_shutdown_lock(xmlNode *rsc_entry, pe_resource_t *rsc, pe_node_t *node, ++ pe_working_set_t *data_set) ++{ ++ time_t lock_time = 0; // When lock started (i.e. node shutdown time) ++ ++ if ((crm_element_value_epoch(rsc_entry, XML_CONFIG_ATTR_SHUTDOWN_LOCK, ++ &lock_time) == pcmk_ok) && (lock_time != 0)) { ++ ++ if ((data_set->shutdown_lock > 0) ++ && (get_effective_time(data_set) ++ > (lock_time + data_set->shutdown_lock))) { ++ pe_rsc_info(rsc, "Shutdown lock for %s on %s expired", ++ rsc->id, node->details->uname); ++ } else { ++ rsc->lock_node = node; ++ rsc->lock_time = lock_time; ++ } ++ } ++} ++ + static resource_t * + unpack_lrm_rsc_state(node_t * node, xmlNode * rsc_entry, pe_working_set_t * data_set) + { +@@ -2322,18 +2349,30 @@ unpack_lrm_rsc_state(node_t * node, xmlNode * rsc_entry, pe_working_set_t * data + } + } + +- if (op_list == NULL) { +- /* if there are no operations, there is nothing to do */ +- return NULL; ++ if (is_not_set(data_set->flags, pe_flag_shutdown_lock)) { ++ if (op_list == NULL) { ++ // If there are no operations, there is nothing to do ++ return NULL; ++ } + } + + /* find the resource */ + rsc = unpack_find_resource(data_set, node, rsc_id, rsc_entry); + if (rsc == NULL) { +- rsc = process_orphan_resource(rsc_entry, node, data_set); ++ if (op_list == NULL) { ++ // If there are no operations, there is nothing to do ++ return NULL; ++ } else { ++ rsc = process_orphan_resource(rsc_entry, node, data_set); ++ } + } + CRM_ASSERT(rsc != NULL); + ++ // Check whether the resource is "shutdown-locked" to this node ++ if (is_set(data_set->flags, pe_flag_shutdown_lock)) { ++ unpack_shutdown_lock(rsc_entry, rsc, node, data_set); ++ } ++ + /* process operations */ + saved_role = rsc->role; + on_fail = action_fail_ignore; +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 09f9e51..7366716 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1047,8 +1047,23 @@ apply_shutdown_lock(pe_resource_t *rsc, pe_working_set_t *data_set) + return; + } + ++ if (rsc->lock_node != NULL) { ++ // The lock was obtained from resource history ++ ++ if (rsc->running_on != NULL) { ++ /* The resource was started elsewhere even though it is now ++ * considered locked. This shouldn't be possible, but as a ++ * failsafe, we don't want to disturb the resource now. ++ */ ++ pe_rsc_info(rsc, ++ "Cancelling shutdown lock because %s is already active", ++ rsc->id); ++ rsc->lock_node = NULL; ++ rsc->lock_time = 0; ++ } ++ + // Only a resource active on exactly one node can be locked +- if (pcmk__list_of_1(rsc->running_on)) { ++ } else if (pcmk__list_of_1(rsc->running_on)) { + pe_node_t *node = rsc->running_on->data; + + if (node->details->shutdown) { +diff --git a/pengine/graph.c b/pengine/graph.c +index cba30d0..33168ca 100644 +--- a/pengine/graph.c ++++ b/pengine/graph.c +@@ -1,5 +1,7 @@ + /* +- * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net> ++ * Copyright 2004-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public +@@ -998,6 +1000,26 @@ add_downed_nodes(xmlNode *xml, const action_t *action, + } + } + ++static bool ++should_lock_action(pe_action_t *action) ++{ ++ // Only actions taking place on resource's lock node are locked ++ if ((action->rsc->lock_node == NULL) || (action->node == NULL) ++ || (action->node->details != action->rsc->lock_node->details)) { ++ return false; ++ } ++ ++ /* During shutdown, only stops are locked (otherwise, another action such as ++ * a demote would cause the controller to clear the lock) ++ */ ++ if (action->node->details->shutdown && action->task ++ && strcmp(action->task, RSC_STOP)) { ++ return false; ++ } ++ ++ return true; ++} ++ + static xmlNode * + action2xml(action_t * action, gboolean as_input, pe_working_set_t *data_set) + { +@@ -1104,6 +1126,14 @@ action2xml(action_t * action, gboolean as_input, pe_working_set_t *data_set) + XML_ATTR_TYPE + }; + ++ /* If a resource is locked to a node via shutdown-lock, mark its actions ++ * so the controller can preserve the lock when the action completes. ++ */ ++ if (should_lock_action(action)) { ++ crm_xml_add_ll(action_xml, XML_CONFIG_ATTR_SHUTDOWN_LOCK, ++ (long long) action->rsc->lock_time); ++ } ++ + // List affected resource + + rsc_xml = create_xml_node(action_xml, +-- +1.8.3.1 + diff --git a/SOURCES/shutdown-lock-04.patch b/SOURCES/shutdown-lock-04.patch new file mode 100644 index 0000000..5993772 --- /dev/null +++ b/SOURCES/shutdown-lock-04.patch @@ -0,0 +1,286 @@ +From 538aeef2523017a9b5c1ba950c984ed16efdc9a0 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 22 Nov 2019 17:03:20 -0600 +Subject: [PATCH 04/10] Low: controller: mark shutdown-locked resources in + resource history + +When a graph action indicates that the resource should be shutdown-locked +to its node, remember the shutdown lock time in active_op_t so we can remember +that when the result comes back. When the result does come back, add +"shutdown-lock" to its lrm_resource entry in the CIB status section -- as +the timestamp if it's a successful stop or a probe finding the resource +inactive, or as 0 to clear the lock for any other operation. +--- + crmd/control.c | 9 ++++++- + crmd/crmd_lrm.h | 1 + + crmd/crmd_utils.h | 1 + + crmd/lrm.c | 46 ++++++++++++++++++++++++++++++++---- + crmd/te_callbacks.c | 68 +++++++++++++++++++++++++++++++++++++---------------- + 5 files changed, 99 insertions(+), 26 deletions(-) + +diff --git a/crmd/control.c b/crmd/control.c +index cd4223f..47dabf1 100644 +--- a/crmd/control.c ++++ b/crmd/control.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2019 the Pacemaker project contributors ++ * Copyright 2004-2020 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -51,6 +51,7 @@ gboolean fsa_has_quorum = FALSE; + crm_trigger_t *fsa_source = NULL; + crm_trigger_t *config_read = NULL; + bool no_quorum_suicide_escalation = FALSE; ++bool controld_shutdown_lock_enabled = false; + + /* A_HA_CONNECT */ + void +@@ -971,7 +972,10 @@ pe_cluster_option crmd_opts[] = { + { "stonith-max-attempts",NULL,"integer",NULL,"10",&check_positive_number, + "How many times stonith can fail before it will no longer be attempted on a target" + }, ++ ++ // Already documented in libpe_status (other values must be kept identical) + { "no-quorum-policy", "no_quorum_policy", "enum", "stop, freeze, ignore, suicide", "stop", &check_quorum, NULL, NULL }, ++ { XML_CONFIG_ATTR_SHUTDOWN_LOCK, NULL, "boolean", NULL, "false", &check_boolean, NULL, NULL }, + + #if SUPPORT_PLUGIN + { XML_ATTR_EXPECTED_VOTES, NULL, "integer", NULL, "2", &check_number, "The number of nodes expected to be in the cluster", "Used to calculate quorum in openais based clusters." }, +@@ -1094,6 +1098,9 @@ config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void + value = crmd_pref(config_hash, "crmd-finalization-timeout"); + finalization_timer->period_ms = crm_get_msec(value); + ++ value = crmd_pref(config_hash, XML_CONFIG_ATTR_SHUTDOWN_LOCK); ++ controld_shutdown_lock_enabled = crm_is_true(value); ++ + #if SUPPORT_COROSYNC + if (is_classic_ais_cluster()) { + value = crmd_pref(config_hash, XML_ATTR_EXPECTED_VOTES); +diff --git a/crmd/crmd_lrm.h b/crmd/crmd_lrm.h +index 7d35264..ecc2511 100644 +--- a/crmd/crmd_lrm.h ++++ b/crmd/crmd_lrm.h +@@ -46,6 +46,7 @@ typedef struct active_op_s { + int interval; + uint32_t flags; // bitmask of active_op_e + unsigned int start_time; ++ time_t lock_time; + char *rsc_id; + char *op_type; + char *op_key; +diff --git a/crmd/crmd_utils.h b/crmd/crmd_utils.h +index eeaa8b7..9ecce88 100644 +--- a/crmd/crmd_utils.h ++++ b/crmd/crmd_utils.h +@@ -63,6 +63,7 @@ fsa_cib_anon_update(const char *section, xmlNode *data) { + } + + extern gboolean fsa_has_quorum; ++extern bool controld_shutdown_lock_enabled; + extern int last_peer_update; + extern int last_resource_update; + +diff --git a/crmd/lrm.c b/crmd/lrm.c +index 27fdd8b..9156ab8 100644 +--- a/crmd/lrm.c ++++ b/crmd/lrm.c +@@ -47,7 +47,8 @@ static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, + + static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, + int log_level); +-static int do_update_resource(const char *node_name, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op); ++static int do_update_resource(const char *node_name, lrmd_rsc_info_t *rsc, ++ lrmd_event_data_t *op, time_t lock_time); + + static void + lrm_connection_destroy(void) +@@ -2168,7 +2169,7 @@ record_pending_op(const char *node_name, lrmd_rsc_info_t *rsc, lrmd_event_data_t + crm_debug("Recording pending op %s_%s_%d on %s in the CIB", + op->rsc_id, op->op_type, op->interval, node_name); + +- do_update_resource(node_name, rsc, op); ++ do_update_resource(node_name, rsc, op, 0); + } + + static void +@@ -2309,7 +2310,11 @@ do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, + pending->op_key = strdup(op_id); + pending->rsc_id = strdup(rsc->id); + pending->start_time = time(NULL); +- pending->user_data = strdup(op->user_data); ++ pending->user_data = op->user_data? strdup(op->user_data) : NULL; ++ if (crm_element_value_epoch(msg, XML_CONFIG_ATTR_SHUTDOWN_LOCK, ++ &(pending->lock_time)) != pcmk_ok) { ++ pending->lock_time = 0; ++ } + g_hash_table_replace(lrm_state->pending_ops, call_id_s, pending); + + if (op->interval > 0 && op->start_delay > START_DELAY_THRESHOLD) { +@@ -2356,8 +2361,28 @@ cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *use + } + } + ++/* Only successful stops, and probes that found the resource inactive, get locks ++ * recorded in the history. This ensures the resource stays locked to the node ++ * until it is active there again after the node comes back up. ++ */ ++static bool ++should_preserve_lock(lrmd_event_data_t *op) ++{ ++ if (!controld_shutdown_lock_enabled) { ++ return false; ++ } ++ if (!strcmp(op->op_type, RSC_STOP) && (op->rc == PCMK_OCF_OK)) { ++ return true; ++ } ++ if (!strcmp(op->op_type, RSC_STATUS) && (op->rc == PCMK_OCF_NOT_RUNNING)) { ++ return true; ++ } ++ return false; ++} ++ + static int +-do_update_resource(const char *node_name, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op) ++do_update_resource(const char *node_name, lrmd_rsc_info_t *rsc, ++ lrmd_event_data_t *op, time_t lock_time) + { + /* + <status> +@@ -2412,6 +2437,16 @@ do_update_resource(const char *node_name, lrmd_rsc_info_t * rsc, lrmd_event_data + crm_xml_add(iter, XML_ATTR_TYPE, rsc->type); + crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->class); + crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER, rsc->provider); ++ if (lock_time != 0) { ++ /* Actions on a locked resource should either preserve the lock by ++ * recording it with the action result, or clear it. ++ */ ++ if (!should_preserve_lock(op)) { ++ lock_time = 0; ++ } ++ crm_xml_add_ll(iter, XML_CONFIG_ATTR_SHUTDOWN_LOCK, ++ (long long) lock_time); ++ } + + if (op->params) { + container = g_hash_table_lookup(op->params, CRM_META"_"XML_RSC_ATTR_CONTAINER); +@@ -2600,7 +2635,8 @@ process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, + if (controld_action_is_recordable(op->op_type)) { + if (node_name && rsc) { + // We should record the result, and happily, we can +- update_id = do_update_resource(node_name, rsc, op); ++ update_id = do_update_resource(node_name, rsc, op, ++ pending? pending->lock_time : 0); + need_direct_ack = FALSE; + + } else if (op->rsc_deleted) { +diff --git a/crmd/te_callbacks.c b/crmd/te_callbacks.c +index 9faf932..46a4393 100644 +--- a/crmd/te_callbacks.c ++++ b/crmd/te_callbacks.c +@@ -1,5 +1,7 @@ + /* +- * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net> ++ * Copyright 2004-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public +@@ -52,6 +54,18 @@ update_stonith_max_attempts(const char* value) + stonith_max_attempts = crm_int_helper(value, NULL); + } + } ++ ++// An explicit shutdown-lock of 0 means the lock has been cleared ++static bool ++shutdown_lock_cleared(xmlNode *lrm_resource) ++{ ++ time_t shutdown_lock = 0; ++ ++ return (crm_element_value_epoch(lrm_resource, XML_CONFIG_ATTR_SHUTDOWN_LOCK, ++ &shutdown_lock) == pcmk_ok) ++ && (shutdown_lock == 0); ++} ++ + static void + te_update_diff_v1(const char *event, xmlNode *diff) + { +@@ -130,33 +144,42 @@ te_update_diff_v1(const char *event, xmlNode *diff) + } + freeXpathObject(xpathObj); + ++ // Check for lrm_resource entries ++ xpathObj = xpath_search(diff, ++ "//" F_CIB_UPDATE_RESULT ++ "//" XML_TAG_DIFF_ADDED ++ "//" XML_LRM_TAG_RESOURCE); ++ max = numXpathResults(xpathObj); ++ + /* +- * Updates by, or in response to, TE actions will never contain updates +- * for more than one resource at a time, so such updates indicate an +- * LRM refresh. +- * +- * In that case, start a new transition rather than check each result +- * individually, which can result in _huge_ speedups in large clusters. ++ * Updates by, or in response to, graph actions will never affect more than ++ * one resource at a time, so such updates indicate an LRM refresh. In that ++ * case, start a new transition rather than check each result individually, ++ * which can result in _huge_ speedups in large clusters. + * + * Unfortunately, we can only do so when there are no pending actions. + * Otherwise, we could mistakenly throw away those results here, and + * the cluster will stall waiting for them and time out the operation. + */ +- if (transition_graph->pending == 0) { +- xpathObj = xpath_search(diff, +- "//" F_CIB_UPDATE_RESULT +- "//" XML_TAG_DIFF_ADDED +- "//" XML_LRM_TAG_RESOURCE); +- max = numXpathResults(xpathObj); +- if (max > 1) { +- crm_debug("Ignoring resource operation updates due to LRM refresh of %d resources", +- max); +- crm_log_xml_trace(diff, "lrm-refresh"); +- abort_transition(INFINITY, tg_restart, "LRM Refresh", NULL); +- goto bail; ++ if ((transition_graph->pending == 0) && (max > 1)) { ++ crm_debug("Ignoring resource operation updates due to history refresh of %d resources", ++ max); ++ crm_log_xml_trace(diff, "lrm-refresh"); ++ abort_transition(INFINITY, tg_restart, "History refresh", NULL); ++ goto bail; ++ } ++ ++ if (max == 1) { ++ xmlNode *lrm_resource = getXpathResult(xpathObj, 0); ++ ++ if (shutdown_lock_cleared(lrm_resource)) { ++ // @TODO would be more efficient to abort once after transition done ++ abort_transition(INFINITY, tg_restart, "Shutdown lock cleared", ++ lrm_resource); ++ // Still process results, so we stop timers and update failcounts + } +- freeXpathObject(xpathObj); + } ++ freeXpathObject(xpathObj); + + /* Process operation updates */ + xpathObj = +@@ -229,6 +252,11 @@ process_lrm_resource_diff(xmlNode *lrm_resource, const char *node) + rsc_op = __xml_next(rsc_op)) { + process_graph_event(rsc_op, node); + } ++ if (shutdown_lock_cleared(lrm_resource)) { ++ // @TODO would be more efficient to abort once after transition done ++ abort_transition(INFINITY, tg_restart, "Shutdown lock cleared", ++ lrm_resource); ++ } + } + + static void +-- +1.8.3.1 + diff --git a/SOURCES/shutdown-lock-05.patch b/SOURCES/shutdown-lock-05.patch new file mode 100644 index 0000000..14f1161 --- /dev/null +++ b/SOURCES/shutdown-lock-05.patch @@ -0,0 +1,159 @@ +From ba17007f04d2fdbd2147c14c7eedb0de137ff448 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 13 Dec 2019 11:38:49 -0600 +Subject: [PATCH 05/10] Low: controller: don't clear shutdown locks when node + rejoins + +Add new controld_delete_node_state() values for clearing resource history +while preserving shutdown locks. This is accomplished by deleting all +unlocked lrm_resource entries and all lrm_rsc_op entries, instead of the entire +lrm subsection. +--- + crmd/cib.c | 22 +++++++++++++++++++++- + crmd/crmd_utils.h | 2 ++ + crmd/join_dc.c | 7 +++++-- + crmd/remote_lrmd_ra.c | 18 +++++++++++------- + 4 files changed, 39 insertions(+), 10 deletions(-) + +diff --git a/crmd/cib.c b/crmd/cib.c +index e8c6376..a9e4ed3 100644 +--- a/crmd/cib.c ++++ b/crmd/cib.c +@@ -247,12 +247,21 @@ cib_delete_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, + // Node's lrm section (name 1x) + #define XPATH_NODE_LRM XPATH_NODE_STATE "/" XML_CIB_TAG_LRM + ++// Node's lrm_rsc_op entries and lrm_resource entries without lock (name 2x) ++#define XPATH_NODE_LRM_UNLOCKED XPATH_NODE_STATE "//" XML_LRM_TAG_RSC_OP \ ++ "|" XPATH_NODE_STATE \ ++ "//" XML_LRM_TAG_RESOURCE \ ++ "[not(@" XML_CONFIG_ATTR_SHUTDOWN_LOCK ")]" ++ + // Node's transient_attributes section (name 1x) + #define XPATH_NODE_ATTRS XPATH_NODE_STATE "/" XML_TAG_TRANSIENT_NODEATTRS + + // Everything under node_state (name 1x) + #define XPATH_NODE_ALL XPATH_NODE_STATE "/*" + ++// Unlocked history + transient attributes (name 3x) ++#define XPATH_NODE_ALL_UNLOCKED XPATH_NODE_LRM_UNLOCKED "|" XPATH_NODE_ATTRS ++ + /*! + * \internal + * \brief Delete subsection of a node's CIB node_state +@@ -274,6 +283,11 @@ controld_delete_node_state(const char *uname, enum controld_section_e section, + xpath = crm_strdup_printf(XPATH_NODE_LRM, uname); + desc = crm_strdup_printf("resource history for node %s", uname); + break; ++ case controld_section_lrm_unlocked: ++ xpath = crm_strdup_printf(XPATH_NODE_LRM_UNLOCKED, uname, uname); ++ desc = crm_strdup_printf("resource history (other than shutdown " ++ "locks) for node %s", uname); ++ break; + case controld_section_attrs: + xpath = crm_strdup_printf(XPATH_NODE_ATTRS, uname); + desc = crm_strdup_printf("transient attributes for node %s", uname); +@@ -282,6 +296,12 @@ controld_delete_node_state(const char *uname, enum controld_section_e section, + xpath = crm_strdup_printf(XPATH_NODE_ALL, uname); + desc = crm_strdup_printf("all state for node %s", uname); + break; ++ case controld_section_all_unlocked: ++ xpath = crm_strdup_printf(XPATH_NODE_ALL_UNLOCKED, ++ uname, uname, uname); ++ desc = crm_strdup_printf("all state (other than shutdown locks) " ++ "for node %s", uname); ++ break; + } + + if (fsa_cib_conn == NULL) { +@@ -290,7 +310,7 @@ controld_delete_node_state(const char *uname, enum controld_section_e section, + } else { + int call_id; + +- options |= cib_quorum_override|cib_xpath; ++ options |= cib_quorum_override|cib_xpath|cib_multiple; + call_id = fsa_cib_conn->cmds->delete(fsa_cib_conn, xpath, NULL, options); + crm_info("Deleting %s (via CIB call %d) " CRM_XS " xpath=%s", + desc, call_id, xpath); +diff --git a/crmd/crmd_utils.h b/crmd/crmd_utils.h +index 9ecce88..77dcfc2 100644 +--- a/crmd/crmd_utils.h ++++ b/crmd/crmd_utils.h +@@ -120,8 +120,10 @@ bool controld_action_is_recordable(const char *action); + // Subsections of node_state + enum controld_section_e { + controld_section_lrm, ++ controld_section_lrm_unlocked, + controld_section_attrs, + controld_section_all, ++ controld_section_all_unlocked + }; + + void controld_delete_node_state(const char *uname, +diff --git a/crmd/join_dc.c b/crmd/join_dc.c +index 8284695..1553078 100644 +--- a/crmd/join_dc.c ++++ b/crmd/join_dc.c +@@ -534,6 +534,7 @@ do_dc_join_ack(long long action, + int join_id = -1; + int call_id = 0; + ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg); ++ enum controld_section_e section = controld_section_lrm; + + const char *op = crm_element_value(join_ack->msg, F_CRM_TASK); + const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM); +@@ -583,8 +584,10 @@ do_dc_join_ack(long long action, + /* Update CIB with node's current LRM state. A new transition will be + * triggered later, when the CIB notifies us of the change. + */ +- controld_delete_node_state(join_from, controld_section_lrm, +- cib_scope_local); ++ if (controld_shutdown_lock_enabled) { ++ section = controld_section_lrm_unlocked; ++ } ++ controld_delete_node_state(join_from, section, cib_scope_local); + if (safe_str_eq(join_from, fsa_our_uname)) { + xmlNode *now_dc_lrmd_state = controld_query_executor_state(fsa_our_uname); + +diff --git a/crmd/remote_lrmd_ra.c b/crmd/remote_lrmd_ra.c +index c4f58d6..3870431 100644 +--- a/crmd/remote_lrmd_ra.c ++++ b/crmd/remote_lrmd_ra.c +@@ -1,5 +1,5 @@ +-/* +- * Copyright 2013-2019 the Pacemaker project contributors ++/* ++ * Copyright 2013-2020 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -191,17 +191,21 @@ remote_node_up(const char *node_name) + int call_opt, call_id = 0; + xmlNode *update, *state; + crm_node_t *node; ++ enum controld_section_e section = controld_section_all; + + CRM_CHECK(node_name != NULL, return); + crm_info("Announcing pacemaker_remote node %s", node_name); + +- /* Clear node's entire state (resource history and transient attributes). +- * The transient attributes should and normally will be cleared when the +- * node leaves, but since remote node state has a number of corner cases, +- * clear them here as well, to be sure. ++ /* Clear node's entire state (resource history and transient attributes) ++ * other than shutdown locks. The transient attributes should and normally ++ * will be cleared when the node leaves, but since remote node state has a ++ * number of corner cases, clear them here as well, to be sure. + */ + call_opt = crmd_cib_smart_opt(); +- controld_delete_node_state(node_name, controld_section_all, call_opt); ++ if (controld_shutdown_lock_enabled) { ++ section = controld_section_all_unlocked; ++ } ++ controld_delete_node_state(node_name, section, call_opt); + + /* Clear node's probed attribute */ + update_attrd(node_name, CRM_OP_PROBED, NULL, NULL, TRUE); +-- +1.8.3.1 + diff --git a/SOURCES/shutdown-lock-06.patch b/SOURCES/shutdown-lock-06.patch new file mode 100644 index 0000000..dca6b41 --- /dev/null +++ b/SOURCES/shutdown-lock-06.patch @@ -0,0 +1,39 @@ +From f629c2de1d011e8ed7a3f1d14811681672f424d5 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 16 Jan 2020 12:34:21 -0600 +Subject: [PATCH 06/10] Low: scheduler: display when a resource is + shutdown-locked to a node + +... so it shows up in logs and cluster status displays +--- + lib/pengine/native.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/lib/pengine/native.c b/lib/pengine/native.c +index 3c9f8f5..c66cada 100644 +--- a/lib/pengine/native.c ++++ b/lib/pengine/native.c +@@ -538,6 +538,9 @@ common_print(resource_t * rsc, const char *pre_text, const char *name, node_t *n + return; + } + ++ if ((node == NULL) && (rsc->lock_node != NULL)) { ++ node = rsc->lock_node; ++ } + if ((options & pe_print_rsconly) || g_list_length(rsc->running_on) > 1) { + node = NULL; + } +@@ -593,6 +596,10 @@ common_print(resource_t * rsc, const char *pre_text, const char *name, node_t *n + flagOffset += snprintf(flagBuffer + flagOffset, LINE_MAX - flagOffset, + "%sUNCLEAN", comma_if(flagOffset)); + } ++ if (node == rsc->lock_node) { ++ flagOffset += snprintf(flagBuffer + flagOffset, LINE_MAX - flagOffset, ++ "%sLOCKED", comma_if(flagOffset)); ++ } + } + + if (options & pe_print_pending) { +-- +1.8.3.1 + diff --git a/SOURCES/shutdown-lock-07.patch b/SOURCES/shutdown-lock-07.patch new file mode 100644 index 0000000..11239fe --- /dev/null +++ b/SOURCES/shutdown-lock-07.patch @@ -0,0 +1,29 @@ +From e487cd68049b1da2a9646bba2f0fb26cae4022d1 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Tue, 14 Jan 2020 16:01:16 -0600 +Subject: [PATCH 07/10] Low: tools: crm_resource resource checks should show + shutdown locks + +--- + tools/crm_resource_runtime.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c +index 9e70db8..9b93928 100644 +--- a/tools/crm_resource_runtime.c ++++ b/tools/crm_resource_runtime.c +@@ -907,6 +907,11 @@ cli_resource_check(cib_t * cib_conn, resource_t *rsc) + } + free(managed); + ++ if (rsc->lock_node) { ++ printf("%s * '%s' is locked to node %s due to shutdown\n", ++ (printed? "" : "\n"), parent->id, rsc->lock_node->details->uname); ++ } ++ + if (printed) { + printf("\n"); + } +-- +1.8.3.1 + diff --git a/SOURCES/shutdown-lock-08.patch b/SOURCES/shutdown-lock-08.patch new file mode 100644 index 0000000..57ebb49 --- /dev/null +++ b/SOURCES/shutdown-lock-08.patch @@ -0,0 +1,191 @@ +From 3f19b4333897a0392e8a5efc4742732b8e6f0efb Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Tue, 14 Jan 2020 12:53:39 -0600 +Subject: [PATCH 08/10] Low: controller: allow CRM_OP_LRM_DELETE to clear CIB + only + +Normally, CRM_OP_LRM_DELETE is relayed to the affected node's controller, which +clears the resource from the executor and CIB as well the its own bookkeeping. + +Now, we want to be able to use it to clear shutdown locks for nodes that are +down. Let it take a new "mode" attribute, and if it is "cib", clear the +resource from the CIB locally without relaying the operation or doing anything +else. +--- + crmd/lrm.c | 4 ++- + crmd/messages.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++-- + crmd/te_actions.c | 7 ++++ + include/crm_internal.h | 2 ++ + 4 files changed, 106 insertions(+), 4 deletions(-) + +diff --git a/crmd/lrm.c b/crmd/lrm.c +index 9156ab8..bdf7b94 100644 +--- a/crmd/lrm.c ++++ b/crmd/lrm.c +@@ -1764,7 +1764,9 @@ do_lrm_invoke(long long action, + crm_trace("LRM %s command from %s", crm_op, from_sys); + + if (safe_str_eq(crm_op, CRM_OP_LRM_DELETE)) { +- crm_rsc_delete = TRUE; // Only crm_resource uses this op ++ if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) { ++ crm_rsc_delete = TRUE; // from crm_resource ++ } + operation = CRMD_ACTION_DELETE; + + } else if (safe_str_eq(crm_op, CRM_OP_LRM_FAIL)) { +diff --git a/crmd/messages.c b/crmd/messages.c +index f1599ab..8839d65 100644 +--- a/crmd/messages.c ++++ b/crmd/messages.c +@@ -430,6 +430,14 @@ relay_message(xmlNode * msg, gboolean originated_locally) + + } else if (safe_str_eq(fsa_our_uname, host_to)) { + is_local = 1; ++ } else if (is_for_crm && safe_str_eq(task, CRM_OP_LRM_DELETE)) { ++ xmlNode *msg_data = get_message_xml(msg, F_CRM_DATA); ++ const char *mode = crm_element_value(msg_data, PCMK__XA_MODE); ++ ++ if (safe_str_eq(mode, XML_TAG_CIB)) { ++ // Local delete of an offline node's resource history ++ is_local = 1; ++ } + } + + if (is_for_dc || is_for_dcib || is_for_te) { +@@ -669,6 +677,86 @@ handle_failcount_op(xmlNode * stored_msg) + return I_NULL; + } + ++static enum crmd_fsa_input ++handle_lrm_delete(xmlNode *stored_msg) ++{ ++ const char *mode = NULL; ++ xmlNode *msg_data = get_message_xml(stored_msg, F_CRM_DATA); ++ ++ CRM_CHECK(msg_data != NULL, return I_NULL); ++ ++ /* CRM_OP_LRM_DELETE has two distinct modes. The default behavior is to ++ * relay the operation to the affected node, which will unregister the ++ * resource from the local executor, clear the resource's history from the ++ * CIB, and do some bookkeeping in the controller. ++ * ++ * However, if the affected node is offline, the client will specify ++ * mode="cib" which means the controller receiving the operation should ++ * clear the resource's history from the CIB and nothing else. This is used ++ * to clear shutdown locks. ++ */ ++ mode = crm_element_value(msg_data, PCMK__XA_MODE); ++ if ((mode == NULL) || strcmp(mode, XML_TAG_CIB)) { ++ // Relay to affected node ++ crm_xml_add(stored_msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD); ++ return I_ROUTER; ++ ++ } else { ++ // Delete CIB history locally (compare with do_lrm_delete()) ++ const char *from_sys = NULL; ++ const char *user_name = NULL; ++ const char *rsc_id = NULL; ++ const char *node = NULL; ++ xmlNode *rsc_xml = NULL; ++ int rc = pcmk_rc_ok; ++ ++ rsc_xml = first_named_child(msg_data, XML_CIB_TAG_RESOURCE); ++ CRM_CHECK(rsc_xml != NULL, return I_NULL); ++ ++ rsc_id = ID(rsc_xml); ++ from_sys = crm_element_value(stored_msg, F_CRM_SYS_FROM); ++ node = crm_element_value(msg_data, XML_LRM_ATTR_TARGET); ++#if ENABLE_ACL ++ user_name = crm_acl_get_set_user(stored_msg, F_CRM_USER, NULL); ++#endif ++ crm_debug("Handling " CRM_OP_LRM_DELETE " for %s on %s locally%s%s " ++ "(clearing CIB resource history only)", rsc_id, node, ++ (user_name? " for user " : ""), (user_name? user_name : "")); ++#if ENABLE_ACL ++ rc = controld_delete_resource_history(rsc_id, node, user_name, ++ cib_dryrun|cib_sync_call); ++#endif ++ if (rc == pcmk_rc_ok) { ++ rc = controld_delete_resource_history(rsc_id, node, user_name, ++ crmd_cib_smart_opt()); ++ } ++ ++ // Notify client if not from graph (compare with notify_deleted()) ++ if (from_sys && strcmp(from_sys, CRM_SYSTEM_TENGINE)) { ++ lrmd_event_data_t *op = NULL; ++ const char *from_host = crm_element_value(stored_msg, ++ F_CRM_HOST_FROM); ++ const char *transition = crm_element_value(msg_data, ++ XML_ATTR_TRANSITION_KEY); ++ ++ crm_info("Notifying %s on %s that %s was%s deleted", ++ from_sys, (from_host? from_host : "local node"), rsc_id, ++ ((rc == pcmk_rc_ok)? "" : " not")); ++ op = lrmd_new_event(rsc_id, CRMD_ACTION_DELETE, 0); ++ op->type = lrmd_event_exec_complete; ++ op->user_data = strdup(transition? transition : FAKE_TE_ID); ++ op->params = crm_str_table_new(); ++ g_hash_table_insert(op->params, strdup(XML_ATTR_CRM_VERSION), ++ strdup(CRM_FEATURE_SET)); ++ controld_rc2event(op, rc); ++ controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id); ++ lrmd_free_event(op); ++ controld_trigger_delete_refresh(from_sys, rsc_id); ++ } ++ return I_NULL; ++ } ++} ++ + /*! + * \brief Handle a CRM_OP_REMOTE_STATE message by updating remote peer cache + * +@@ -902,9 +990,12 @@ handle_request(xmlNode * stored_msg, enum crmd_fsa_cause cause) + crm_debug("Raising I_JOIN_RESULT: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID)); + return I_JOIN_RESULT; + +- } else if (strcmp(op, CRM_OP_LRM_DELETE) == 0 +- || strcmp(op, CRM_OP_LRM_FAIL) == 0 +- || strcmp(op, CRM_OP_LRM_REFRESH) == 0 || strcmp(op, CRM_OP_REPROBE) == 0) { ++ } else if (strcmp(op, CRM_OP_LRM_DELETE) == 0) { ++ return handle_lrm_delete(stored_msg); ++ ++ } else if ((strcmp(op, CRM_OP_LRM_FAIL) == 0) ++ || (strcmp(op, CRM_OP_LRM_REFRESH) == 0) ++ || (strcmp(op, CRM_OP_REPROBE) == 0)) { + + crm_xml_add(stored_msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD); + return I_ROUTER; +diff --git a/crmd/te_actions.c b/crmd/te_actions.c +index 19bb199..ec92df2 100644 +--- a/crmd/te_actions.c ++++ b/crmd/te_actions.c +@@ -239,6 +239,13 @@ te_crm_command(crm_graph_t * graph, crm_action_t * action) + + if (!router_node) { + router_node = on_node; ++ if (safe_str_eq(task, CRM_OP_LRM_DELETE)) { ++ const char *mode = crm_element_value(action->xml, PCMK__XA_MODE); ++ ++ if (safe_str_eq(mode, XML_TAG_CIB)) { ++ router_node = fsa_our_uname; ++ } ++ } + } + + CRM_CHECK(on_node != NULL && strlen(on_node) != 0, +diff --git a/include/crm_internal.h b/include/crm_internal.h +index 0adeb7b..7656bf5 100644 +--- a/include/crm_internal.h ++++ b/include/crm_internal.h +@@ -260,6 +260,8 @@ long crm_read_pidfile(const char *filename); + # define ATTRD_OP_SYNC_RESPONSE "sync-response" + # define ATTRD_OP_CLEAR_FAILURE "clear-failure" + ++# define PCMK__XA_MODE "mode" ++ + # define PCMK_ENV_PHYSICAL_HOST "physical_host" + + +-- +1.8.3.1 + diff --git a/SOURCES/shutdown-lock-09.patch b/SOURCES/shutdown-lock-09.patch new file mode 100644 index 0000000..5af52c6 --- /dev/null +++ b/SOURCES/shutdown-lock-09.patch @@ -0,0 +1,56 @@ +From 8ab8d64976212207935bf165c26db32c1d79c344 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Tue, 14 Jan 2020 16:24:08 -0600 +Subject: [PATCH 09/10] Low: tools: for down nodes, crm_resource --refresh + should clear CIB only + +This provides a mechanism to manually clear shutdown locks. +--- + tools/crm_resource_runtime.c | 16 +++++++++++++--- + 1 file changed, 13 insertions(+), 3 deletions(-) + +diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c +index 9b93928..acb9dbe 100644 +--- a/tools/crm_resource_runtime.c ++++ b/tools/crm_resource_runtime.c +@@ -458,6 +458,7 @@ send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op, + const char *rsc_type = NULL; + xmlNode *params = NULL; + xmlNode *msg_data = NULL; ++ bool cib_only = false; + resource_t *rsc = pe_find_resource(data_set->resources, rsc_id); + + if (rsc == NULL) { +@@ -489,10 +490,14 @@ send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op, + } + + if (!(node->details->online)) { +- CMD_ERR("Node %s is not online", host_uname); +- return -ENOTCONN; ++ if (strcmp(op, CRM_OP_LRM_DELETE) == 0) { ++ cib_only = true; ++ } else { ++ CMD_ERR("Node %s is not online", host_uname); ++ return -ENOTCONN; ++ } + } +- if (node && is_remote_node(node)) { ++ if (!cib_only && node && is_remote_node(node)) { + node = pe__current_node(node->details->remote_rsc); + if (node == NULL) { + CMD_ERR("No lrmd connection detected to remote node %s", host_uname); +@@ -517,6 +522,11 @@ send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op, + crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node); + } + ++ if (cib_only) { ++ // Indicate that only the CIB needs to be cleaned ++ crm_xml_add(msg_data, PCMK__XA_MODE, XML_TAG_CIB); ++ } ++ + xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE); + if (rsc->clone_name) { + crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->clone_name); +-- +1.8.3.1 + diff --git a/SOURCES/shutdown-lock-10.patch b/SOURCES/shutdown-lock-10.patch new file mode 100644 index 0000000..45a9d3f --- /dev/null +++ b/SOURCES/shutdown-lock-10.patch @@ -0,0 +1,225 @@ +From abbfc0ba9afb6c2d1ce54fea2d0cf25ce1d9108a Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Fri, 10 Jan 2020 18:18:07 -0600 +Subject: [PATCH 10/10] Low: scheduler: clear resource history when appropriate + +Tell the controller to clear resource history from the CIB when a resource has +a shutdown lock that expired or was cancelled because the resource is already +active elsewhere. +--- + include/crm/pengine/internal.h | 6 +++++- + include/crm/pengine/status.h | 6 +++++- + lib/pengine/unpack.c | 1 + + lib/pengine/utils.c | 34 ++++++++++++++++++++++++++++++++-- + pengine/allocate.c | 1 + + pengine/graph.c | 16 ++++++++++++++-- + pengine/native.c | 6 ++++++ + 7 files changed, 64 insertions(+), 6 deletions(-) + +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index fc908e8..64b9a50 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -1,5 +1,7 @@ + /* +- * Copyright 2004-2019 Andrew Beekhof <andrew@beekhof.net> ++ * Copyright 2004-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. +@@ -366,4 +368,7 @@ void pe__free_param_checks(pe_working_set_t *data_set); + void pe__free_param_checks(pe_working_set_t *data_set); + + bool pe__shutdown_requested(pe_node_t *node); ++pe_action_t *pe__clear_resource_history(pe_resource_t *rsc, pe_node_t *node, ++ pe_working_set_t *data_set); ++ + #endif +diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h +index 1e8d5bb..9f9fd3b 100644 +--- a/include/crm/pengine/status.h ++++ b/include/crm/pengine/status.h +@@ -1,5 +1,7 @@ + /* +- * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net> ++ * Copyright 2004-2020 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public +@@ -280,6 +282,8 @@ enum pe_action_flags { + + pe_action_reschedule = 0x02000, + pe_action_tracking = 0x04000, ++ ++ pe_action_dc = 0x10000, //! Action may run on DC instead of target + }; + /* *INDENT-ON* */ + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index bb5efa4..9deff67 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -2306,6 +2306,7 @@ unpack_shutdown_lock(xmlNode *rsc_entry, pe_resource_t *rsc, pe_node_t *node, + > (lock_time + data_set->shutdown_lock))) { + pe_rsc_info(rsc, "Shutdown lock for %s on %s expired", + rsc->id, node->details->uname); ++ pe__clear_resource_history(rsc, node, data_set); + } else { + rsc->lock_node = node; + rsc->lock_time = lock_time; +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index c336e37..84d3399 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -490,6 +490,11 @@ custom_action(resource_t * rsc, char *key, const char *task, + } + action->uuid = strdup(key); + ++ if (safe_str_eq(task, CRM_OP_LRM_DELETE)) { ++ // Resource history deletion for a node can be done on the DC ++ pe_set_action_bit(action, pe_action_dc); ++ } ++ + pe_set_action_bit(action, pe_action_runnable); + if (optional) { + pe_rsc_trace(rsc, "Set optional on %s", action->uuid); +@@ -570,7 +575,8 @@ custom_action(resource_t * rsc, char *key, const char *task, + pe_set_action_bit(action, pe_action_optional); + /* action->runnable = FALSE; */ + +- } else if (action->node->details->online == FALSE ++ } else if (is_not_set(action->flags, pe_action_dc) ++ && !(action->node->details->online) + && (!is_container_remote_node(action->node) || action->node->details->remote_requires_reset)) { + pe_clear_action_bit(action, pe_action_runnable); + do_crm_log(warn_level, "Action %s on %s is unrunnable (offline)", +@@ -581,7 +587,8 @@ custom_action(resource_t * rsc, char *key, const char *task, + pe_fence_node(data_set, action->node, "resource actions are unrunnable"); + } + +- } else if (action->node->details->pending) { ++ } else if (is_not_set(action->flags, pe_action_dc) ++ && action->node->details->pending) { + pe_clear_action_bit(action, pe_action_runnable); + do_crm_log(warn_level, "Action %s on %s is unrunnable (pending)", + action->uuid, action->node->details->uname); +@@ -695,6 +702,8 @@ unpack_operation_on_fail(action_t * action) + + value = on_fail; + } ++ } else if (safe_str_eq(action->task, CRM_OP_LRM_DELETE)) { ++ value = "ignore"; + } + + return value; +@@ -2566,3 +2575,24 @@ pe__shutdown_requested(pe_node_t *node) + + return shutdown && strcmp(shutdown, "0"); + } ++ ++/*! ++ * \internal ++ * \brief Create an action to clear a resource's history from CIB ++ * ++ * \param[in] rsc Resource to clear ++ * \param[in] node Node to clear history on ++ * ++ * \return New action to clear resource history ++ */ ++pe_action_t * ++pe__clear_resource_history(pe_resource_t *rsc, pe_node_t *node, ++ pe_working_set_t *data_set) ++{ ++ char *key = NULL; ++ ++ CRM_ASSERT(rsc && node); ++ key = generate_op_key(rsc->id, CRM_OP_LRM_DELETE, 0); ++ return custom_action(rsc, key, CRM_OP_LRM_DELETE, node, FALSE, TRUE, ++ data_set); ++} +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 7366716..946f063 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1058,6 +1058,7 @@ apply_shutdown_lock(pe_resource_t *rsc, pe_working_set_t *data_set) + pe_rsc_info(rsc, + "Cancelling shutdown lock because %s is already active", + rsc->id); ++ pe__clear_resource_history(rsc, rsc->lock_node, data_set); + rsc->lock_node = NULL; + rsc->lock_time = 0; + } +diff --git a/pengine/graph.c b/pengine/graph.c +index 33168ca..a045549 100644 +--- a/pengine/graph.c ++++ b/pengine/graph.c +@@ -596,10 +596,11 @@ update_action(action_t * then) + + /* 'then' is required, so we must abandon 'first' + * (e.g. a required stop cancels any reload). +- * Only used with reload actions as 'first'. + */ + set_bit(other->action->flags, pe_action_optional); +- clear_bit(first->rsc->flags, pe_rsc_reload); ++ if (!strcmp(first->task, CRMD_ACTION_RELOAD)) { ++ clear_bit(first->rsc->flags, pe_rsc_reload); ++ } + } + + if (first->rsc && then->rsc && (first->rsc != then->rsc) +@@ -1051,6 +1052,11 @@ action2xml(action_t * action, gboolean as_input, pe_working_set_t *data_set) + } else if (safe_str_eq(action->task, CRM_OP_LRM_REFRESH)) { + action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT); + ++ } else if (safe_str_eq(action->task, CRM_OP_LRM_DELETE)) { ++ // CIB-only clean-up for shutdown locks ++ action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT); ++ crm_xml_add(action_xml, PCMK__XA_MODE, XML_TAG_CIB); ++ + /* } else if(safe_str_eq(action->task, RSC_PROBED)) { */ + /* action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT); */ + +@@ -1063,6 +1069,7 @@ action2xml(action_t * action, gboolean as_input, pe_working_set_t *data_set) + + } else { + action_xml = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP); ++ + #if ENABLE_VERSIONED_ATTRS + rsc_details = pe_rsc_action_details(action); + #endif +@@ -1404,6 +1411,11 @@ should_dump_action(action_t * action) + log_action(LOG_DEBUG, "Unallocated action", action, FALSE); + return FALSE; + ++ } else if (is_set(action->flags, pe_action_dc)) { ++ crm_trace("Action %s (%d) should be dumped: " ++ "can run on DC instead of %s", ++ action->uuid, action->id, action->node->details->uname); ++ + } else if(is_container_remote_node(action->node) && action->node->details->remote_requires_reset == FALSE) { + crm_trace("Assuming action %s for %s will be runnable", action->uuid, action->node->details->uname); + +diff --git a/pengine/native.c b/pengine/native.c +index 1abaf29..b639fae 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -1439,6 +1439,12 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) + pe_order_runnable_left, data_set); + } + ++ // Don't clear resource history if probing on same node ++ custom_action_order(rsc, generate_op_key(rsc->id, CRM_OP_LRM_DELETE, 0), ++ NULL, rsc, generate_op_key(rsc->id, RSC_STATUS, 0), ++ NULL, pe_order_same_node|pe_order_then_cancels_first, ++ data_set); ++ + // Certain checks need allowed nodes + if (check_unfencing || check_utilization || rsc->container) { + allowed_nodes = allowed_nodes_as_list(rsc, data_set); +-- +1.8.3.1 + diff --git a/SOURCES/shutdown-lock-11.patch b/SOURCES/shutdown-lock-11.patch new file mode 100644 index 0000000..d2f46af --- /dev/null +++ b/SOURCES/shutdown-lock-11.patch @@ -0,0 +1,726 @@ +From 6fa96ab26752f03ef3d9f08b22e91dba0ca6085c Mon Sep 17 00:00:00 2001 +From: Ken Gaillot <kgaillot@redhat.com> +Date: Thu, 16 Jan 2020 14:48:05 -0600 +Subject: [PATCH] Test: scheduler: add regression tests for shutdown locks + +--- + pengine/regression.sh | 2 + + pengine/test10/shutdown-lock-expiration.dot | 11 ++ + pengine/test10/shutdown-lock-expiration.exp | 68 +++++++++ + pengine/test10/shutdown-lock-expiration.scores | 17 +++ + pengine/test10/shutdown-lock-expiration.summary | 31 ++++ + pengine/test10/shutdown-lock-expiration.xml | 186 ++++++++++++++++++++++++ + pengine/test10/shutdown-lock.dot | 11 ++ + pengine/test10/shutdown-lock.exp | 64 ++++++++ + pengine/test10/shutdown-lock.scores | 17 +++ + pengine/test10/shutdown-lock.summary | 31 ++++ + pengine/test10/shutdown-lock.xml | 185 +++++++++++++++++++++++ + 11 files changed, 623 insertions(+) + create mode 100644 pengine/test10/shutdown-lock-expiration.dot + create mode 100644 pengine/test10/shutdown-lock-expiration.exp + create mode 100644 pengine/test10/shutdown-lock-expiration.scores + create mode 100644 pengine/test10/shutdown-lock-expiration.summary + create mode 100644 pengine/test10/shutdown-lock-expiration.xml + create mode 100644 pengine/test10/shutdown-lock.dot + create mode 100644 pengine/test10/shutdown-lock.exp + create mode 100644 pengine/test10/shutdown-lock.scores + create mode 100644 pengine/test10/shutdown-lock.summary + create mode 100644 pengine/test10/shutdown-lock.xml + +diff --git a/pengine/regression.sh b/pengine/regression.sh +index f2226ed..abb9708 100755 +--- a/pengine/regression.sh ++++ b/pengine/regression.sh +@@ -894,6 +894,8 @@ do_test remote-connection-unrecoverable "Remote connection host must be fenced, + echo "" + do_test resource-discovery "Exercises resource-discovery location constraint option." + do_test rsc-discovery-per-node "Disable resource discovery per node" ++do_test shutdown-lock "Ensure shutdown lock works properly" ++do_test shutdown-lock-expiration "Ensure shutdown lock expiration works properly" + + echo "" + do_test isolation-start-all "Start docker isolated resources." +diff --git a/pengine/test10/shutdown-lock-expiration.dot b/pengine/test10/shutdown-lock-expiration.dot +new file mode 100644 +index 0000000..cd1a4fd +--- /dev/null ++++ b/pengine/test10/shutdown-lock-expiration.dot +@@ -0,0 +1,11 @@ ++digraph "g" { ++"Fencing_monitor_120000 node3" [ style=bold color="green" fontcolor="black"] ++"Fencing_start_0 node3" -> "Fencing_monitor_120000 node3" [ style = bold] ++"Fencing_start_0 node3" [ style=bold color="green" fontcolor="black"] ++"Fencing_stop_0 node3" -> "Fencing_start_0 node3" [ style = bold] ++"Fencing_stop_0 node3" [ style=bold color="green" fontcolor="black"] ++"rsc2_lrm_delete_0 node2" [ style=bold color="green" fontcolor="black"] ++"rsc2_monitor_10000 node4" [ style=bold color="green" fontcolor="black"] ++"rsc2_start_0 node4" -> "rsc2_monitor_10000 node4" [ style = bold] ++"rsc2_start_0 node4" [ style=bold color="green" fontcolor="black"] ++} +diff --git a/pengine/test10/shutdown-lock-expiration.exp b/pengine/test10/shutdown-lock-expiration.exp +new file mode 100644 +index 0000000..6f3d139 +--- /dev/null ++++ b/pengine/test10/shutdown-lock-expiration.exp +@@ -0,0 +1,68 @@ ++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="1" transition_id="0"> ++ <synapse id="0"> ++ <action_set> ++ <rsc_op id="4" operation="stop" operation_key="Fencing_stop_0" on_node="node3" on_node_uuid="3"> ++ <primitive id="Fencing" class="stonith" type="fence_xvm"/> ++ <attributes CRM_meta_name="stop" CRM_meta_on_node="node3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="60000" key_file="/etc/pacemaker/fence_xvm.key" multicast_address="239.255.100.100" pcmk_host_list="node1 node2 node3 node4 node5"/> ++ </rsc_op> ++ </action_set> ++ <inputs/> ++ </synapse> ++ <synapse id="1"> ++ <action_set> ++ <rsc_op id="3" operation="start" operation_key="Fencing_start_0" on_node="node3" on_node_uuid="3"> ++ <primitive id="Fencing" class="stonith" type="fence_xvm"/> ++ <attributes CRM_meta_name="start" CRM_meta_on_node="node3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="60000" key_file="/etc/pacemaker/fence_xvm.key" multicast_address="239.255.100.100" pcmk_host_list="node1 node2 node3 node4 node5"/> ++ </rsc_op> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="4" operation="stop" operation_key="Fencing_stop_0" on_node="node3" on_node_uuid="3"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="2"> ++ <action_set> ++ <rsc_op id="2" operation="monitor" operation_key="Fencing_monitor_120000" on_node="node3" on_node_uuid="3"> ++ <primitive id="Fencing" class="stonith" type="fence_xvm"/> ++ <attributes CRM_meta_interval="120000" CRM_meta_name="monitor" CRM_meta_on_node="node3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="120000" key_file="/etc/pacemaker/fence_xvm.key" multicast_address="239.255.100.100" pcmk_host_list="node1 node2 node3 node4 node5"/> ++ </rsc_op> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="3" operation="start" operation_key="Fencing_start_0" on_node="node3" on_node_uuid="3"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="3"> ++ <action_set> ++ <rsc_op id="6" operation="monitor" operation_key="rsc2_monitor_10000" on_node="node4" on_node_uuid="4"> ++ <primitive id="rsc2" class="ocf" provider="pacemaker" type="Dummy"/> ++ <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="node4" CRM_meta_on_node_uuid="4" CRM_meta_timeout="20000" /> ++ </rsc_op> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="5" operation="start" operation_key="rsc2_start_0" on_node="node4" on_node_uuid="4"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="4"> ++ <action_set> ++ <rsc_op id="5" operation="start" operation_key="rsc2_start_0" on_node="node4" on_node_uuid="4"> ++ <primitive id="rsc2" class="ocf" provider="pacemaker" type="Dummy"/> ++ <attributes CRM_meta_name="start" CRM_meta_on_node="node4" CRM_meta_on_node_uuid="4" CRM_meta_timeout="20000" /> ++ </rsc_op> ++ </action_set> ++ <inputs/> ++ </synapse> ++ <synapse id="5"> ++ <action_set> ++ <crm_event mode="cib" id="1" operation="lrm_delete" operation_key="rsc2_lrm_delete_0" on_node="node2" on_node_uuid="2"> ++ <primitive id="rsc2" class="ocf" provider="pacemaker" type="Dummy"/> ++ <attributes CRM_meta_on_node="node2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="90000" /> ++ </crm_event> ++ </action_set> ++ <inputs/> ++ </synapse> ++</transition_graph> +diff --git a/pengine/test10/shutdown-lock-expiration.scores b/pengine/test10/shutdown-lock-expiration.scores +new file mode 100644 +index 0000000..e5d435d +--- /dev/null ++++ b/pengine/test10/shutdown-lock-expiration.scores +@@ -0,0 +1,17 @@ ++Allocation scores: ++Using the original execution date of: 2020-01-06 22:11:40Z ++native_color: Fencing allocation score on node1: 0 ++native_color: Fencing allocation score on node2: 0 ++native_color: Fencing allocation score on node3: 0 ++native_color: Fencing allocation score on node4: 0 ++native_color: Fencing allocation score on node5: 0 ++native_color: rsc1 allocation score on node1: INFINITY ++native_color: rsc1 allocation score on node2: -INFINITY ++native_color: rsc1 allocation score on node3: -INFINITY ++native_color: rsc1 allocation score on node4: -INFINITY ++native_color: rsc1 allocation score on node5: -INFINITY ++native_color: rsc2 allocation score on node1: 0 ++native_color: rsc2 allocation score on node2: INFINITY ++native_color: rsc2 allocation score on node3: 0 ++native_color: rsc2 allocation score on node4: 0 ++native_color: rsc2 allocation score on node5: 0 +diff --git a/pengine/test10/shutdown-lock-expiration.summary b/pengine/test10/shutdown-lock-expiration.summary +new file mode 100644 +index 0000000..08c93aa +--- /dev/null ++++ b/pengine/test10/shutdown-lock-expiration.summary +@@ -0,0 +1,31 @@ ++Using the original execution date of: 2020-01-06 22:11:40Z ++ ++Current cluster status: ++Online: [ node3 node4 node5 ] ++OFFLINE: [ node1 node2 ] ++ ++ Fencing (stonith:fence_xvm): Started node3 ++ rsc1 (ocf::pacemaker:Dummy): Stopped node1 (LOCKED) ++ rsc2 (ocf::pacemaker:Dummy): Stopped ++ ++Transition Summary: ++ * Restart Fencing ( node3 ) due to resource definition change ++ * Start rsc2 ( node4 ) ++ ++Executing cluster transition: ++ * Resource action: Fencing stop on node3 ++ * Resource action: Fencing start on node3 ++ * Resource action: Fencing monitor=120000 on node3 ++ * Resource action: rsc2 start on node4 ++ * Cluster action: lrm_delete for rsc2 on node2 ++ * Resource action: rsc2 monitor=10000 on node4 ++Using the original execution date of: 2020-01-06 22:11:40Z ++ ++Revised cluster status: ++Online: [ node3 node4 node5 ] ++OFFLINE: [ node1 node2 ] ++ ++ Fencing (stonith:fence_xvm): Started node3 ++ rsc1 (ocf::pacemaker:Dummy): Stopped node1 (LOCKED) ++ rsc2 (ocf::pacemaker:Dummy): Started node4 ++ +diff --git a/pengine/test10/shutdown-lock-expiration.xml b/pengine/test10/shutdown-lock-expiration.xml +new file mode 100644 +index 0000000..6caa7cf +--- /dev/null ++++ b/pengine/test10/shutdown-lock-expiration.xml +@@ -0,0 +1,186 @@ ++<cib crm_feature_set="3.0.14" validate-with="pacemaker-2.10" epoch="155" num_updates="14" admin_epoch="0" cib-last-written="Mon Jan 6 16:05:00 2020" update-origin="node2" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="3" execution-date="1578348700"> ++ <configuration> ++ <crm_config> ++ <cluster_property_set id="cib-bootstrap-options"> ++ <nvpair id="cts-stonith-enabled" name="stonith-enabled" value="1"/> ++ <nvpair id="cts-start-failure-is-fatal" name="start-failure-is-fatal" value="false"/> ++ <nvpair id="cts-pe-input-series-max" name="pe-input-series-max" value="5000"/> ++ <nvpair id="cts-shutdown-escalation" name="shutdown-escalation" value="5min"/> ++ <nvpair id="cts-batch-limit" name="batch-limit" value="10"/> ++ <nvpair id="cts-dc-deadtime" name="dc-deadtime" value="5s"/> ++ <nvpair id="cts-no-quorum-policy" name="no-quorum-policy" value="stop"/> ++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/> ++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/> ++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="mycluster"/> ++ <!-- This regression test ensures that shutdown lock expiration works properly, for both ++ non-expired locks (rsc1 on node1) and expired locks (rsc2 on node2). This will also ++ ensure that the next recheck time is calculated properly. ++ --> ++ <nvpair id="cib-bootstrap-options-shutdown-lock" name="shutdown-lock" value="true"/> ++ <nvpair id="cib-bootstrap-options-shutdown-lock-limit" name="shutdown-lock-limit" value="5m"/> ++ </cluster_property_set> ++ </crm_config> ++ <nodes> ++ <node id="1" uname="node1"> ++ <instance_attributes id="nodes-1"> ++ <nvpair id="nodes-1-standby" name="standby" value="off"/> ++ </instance_attributes> ++ </node> ++ <node id="2" uname="node2"/> ++ <node id="3" uname="node3"> ++ <instance_attributes id="nodes-3"> ++ <nvpair id="nodes-3-standby" name="standby" value="off"/> ++ </instance_attributes> ++ </node> ++ <node id="4" uname="node4"> ++ <instance_attributes id="nodes-4"> ++ <nvpair id="nodes-4-standby" name="standby" value="off"/> ++ </instance_attributes> ++ </node> ++ <node id="5" uname="node5"/> ++ </nodes> ++ <resources> ++ <primitive class="stonith" id="Fencing" type="fence_xvm"> ++ <meta_attributes id="Fencing-meta"> ++ <nvpair id="Fencing-migration-threshold" name="migration-threshold" value="5"/> ++ </meta_attributes> ++ <instance_attributes id="Fencing-params"> ++ <nvpair id="Fencing-key_file" name="key_file" value="/etc/pacemaker/fence_xvm.key"/> ++ <nvpair id="Fencing-multicast_address" name="multicast_address" value="239.255.100.100"/> ++ <nvpair id="Fencing-pcmk_host_list" name="pcmk_host_list" value="node1 node2 node3 node4 node5"/> ++ </instance_attributes> ++ <operations> ++ <op id="Fencing-monitor-120s" interval="120s" name="monitor" timeout="120s"/> ++ <op id="Fencing-stop-0" interval="0" name="stop" timeout="60s"/> ++ <op id="Fencing-start-0" interval="0" name="start" timeout="60s"/> ++ </operations> ++ </primitive> ++ <primitive class="ocf" id="rsc1" provider="pacemaker" type="Dummy"> ++ <operations> ++ <op id="rsc1-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/> ++ <op id="rsc1-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/> ++ <op id="rsc1-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/> ++ <op id="rsc1-reload-interval-0s" interval="0s" name="reload" timeout="20s"/> ++ <op id="rsc1-start-interval-0s" interval="0s" name="start" timeout="20s"/> ++ <op id="rsc1-stop-interval-0s" interval="0s" name="stop" timeout="20s"/> ++ </operations> ++ </primitive> ++ <primitive class="ocf" id="rsc2" provider="pacemaker" type="Dummy"> ++ <operations> ++ <op id="rsc2-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/> ++ <op id="rsc2-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/> ++ <op id="rsc2-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/> ++ <op id="rsc2-reload-interval-0s" interval="0s" name="reload" timeout="20s"/> ++ <op id="rsc2-start-interval-0s" interval="0s" name="start" timeout="20s"/> ++ <op id="rsc2-stop-interval-0s" interval="0s" name="stop" timeout="20s"/> ++ </operations> ++ </primitive> ++ </resources> ++ <constraints> ++ <rsc_location id="location-rsc1-node1-INFINITY" node="node1" rsc="rsc1" score="INFINITY"/> ++ <rsc_location id="location-rsc2-node2-INFINITY" node="node2" rsc="rsc2" score="INFINITY"/> ++ </constraints> ++ <op_defaults> ++ <meta_attributes id="cts-op_defaults-meta"> ++ <nvpair id="cts-op_defaults-timeout" name="timeout" value="90s"/> ++ </meta_attributes> ++ </op_defaults> ++ <alerts> ++ <alert id="alert-1" path="/var/lib/pacemaker/notify.sh"> ++ <recipient id="alert-1-recipient-1" value="/run/crm/alert.log"/> ++ </alert> ++ </alerts> ++ <rsc_defaults> ++ <meta_attributes id="rsc_defaults-options"/> ++ </rsc_defaults> ++ </configuration> ++ <status> ++ <node_state id="2" uname="node2" in_ccm="false" crmd="offline" crm-debug-origin="post_cache_update" join="down" expected="down"> ++ <lrm id="2"> ++ <lrm_resources> ++ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker" shutdown-lock="1578348332"> ++ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="8:6:0:398bf005-bb15-4098-9a72-d8373456b457" transition-magic="0:0;8:6:0:398bf005-bb15-4098-9a72-d8373456b457" exit-reason="" on_node="node2" call-id="24" rc-code="0" op-status="0" interval="0" last-rc-change="1578348332" last-run="1578348332" exec-time="24" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ <lrm_rsc_op id="rsc2_monitor_10000" operation_key="rsc2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="9:4:0:398bf005-bb15-4098-9a72-d8373456b457" transition-magic="0:0;9:4:0:398bf005-bb15-4098-9a72-d8373456b457" exit-reason="" on_node="node2" call-id="21" rc-code="0" op-status="0" interval="10000" last-rc-change="1578348257" exec-time="21" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith"> ++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="3:3:7:398bf005-bb15-4098-9a72-d8373456b457" transition-magic="0:7;3:3:7:398bf005-bb15-4098-9a72-d8373456b457" exit-reason="" on_node="node2" call-id="10" rc-code="7" op-status="0" interval="0" last-rc-change="1578348256" last-run="1578348256" exec-time="6" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/> ++ </lrm_resource> ++ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="4:3:7:398bf005-bb15-4098-9a72-d8373456b457" transition-magic="0:7;4:3:7:398bf005-bb15-4098-9a72-d8373456b457" exit-reason="" on_node="node2" call-id="14" rc-code="7" op-status="0" interval="0" last-rc-change="1578348257" last-run="1578348257" exec-time="38" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state id="5" uname="node5" in_ccm="true" crmd="online" crm-debug-origin="post_cache_update" join="member" expected="member"> ++ <transient_attributes id="5"> ++ <instance_attributes id="status-5"/> ++ </transient_attributes> ++ <lrm id="5"> ++ <lrm_resources> ++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith"> ++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="61:0:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;61:0:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node5" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1578347670" last-run="1578347670" exec-time="7" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/> ++ </lrm_resource> ++ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="6:59:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;6:59:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node5" call-id="104" rc-code="7" op-status="0" interval="0" last-rc-change="1578347819" last-run="1578347819" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="12:62:0:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:0;12:62:0:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node5" call-id="114" rc-code="0" op-status="0" interval="0" last-rc-change="1578347832" last-run="1578347832" exec-time="29" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state id="3" uname="node3" in_ccm="true" crmd="online" crm-debug-origin="post_cache_update" join="member" expected="member"> ++ <transient_attributes id="3"> ++ <instance_attributes id="status-3"/> ++ </transient_attributes> ++ <lrm id="3"> ++ <lrm_resources> ++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith"> ++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="4:68:0:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:0;4:68:0:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node3" call-id="115" rc-code="0" op-status="0" interval="0" last-rc-change="1578347951" last-run="1578347951" exec-time="41" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/> ++ <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="5:68:0:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:0;5:68:0:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node3" call-id="117" rc-code="0" op-status="0" interval="120000" last-rc-change="1578347951" exec-time="32" queue-time="1" op-digest="cb34bc19df153021ce8f301baa293f35"/> ++ </lrm_resource> ++ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="4:59:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;4:59:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node3" call-id="105" rc-code="7" op-status="0" interval="0" last-rc-change="1578347819" last-run="1578347819" exec-time="30" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="5:60:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;5:60:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node3" call-id="109" rc-code="7" op-status="0" interval="0" last-rc-change="1578347822" last-run="1578347822" exec-time="48" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state id="4" uname="node4" in_ccm="true" crmd="online" crm-debug-origin="post_cache_update" join="member" expected="member"> ++ <transient_attributes id="4"> ++ <instance_attributes id="status-4"/> ++ </transient_attributes> ++ <lrm id="4"> ++ <lrm_resources> ++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith"> ++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="46:0:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;46:0:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node4" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1578347670" last-run="1578347670" exec-time="7" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/> ++ </lrm_resource> ++ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="10:61:0:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:0;10:61:0:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node4" call-id="121" rc-code="0" op-status="0" interval="0" last-rc-change="1578347828" last-run="1578347828" exec-time="25" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="6:60:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;6:60:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node4" call-id="119" rc-code="7" op-status="0" interval="0" last-rc-change="1578347822" last-run="1578347822" exec-time="29" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state id="1" uname="node1" in_ccm="false" crmd="offline" crm-debug-origin="post_cache_update" join="down" expected="down"> ++ <lrm id="1"> ++ <lrm_resources> ++ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker" shutdown-lock="1578348439"> ++ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="5:7:0:398bf005-bb15-4098-9a72-d8373456b457" transition-magic="0:0;5:7:0:398bf005-bb15-4098-9a72-d8373456b457" exit-reason="" on_node="node1" call-id="25" rc-code="0" op-status="0" interval="0" last-rc-change="1578348439" last-run="1578348439" exec-time="22" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ <lrm_rsc_op id="rsc1_monitor_10000" operation_key="rsc1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="6:2:0:398bf005-bb15-4098-9a72-d8373456b457" transition-magic="0:0;6:2:0:398bf005-bb15-4098-9a72-d8373456b457" exit-reason="" on_node="node1" call-id="21" rc-code="0" op-status="0" interval="10000" last-rc-change="1578348254" exec-time="16" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith"> ++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="2:1:7:398bf005-bb15-4098-9a72-d8373456b457" transition-magic="0:7;2:1:7:398bf005-bb15-4098-9a72-d8373456b457" exit-reason="" on_node="node1" call-id="10" rc-code="7" op-status="0" interval="0" last-rc-change="1578348254" last-run="1578348254" exec-time="4" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/> ++ </lrm_resource> ++ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="4:1:7:398bf005-bb15-4098-9a72-d8373456b457" transition-magic="0:7;4:1:7:398bf005-bb15-4098-9a72-d8373456b457" exit-reason="" on_node="node1" call-id="18" rc-code="7" op-status="0" interval="0" last-rc-change="1578348254" last-run="1578348254" exec-time="35" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ </status> ++</cib> +diff --git a/pengine/test10/shutdown-lock.dot b/pengine/test10/shutdown-lock.dot +new file mode 100644 +index 0000000..532329d +--- /dev/null ++++ b/pengine/test10/shutdown-lock.dot +@@ -0,0 +1,11 @@ ++digraph "g" { ++"Fencing_monitor_120000 node3" [ style=bold color="green" fontcolor="black"] ++"Fencing_start_0 node3" -> "Fencing_monitor_120000 node3" [ style = bold] ++"Fencing_start_0 node3" [ style=bold color="green" fontcolor="black"] ++"Fencing_stop_0 node1" -> "Fencing_start_0 node3" [ style = bold] ++"Fencing_stop_0 node1" -> "do_shutdown node1" [ style = bold] ++"Fencing_stop_0 node1" [ style=bold color="green" fontcolor="black"] ++"do_shutdown node1" [ style=bold color="green" fontcolor="black"] ++"rsc1_stop_0 node1" -> "do_shutdown node1" [ style = bold] ++"rsc1_stop_0 node1" [ style=bold color="green" fontcolor="black"] ++} +diff --git a/pengine/test10/shutdown-lock.exp b/pengine/test10/shutdown-lock.exp +new file mode 100644 +index 0000000..e8bf9d8 +--- /dev/null ++++ b/pengine/test10/shutdown-lock.exp +@@ -0,0 +1,64 @@ ++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0"> ++ <synapse id="0"> ++ <action_set> ++ <rsc_op id="5" operation="monitor" operation_key="Fencing_monitor_120000" on_node="node3" on_node_uuid="3"> ++ <primitive id="Fencing" class="stonith" type="fence_xvm"/> ++ <attributes CRM_meta_interval="120000" CRM_meta_name="monitor" CRM_meta_on_node="node3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="120000" key_file="/etc/pacemaker/fence_xvm.key" multicast_address="239.255.100.100" pcmk_host_list="node1 node2 node3 node4 node5"/> ++ </rsc_op> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="4" operation="start" operation_key="Fencing_start_0" on_node="node3" on_node_uuid="3"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="1"> ++ <action_set> ++ <rsc_op id="4" operation="start" operation_key="Fencing_start_0" on_node="node3" on_node_uuid="3"> ++ <primitive id="Fencing" class="stonith" type="fence_xvm"/> ++ <attributes CRM_meta_name="start" CRM_meta_on_node="node3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="60000" key_file="/etc/pacemaker/fence_xvm.key" multicast_address="239.255.100.100" pcmk_host_list="node1 node2 node3 node4 node5"/> ++ </rsc_op> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="3" operation="stop" operation_key="Fencing_stop_0" on_node="node1" on_node_uuid="1"/> ++ </trigger> ++ </inputs> ++ </synapse> ++ <synapse id="2"> ++ <action_set> ++ <rsc_op id="3" operation="stop" operation_key="Fencing_stop_0" on_node="node1" on_node_uuid="1"> ++ <primitive id="Fencing" class="stonith" type="fence_xvm"/> ++ <attributes CRM_meta_name="stop" CRM_meta_on_node="node1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="60000" key_file="/etc/pacemaker/fence_xvm.key" multicast_address="239.255.100.100" pcmk_host_list="node1 node2 node3 node4 node5"/> ++ </rsc_op> ++ </action_set> ++ <inputs/> ++ </synapse> ++ <synapse id="3"> ++ <action_set> ++ <rsc_op id="6" operation="stop" operation_key="rsc1_stop_0" on_node="node1" on_node_uuid="1" shutdown-lock="1578347951"> ++ <primitive id="rsc1" class="ocf" provider="pacemaker" type="Dummy"/> ++ <attributes CRM_meta_name="stop" CRM_meta_on_node="node1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" /> ++ </rsc_op> ++ </action_set> ++ <inputs/> ++ </synapse> ++ <synapse id="4"> ++ <action_set> ++ <crm_event id="7" operation="do_shutdown" operation_key="do_shutdown-node1" on_node="node1" on_node_uuid="1"> ++ <attributes CRM_meta_on_node="node1" CRM_meta_on_node_uuid="1" CRM_meta_op_no_wait="true" /> ++ <downed> ++ <node id="1"/> ++ </downed> ++ </crm_event> ++ </action_set> ++ <inputs> ++ <trigger> ++ <rsc_op id="3" operation="stop" operation_key="Fencing_stop_0" on_node="node1" on_node_uuid="1"/> ++ </trigger> ++ <trigger> ++ <rsc_op id="6" operation="stop" operation_key="rsc1_stop_0" on_node="node1" on_node_uuid="1"/> ++ </trigger> ++ </inputs> ++ </synapse> ++</transition_graph> +diff --git a/pengine/test10/shutdown-lock.scores b/pengine/test10/shutdown-lock.scores +new file mode 100644 +index 0000000..e09ebfb +--- /dev/null ++++ b/pengine/test10/shutdown-lock.scores +@@ -0,0 +1,17 @@ ++Allocation scores: ++Using the original execution date of: 2020-01-06 21:59:11Z ++native_color: Fencing allocation score on node1: 0 ++native_color: Fencing allocation score on node2: 0 ++native_color: Fencing allocation score on node3: 0 ++native_color: Fencing allocation score on node4: 0 ++native_color: Fencing allocation score on node5: 0 ++native_color: rsc1 allocation score on node1: INFINITY ++native_color: rsc1 allocation score on node2: -INFINITY ++native_color: rsc1 allocation score on node3: -INFINITY ++native_color: rsc1 allocation score on node4: -INFINITY ++native_color: rsc1 allocation score on node5: -INFINITY ++native_color: rsc2 allocation score on node1: -INFINITY ++native_color: rsc2 allocation score on node2: INFINITY ++native_color: rsc2 allocation score on node3: -INFINITY ++native_color: rsc2 allocation score on node4: -INFINITY ++native_color: rsc2 allocation score on node5: -INFINITY +diff --git a/pengine/test10/shutdown-lock.summary b/pengine/test10/shutdown-lock.summary +new file mode 100644 +index 0000000..6ed56d1 +--- /dev/null ++++ b/pengine/test10/shutdown-lock.summary +@@ -0,0 +1,31 @@ ++Using the original execution date of: 2020-01-06 21:59:11Z ++ ++Current cluster status: ++Online: [ node1 node3 node4 node5 ] ++OFFLINE: [ node2 ] ++ ++ Fencing (stonith:fence_xvm): Started node1 ++ rsc1 (ocf::pacemaker:Dummy): Started node1 ++ rsc2 (ocf::pacemaker:Dummy): Stopped node2 (LOCKED) ++ ++Transition Summary: ++ * Shutdown node1 ++ * Move Fencing ( node1 -> node3 ) ++ * Stop rsc1 ( node1 ) due to node availability ++ ++Executing cluster transition: ++ * Resource action: Fencing stop on node1 ++ * Resource action: rsc1 stop on node1 ++ * Cluster action: do_shutdown on node1 ++ * Resource action: Fencing start on node3 ++ * Resource action: Fencing monitor=120000 on node3 ++Using the original execution date of: 2020-01-06 21:59:11Z ++ ++Revised cluster status: ++Online: [ node1 node3 node4 node5 ] ++OFFLINE: [ node2 ] ++ ++ Fencing (stonith:fence_xvm): Started node3 ++ rsc1 (ocf::pacemaker:Dummy): Stopped ++ rsc2 (ocf::pacemaker:Dummy): Stopped node2 (LOCKED) ++ +diff --git a/pengine/test10/shutdown-lock.xml b/pengine/test10/shutdown-lock.xml +new file mode 100644 +index 0000000..40b807d +--- /dev/null ++++ b/pengine/test10/shutdown-lock.xml +@@ -0,0 +1,185 @@ ++<cib crm_feature_set="3.0.14" validate-with="pacemaker-2.10" epoch="154" num_updates="21" admin_epoch="0" cib-last-written="Mon Jan 6 15:58:11 2020" update-origin="node1" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1" execution-date="1578347951"> ++ <configuration> ++ <crm_config> ++ <cluster_property_set id="cib-bootstrap-options"> ++ <nvpair id="cts-stonith-enabled" name="stonith-enabled" value="1"/> ++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/> ++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/> ++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="mycluster"/> ++ <!-- This regression test ensures that resources are properly locked to a node when shutdown-lock ++ is true, both when active on a node that is shutting down (rsc1 on node1), and when ++ inactive on a node already shut down (rsc2 on node2). It also ensures that stonith-class ++ resources are not locked. ++ --> ++ <nvpair id="cib-bootstrap-options-shutdown-lock" name="shutdown-lock" value="true"/> ++ </cluster_property_set> ++ </crm_config> ++ <nodes> ++ <node id="1" uname="node1"> ++ <instance_attributes id="nodes-1"> ++ <nvpair id="nodes-1-standby" name="standby" value="off"/> ++ </instance_attributes> ++ </node> ++ <node id="2" uname="node2"/> ++ <node id="3" uname="node3"> ++ <instance_attributes id="nodes-3"> ++ <nvpair id="nodes-3-standby" name="standby" value="off"/> ++ </instance_attributes> ++ </node> ++ <node id="4" uname="node4"> ++ <instance_attributes id="nodes-4"> ++ <nvpair id="nodes-4-standby" name="standby" value="off"/> ++ </instance_attributes> ++ </node> ++ <node id="5" uname="node5"/> ++ </nodes> ++ <resources> ++ <primitive class="stonith" id="Fencing" type="fence_xvm"> ++ <meta_attributes id="Fencing-meta"> ++ <nvpair id="Fencing-migration-threshold" name="migration-threshold" value="5"/> ++ </meta_attributes> ++ <instance_attributes id="Fencing-params"> ++ <nvpair id="Fencing-key_file" name="key_file" value="/etc/pacemaker/fence_xvm.key"/> ++ <nvpair id="Fencing-multicast_address" name="multicast_address" value="239.255.100.100"/> ++ <nvpair id="Fencing-pcmk_host_list" name="pcmk_host_list" value="node1 node2 node3 node4 node5"/> ++ </instance_attributes> ++ <operations> ++ <op id="Fencing-monitor-120s" interval="120s" name="monitor" timeout="120s"/> ++ <op id="Fencing-stop-0" interval="0" name="stop" timeout="60s"/> ++ <op id="Fencing-start-0" interval="0" name="start" timeout="60s"/> ++ </operations> ++ </primitive> ++ <primitive class="ocf" id="rsc1" provider="pacemaker" type="Dummy"> ++ <operations> ++ <op id="rsc1-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/> ++ <op id="rsc1-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/> ++ <op id="rsc1-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/> ++ <op id="rsc1-reload-interval-0s" interval="0s" name="reload" timeout="20s"/> ++ <op id="rsc1-start-interval-0s" interval="0s" name="start" timeout="20s"/> ++ <op id="rsc1-stop-interval-0s" interval="0s" name="stop" timeout="20s"/> ++ </operations> ++ </primitive> ++ <primitive class="ocf" id="rsc2" provider="pacemaker" type="Dummy"> ++ <operations> ++ <op id="rsc2-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/> ++ <op id="rsc2-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/> ++ <op id="rsc2-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/> ++ <op id="rsc2-reload-interval-0s" interval="0s" name="reload" timeout="20s"/> ++ <op id="rsc2-start-interval-0s" interval="0s" name="start" timeout="20s"/> ++ <op id="rsc2-stop-interval-0s" interval="0s" name="stop" timeout="20s"/> ++ </operations> ++ </primitive> ++ </resources> ++ <constraints> ++ <rsc_location id="location-rsc1-node1-INFINITY" node="node1" rsc="rsc1" score="INFINITY"/> ++ <rsc_location id="location-rsc2-node2-INFINITY" node="node2" rsc="rsc2" score="INFINITY"/> ++ </constraints> ++ <op_defaults> ++ <meta_attributes id="cts-op_defaults-meta"> ++ <nvpair id="cts-op_defaults-timeout" name="timeout" value="90s"/> ++ </meta_attributes> ++ </op_defaults> ++ <alerts> ++ <alert id="alert-1" path="/var/lib/pacemaker/notify.sh"> ++ <recipient id="alert-1-recipient-1" value="/run/crm/alert.log"/> ++ </alert> ++ </alerts> ++ <rsc_defaults> ++ <meta_attributes id="rsc_defaults-options"/> ++ </rsc_defaults> ++ </configuration> ++ <status> ++ <node_state id="2" uname="node2" in_ccm="false" crmd="offline" crm-debug-origin="post_cache_update" join="down" expected="down"> ++ <lrm id="2"> ++ <lrm_resources> ++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith"> ++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="16:0:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;16:0:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node2" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1578347670" last-run="1578347670" exec-time="6" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/> ++ </lrm_resource> ++ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="3:59:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;3:59:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node2" call-id="141" rc-code="7" op-status="0" interval="0" last-rc-change="1578347819" last-run="1578347819" exec-time="44" queue-time="1" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker" shutdown-lock="1578347927"> ++ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="8:67:0:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:0;8:67:0:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node2" call-id="155" rc-code="0" op-status="0" interval="0" last-rc-change="1578347927" last-run="1578347927" exec-time="28" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ <lrm_rsc_op id="rsc2_monitor_10000" operation_key="rsc2_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="14:62:0:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:0;14:62:0:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node2" call-id="148" rc-code="0" op-status="0" interval="10000" last-rc-change="1578347832" exec-time="18" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state id="5" uname="node5" in_ccm="true" crmd="online" crm-debug-origin="post_cache_update" join="member" expected="member"> ++ <transient_attributes id="5"> ++ <instance_attributes id="status-5"/> ++ </transient_attributes> ++ <lrm id="5"> ++ <lrm_resources> ++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith"> ++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="61:0:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;61:0:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node5" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1578347670" last-run="1578347670" exec-time="7" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/> ++ </lrm_resource> ++ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="6:59:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;6:59:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node5" call-id="104" rc-code="7" op-status="0" interval="0" last-rc-change="1578347819" last-run="1578347819" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="12:62:0:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:0;12:62:0:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node5" call-id="114" rc-code="0" op-status="0" interval="0" last-rc-change="1578347832" last-run="1578347832" exec-time="29" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state id="3" uname="node3" in_ccm="true" crmd="online" crm-debug-origin="post_cache_update" join="member" expected="member"> ++ <transient_attributes id="3"> ++ <instance_attributes id="status-3"/> ++ </transient_attributes> ++ <lrm id="3"> ++ <lrm_resources> ++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith"> ++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="31:0:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;31:0:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node3" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1578347670" last-run="1578347670" exec-time="4" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/> ++ </lrm_resource> ++ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="4:59:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;4:59:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node3" call-id="105" rc-code="7" op-status="0" interval="0" last-rc-change="1578347819" last-run="1578347819" exec-time="30" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="5:60:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;5:60:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node3" call-id="109" rc-code="7" op-status="0" interval="0" last-rc-change="1578347822" last-run="1578347822" exec-time="48" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state id="4" uname="node4" in_ccm="true" crmd="online" crm-debug-origin="post_cache_update" join="member" expected="member"> ++ <transient_attributes id="4"> ++ <instance_attributes id="status-4"/> ++ </transient_attributes> ++ <lrm id="4"> ++ <lrm_resources> ++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith"> ++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="46:0:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;46:0:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node4" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1578347670" last-run="1578347670" exec-time="7" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/> ++ </lrm_resource> ++ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="10:61:0:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:0;10:61:0:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node4" call-id="121" rc-code="0" op-status="0" interval="0" last-rc-change="1578347828" last-run="1578347828" exec-time="25" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="6:60:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;6:60:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node4" call-id="119" rc-code="7" op-status="0" interval="0" last-rc-change="1578347822" last-run="1578347822" exec-time="29" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ <node_state id="1" uname="node1" in_ccm="true" crmd="online" crm-debug-origin="post_cache_update" join="member" expected="member"> ++ <transient_attributes id="1"> ++ <instance_attributes id="status-1"> ++ <nvpair id="status-1-shutdown" name="shutdown" value="1578347951"/> ++ </instance_attributes> ++ </transient_attributes> ++ <lrm id="1"> ++ <lrm_resources> ++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith"> ++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="76:0:0:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:0;76:0:0:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node1" call-id="10" rc-code="0" op-status="0" interval="0" last-rc-change="1578347670" last-run="1578347670" exec-time="121" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/> ++ <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="77:0:0:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:0;77:0:0:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node1" call-id="12" rc-code="0" op-status="0" interval="120000" last-rc-change="1578347670" exec-time="92" queue-time="0" op-digest="cb34bc19df153021ce8f301baa293f35"/> ++ </lrm_resource> ++ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="11:61:0:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:0;11:61:0:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node1" call-id="112" rc-code="0" op-status="0" interval="0" last-rc-change="1578347828" last-run="1578347828" exec-time="22" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ <lrm_rsc_op id="rsc1_monitor_10000" operation_key="rsc1_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="12:61:0:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:0;12:61:0:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node1" call-id="114" rc-code="0" op-status="0" interval="10000" last-rc-change="1578347828" exec-time="21" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker"> ++ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="3:60:7:4502288b-71ea-43d2-a481-d2c360266103" transition-magic="0:7;3:60:7:4502288b-71ea-43d2-a481-d2c360266103" exit-reason="" on_node="node1" call-id="111" rc-code="7" op-status="0" interval="0" last-rc-change="1578347822" last-run="1578347822" exec-time="47" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/> ++ </lrm_resource> ++ </lrm_resources> ++ </lrm> ++ </node_state> ++ </status> ++</cib> +-- +1.8.3.1 + diff --git a/SPECS/pacemaker.spec b/SPECS/pacemaker.spec index 2fb16af..697d55e 100644 --- a/SPECS/pacemaker.spec +++ b/SPECS/pacemaker.spec @@ -13,12 +13,12 @@ ## Upstream pacemaker version, and its package version (specversion ## can be incremented to build packages reliably considered "newer" ## than previously built packages with the same pcmkversion) -%global pcmkversion 1.1.20 -%global specversion 5 +%global pcmkversion 1.1.21 +%global specversion 4 ## Upstream commit (or git tag, such as "Pacemaker-" plus the ## {pcmkversion} macro for an official release) to use for this package -%global commit 3c4c782f70ebdc9c6882859d16b7975193697640 +%global commit f14e36fd4336874705b34266c7cddbe12119106c ## Since git v2.11, the extent of abbreviation is autoscaled by default ## (used to be constant of 7), so we need to convey it for non-tags, too. %global commit_abbrev 7 @@ -80,14 +80,9 @@ } || %{?__transaction_systemd_inhibit:1}%{!?__transaction_systemd_inhibit:0}%{nil \ } || %(test -f /usr/lib/os-release; test $? -ne 0; echo $?)) -%if 0%{?fedora} > 20 || 0%{?rhel} > 7 -%global gnutls_priorities @SYSTEM -%else -%if 0%{?rhel} > 6 +# RHEL: harden the default GnuTLS cipher list %global gnutls_priorities NORMAL:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1:-MD5:-3DES-CBC:-ARCFOUR-128:-ARCFOUR-40 -%endif -%endif - + ## Upstream commit to use for nagios-agents-metadata package %global nagios_hash 105ab8a @@ -168,7 +163,7 @@ Name: pacemaker Summary: Scalable High-Availability cluster resource manager Version: %{pcmkversion} -Release: %{pcmk_release}%{?dist}.2 +Release: %{pcmk_release}%{?dist} %if %{defined _unitdir} License: GPLv2+ and LGPLv2+ %else @@ -184,25 +179,28 @@ Source0: https://github.com/%{github_owner}/%{name}/archive/%{commit}/%{na Source1: nagios-agents-metadata-%{nagios_hash}.tar.gz # upstream commits -Patch1: 001-constraint-fix.patch -Patch2: 002-null-value.patch -Patch3: 003-fence-output.patch -Patch4: 004-group-ordering.patch -Patch5: 005-bug-url.patch -Patch6: 006-fence-output-fix.patch -Patch7: 007-security.patch -Patch8: 008-security-log.patch -Patch9: 009-use-after-free.patch -Patch10: 010-fork-callback.patch -Patch11: 011-remote.patch -Patch12: 012-tls-priorities.patch -Patch13: 013-guest-node.patch -Patch14: 014-guest-node-test.patch +Patch1: 01-rollup.patch +Patch2: 02-regression.patch +Patch3: 03-guest-node.patch +Patch4: 04-guest-node-test.patch +Patch5: 05-status-deletion.patch +Patch6: 06-refactors.patch # patches that aren't from upstream Patch100: lrmd-protocol-version.patch Patch101: 2.0-record-pending-behavior.patch Patch102: 2.0-cleanup-behavior.patch +Patch103: shutdown-lock-01.patch +Patch104: shutdown-lock-02.patch +Patch105: shutdown-lock-03.patch +Patch106: shutdown-lock-04.patch +Patch107: shutdown-lock-05.patch +Patch108: shutdown-lock-06.patch +Patch109: shutdown-lock-07.patch +Patch110: shutdown-lock-08.patch +Patch111: shutdown-lock-09.patch +Patch112: shutdown-lock-10.patch +Patch113: shutdown-lock-11.patch BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) AutoReqProv: on @@ -216,7 +214,7 @@ Provides: pcmk-cluster-manager %{?systemd_requires} -ExclusiveArch: aarch64 i686 ppc64le s390x x86_64 %{arm} +ExclusiveArch: aarch64 i686 ppc64le s390x x86_64 # Pacemaker targets compatibility with python 2.6+ and 3.2+ Requires: python >= 2.6 @@ -305,7 +303,7 @@ License: GPLv2+ and LGPLv2+ Summary: Core Pacemaker libraries Group: System Environment/Daemons Requires(pre): shadow-utils -# RHEL: required for libpe_status API change from 7.6 to 7.7 +# sbd 1.4.0+ supports the libpe_status API for pe_working_set_t Conflicts: sbd < 1.4.0 %description -n %{name}-libs @@ -442,13 +440,15 @@ find . -exec touch \{\} \; %build -export CPPFLAGS="-DRHEL7_COMPAT" - # Early versions of autotools (e.g. RHEL <= 5) do not support --docdir export docdir=%{pcmk_docdir} export systemdunitdir=%{?_unitdir}%{!?_unitdir:no} +# RHEL: enable notification-agent/notification-recipient, +# and change concurrent-fencing default to true +export CPPFLAGS="-DRHEL7_COMPAT -DDEFAULT_CONCURRENT_FENCING_TRUE" + %if %{with hardening} # prefer distro-provided hardening flags in case they are defined # through _hardening_{c,ld}flags macros, configure script will @@ -475,10 +475,10 @@ export LDFLAGS_HARDENED_LIB="%{?_hardening_ldflags}" %{?gnutls_priorities: --with-gnutls-priorities="%{gnutls_priorities}"} \ --with-initdir=%{_initrddir} \ --localstatedir=%{_var} \ - --with-bug-url=https://bugzilla.redhat.com/ \ - --with-nagios \ - --with-nagios-metadata-dir=%{_datadir}/pacemaker/nagios/plugins-metadata/ \ - --with-nagios-plugin-dir=%{_libdir}/nagios/plugins/ \ + --with-bug-url=https://bugzilla.redhat.com/ \ + --with-nagios \ + --with-nagios-metadata-dir=%{_datadir}/pacemaker/nagios/plugins-metadata/ \ + --with-nagios-plugin-dir=%{_libdir}/nagios/plugins/ \ --with-version=%{version}-%{release} %if 0%{?suse_version} >= 1200 @@ -885,15 +885,33 @@ exit 0 %attr(0644,root,root) %{_datadir}/pacemaker/nagios/plugins-metadata/* %changelog -* Mon Nov 11 2019 Ken Gaillot <kgaillot@redhat.com> - 1.1.20-5.2 -- Avoid invalid transition when guest node's host is unclean but can't be fenced -- Resolves: rhbz#1770734 +* Thu Jan 16 2020 Ken Gaillot <kgaillot@redhat.com> - 1.1.21-4 +- Implement shutdown-lock feature +- Resolves: rhbz#1781820 -* Tue Jul 23 2019 Ken Gaillot <kgaillot@redhat.com> - 1.1.20-5.1 -- Handle losing remote node while it is shutting down -- Allow configurable GnuTLS cipher priorities and use stricter default -- Resolves: rhbz#1732335 -- Resolves: rhbz#1733187 +* Thu Nov 7 2019 Ken Gaillot <kgaillot@redhat.com> - 1.1.21-3 +- Avoid invalid transition when guest node's host is unclean but can't be fenced +- Resolves: rhbz#1755659 + +* Tue Aug 27 2019 Ken Gaillot <kgaillot@redhat.com> - 1.1.21-2 +- Add latest upstream bug fixes to rebase roll-up patch +- Resolves: rhbz#1731189 + +* Tue Jul 30 2019 Ken Gaillot <kgaillot@redhat.com> - 1.1.21-1 +- Recover from quiesced DC disk +- Avoid timeouts and excessive stonithd CPU usage at start-up in large clusters +- Default serialized order constraints to symmetrical=false +- Avoid fence loops due to incorrect Pacemaker Remote ordering +- Default concurrent-fencing to true +- Harden GnuTLS priorities +- Rebase on upstream 1.1.21 final version +- Resolves: rhbz#1596125 +- Resolves: rhbz#1625671 +- Resolves: rhbz#1672225 +- Resolves: rhbz#1704870 +- Resolves: rhbz#1710422 +- Resolves: rhbz#1727280 +- Resolves: rhbz#1731189 * Fri May 24 2019 Ken Gaillot <kgaillot@redhat.com> - 1.1.20-5 - Correct memory issue in fence agent output fix