From 8c9ee257541e37e9974a90ee309be7e88ef5182d Mon Sep 17 00:00:00 2001

From: Ken Gaillot <kgaillot@redhat.com>

Date: Fri, 4 Sep 2020 17:02:03 -0500

Subject: [PATCH 1/3] Fix: scheduler: don't select instance to be promoted on a

 guest that can't run

The scheduler chooses clone instance(s) to be promoted from those that pass

can_be_master(). Previously, this ensured the instance was allocated to a node,

but that node might be a guest node, and its guest resource might not be

allocated to a node (i.e. can't run anywhere).

In that case, the promotion would correctly be blocked, but no other instance

would be chosen to be promoted until the next transition.

Now, ensure the guest resource is allocated as well, so another instance is

chosen.

---

 lib/pacemaker/pcmk_sched_promotable.c | 17 +++++++++++++++++

 1 file changed, 17 insertions(+)

diff --git a/lib/pacemaker/pcmk_sched_promotable.c b/lib/pacemaker/pcmk_sched_promotable.c

index 0dbeed2..b976344 100644

--- a/lib/pacemaker/pcmk_sched_promotable.c

+++ b/lib/pacemaker/pcmk_sched_promotable.c

@@ -151,6 +151,14 @@ static void apply_master_location(pe_resource_t *child, GListPtr location_constr

 }

 static pe_node_t *

+guest_location(pe_node_t *guest_node)

+{

+    pe_resource_t *guest = guest_node->details->remote_rsc->container;

+

+    return guest->fns->location(guest, NULL, FALSE);

+}

+

+static pe_node_t *

 can_be_master(pe_resource_t * rsc)

 {

     pe_node_t *node = NULL;

@@ -199,6 +207,15 @@ can_be_master(pe_resource_t * rsc)

     } else if (can_run_resources(node) == FALSE) {

         crm_trace("Node can't run any resources: %s", node->details->uname);

         return NULL;

+

+    /* @TODO It's possible this check should be done in can_run_resources()

+     * instead. We should investigate all its callers to figure out whether that

+     * would be a good idea.

+     */

+    } else if (pe__is_guest_node(node) && (guest_location(node) == NULL)) {

+        pe_rsc_trace(rsc, "%s cannot be promoted: guest %s not allocated",

+                     rsc->id, node->details->remote_rsc->container->id);

+        return NULL;

     }

     get_clone_variant_data(clone_data, parent);

-- 

1.8.3.1

From 9cd4260bda2ec77f9e336e85c31b175656f1d645 Mon Sep 17 00:00:00 2001

From: Ken Gaillot <kgaillot@redhat.com>

Date: Fri, 4 Sep 2020 17:24:48 -0500

Subject: [PATCH 2/3] Test: scheduler: update regression test for promotion

 change

The promotion change in the previous commit affected an existing test for the

better. The summary and graph output did not actually change (other than action

IDs), just the process by which the graph was arrived, which can be summed up

in this trace output difference:

5260a5261

> (can_be_master)       trace: galera:1 cannot be promoted: guest galera-bundle-docker-1 not allocated

5262d5262

< (pcmk__set_instance_roles)    info: Promoting galera:1 (Master galera-bundle-1)

5266c5266

< (pcmk__set_instance_roles)    info: galera-bundle-master: Promoted 2 instances of a possible 3 to master

---

> (pcmk__set_instance_roles)    info: galera-bundle-master: Promoted 1 instances of a possible 3 to master

Previously, 2 instances were selected to be promoted, but only 1 of those

promotes ended up being runnable. Now, only the 1 runnable instance is selected

to begin with.

This shows up in the regression test differences as Slave monitors rather than

Master monitors being scheduled for the unrunnable instance (the monitors

themselves being unrunnable as well, there are no substantive graph changes).

---

 cts/scheduler/guest-host-not-fenceable.dot | 11 ++++++---

 cts/scheduler/guest-host-not-fenceable.exp | 36 +++++++++++++++---------------

 2 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/cts/scheduler/guest-host-not-fenceable.dot b/cts/scheduler/guest-host-not-fenceable.dot

index 98833f6..a086fcb 100644

--- a/cts/scheduler/guest-host-not-fenceable.dot

+++ b/cts/scheduler/guest-host-not-fenceable.dot

@@ -13,7 +13,8 @@

 "galera-bundle-0_stop_0 node1" [ style=bold color="green" fontcolor="black"]

 "galera-bundle-1_monitor_30000 node1" [ style=dashed color="red" fontcolor="black"]

 "galera-bundle-1_start_0 node1" -> "galera-bundle-1_monitor_30000 node1" [ style = dashed]

-"galera-bundle-1_start_0 node1" -> "galera_monitor_10000 galera-bundle-1" [ style = dashed]

+"galera-bundle-1_start_0 node1" -> "galera_monitor_20000 galera-bundle-1" [ style = dashed]

+"galera-bundle-1_start_0 node1" -> "galera_monitor_30000 galera-bundle-1" [ style = dashed]

 "galera-bundle-1_start_0 node1" -> "galera_start_0 galera-bundle-1" [ style = dashed]

 "galera-bundle-1_start_0 node1" [ style=dashed color="red" fontcolor="black"]

 "galera-bundle-1_stop_0 node2" -> "galera-bundle-1_start_0 node1" [ style = dashed]

@@ -92,6 +93,8 @@

 "galera_demote_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]

 "galera_demote_0 galera-bundle-1" -> "galera-bundle-master_demoted_0" [ style = dashed]

 "galera_demote_0 galera-bundle-1" -> "galera_demote_0 galera-bundle-0" [ style = dashed]

+"galera_demote_0 galera-bundle-1" -> "galera_monitor_20000 galera-bundle-1" [ style = dashed]

+"galera_demote_0 galera-bundle-1" -> "galera_monitor_30000 galera-bundle-1" [ style = dashed]

 "galera_demote_0 galera-bundle-1" -> "galera_stop_0 galera-bundle-1" [ style = dashed]

 "galera_demote_0 galera-bundle-1" [ style=dashed color="red" fontcolor="black"]

 "galera_demote_0 galera-bundle-2" -> "galera-bundle-master_demoted_0" [ style = dashed]

@@ -101,15 +104,17 @@

 "galera_demote_0 galera-bundle-2" -> "galera_stop_0 galera-bundle-2" [ style = dashed]

 "galera_demote_0 galera-bundle-2" [ style=dashed color="red" fontcolor="black"]

 "galera_monitor_10000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]

-"galera_monitor_10000 galera-bundle-1" [ style=dashed color="red" fontcolor="black"]

+"galera_monitor_20000 galera-bundle-1" [ style=dashed color="red" fontcolor="black"]

 "galera_monitor_20000 galera-bundle-2" [ style=dashed color="red" fontcolor="black"]

+"galera_monitor_30000 galera-bundle-1" [ style=dashed color="red" fontcolor="black"]

 "galera_monitor_30000 galera-bundle-2" [ style=dashed color="red" fontcolor="black"]

 "galera_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = dashed]

 "galera_start_0 galera-bundle-0" -> "galera_monitor_10000 galera-bundle-0" [ style = dashed]

 "galera_start_0 galera-bundle-0" -> "galera_start_0 galera-bundle-1" [ style = dashed]

 "galera_start_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]

 "galera_start_0 galera-bundle-1" -> "galera-bundle-master_running_0" [ style = dashed]

-"galera_start_0 galera-bundle-1" -> "galera_monitor_10000 galera-bundle-1" [ style = dashed]

+"galera_start_0 galera-bundle-1" -> "galera_monitor_20000 galera-bundle-1" [ style = dashed]

+"galera_start_0 galera-bundle-1" -> "galera_monitor_30000 galera-bundle-1" [ style = dashed]

 "galera_start_0 galera-bundle-1" -> "galera_start_0 galera-bundle-2" [ style = dashed]

 "galera_start_0 galera-bundle-1" [ style=dashed color="red" fontcolor="black"]

 "galera_start_0 galera-bundle-2" -> "galera-bundle-master_running_0" [ style = dashed]

diff --git a/cts/scheduler/guest-host-not-fenceable.exp b/cts/scheduler/guest-host-not-fenceable.exp

index b9293d7..0631c40 100644

--- a/cts/scheduler/guest-host-not-fenceable.exp

+++ b/cts/scheduler/guest-host-not-fenceable.exp

@@ -1,7 +1,7 @@

 <transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">

   <synapse id="0">

     <action_set>

-      <rsc_op id="110" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1">

+      <rsc_op id="111" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1">

         <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>

         <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_active_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_all_hosts="node1 node2 node3 node1 node2 node3 node1 node2 node3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 node1 node2 node3 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="node1 node2 node3 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_physical_host="node1" CRM_meta_timeout="20000"  set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>

       </rsc_op>

@@ -66,7 +66,7 @@

         <pseudo_event id="48" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"/>

       </trigger>

       <trigger>

-        <rsc_op id="110" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1"/>

+        <rsc_op id="111" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="node1"/>

       </trigger>

     </inputs>

   </synapse>

@@ -162,7 +162,7 @@

     </action_set>

     <inputs>

       <trigger>

-        <pseudo_event id="84" operation="demote" operation_key="galera-bundle-master_demote_0"/>

+        <pseudo_event id="85" operation="demote" operation_key="galera-bundle-master_demote_0"/>

       </trigger>

     </inputs>

   </synapse>

@@ -181,13 +181,13 @@

         <rsc_op id="68" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"/>

       </trigger>

       <trigger>

-        <pseudo_event id="80" operation="stop" operation_key="galera-bundle-master_stop_0"/>

+        <pseudo_event id="81" operation="stop" operation_key="galera-bundle-master_stop_0"/>

       </trigger>

     </inputs>

   </synapse>

   <synapse id="13" priority="1000000">

     <action_set>

-      <pseudo_event id="85" operation="demoted" operation_key="galera-bundle-master_demoted_0">

+      <pseudo_event id="86" operation="demoted" operation_key="galera-bundle-master_demoted_0">

         <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="3" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />

       </pseudo_event>

     </action_set>

@@ -196,25 +196,25 @@

         <rsc_op id="68" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"/>

       </trigger>

       <trigger>

-        <pseudo_event id="84" operation="demote" operation_key="galera-bundle-master_demote_0"/>

+        <pseudo_event id="85" operation="demote" operation_key="galera-bundle-master_demote_0"/>

       </trigger>

     </inputs>

   </synapse>

   <synapse id="14">

     <action_set>

-      <pseudo_event id="84" operation="demote" operation_key="galera-bundle-master_demote_0">

+      <pseudo_event id="85" operation="demote" operation_key="galera-bundle-master_demote_0">

         <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="3" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />

       </pseudo_event>

     </action_set>

     <inputs>

       <trigger>

-        <pseudo_event id="88" operation="demote" operation_key="galera-bundle_demote_0"/>

+        <pseudo_event id="89" operation="demote" operation_key="galera-bundle_demote_0"/>

       </trigger>

     </inputs>

   </synapse>

   <synapse id="15" priority="1000000">

     <action_set>

-      <pseudo_event id="81" operation="stopped" operation_key="galera-bundle-master_stopped_0">

+      <pseudo_event id="82" operation="stopped" operation_key="galera-bundle-master_stopped_0">

         <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="3" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />

       </pseudo_event>

     </action_set>

@@ -223,13 +223,13 @@

         <rsc_op id="10" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="node1"/>

       </trigger>

       <trigger>

-        <pseudo_event id="80" operation="stop" operation_key="galera-bundle-master_stop_0"/>

+        <pseudo_event id="81" operation="stop" operation_key="galera-bundle-master_stop_0"/>

       </trigger>

     </inputs>

   </synapse>

   <synapse id="16">

     <action_set>

-      <pseudo_event id="80" operation="stop" operation_key="galera-bundle-master_stop_0">

+      <pseudo_event id="81" operation="stop" operation_key="galera-bundle-master_stop_0">

         <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="3" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />

       </pseudo_event>

     </action_set>

@@ -238,7 +238,7 @@

         <pseudo_event id="66" operation="stop" operation_key="galera-bundle_stop_0"/>

       </trigger>

       <trigger>

-        <pseudo_event id="85" operation="demoted" operation_key="galera-bundle-master_demoted_0"/>

+        <pseudo_event id="86" operation="demoted" operation_key="galera-bundle-master_demoted_0"/>

       </trigger>

     </inputs>

   </synapse>

@@ -292,22 +292,22 @@

   </synapse>

   <synapse id="20" priority="1000000">

     <action_set>

-      <pseudo_event id="89" operation="demoted" operation_key="galera-bundle_demoted_0">

+      <pseudo_event id="90" operation="demoted" operation_key="galera-bundle_demoted_0">

         <attributes CRM_meta_timeout="20000" />

       </pseudo_event>

     </action_set>

     <inputs>

       <trigger>

-        <pseudo_event id="85" operation="demoted" operation_key="galera-bundle-master_demoted_0"/>

+        <pseudo_event id="86" operation="demoted" operation_key="galera-bundle-master_demoted_0"/>

       </trigger>

       <trigger>

-        <pseudo_event id="88" operation="demote" operation_key="galera-bundle_demote_0"/>

+        <pseudo_event id="89" operation="demote" operation_key="galera-bundle_demote_0"/>

       </trigger>

     </inputs>

   </synapse>

   <synapse id="21">

     <action_set>

-      <pseudo_event id="88" operation="demote" operation_key="galera-bundle_demote_0">

+      <pseudo_event id="89" operation="demote" operation_key="galera-bundle_demote_0">

         <attributes CRM_meta_timeout="20000" />

       </pseudo_event>

     </action_set>

@@ -324,7 +324,7 @@

         <rsc_op id="52" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="node1" on_node_uuid="1"/>

       </trigger>

       <trigger>

-        <pseudo_event id="81" operation="stopped" operation_key="galera-bundle-master_stopped_0"/>

+        <pseudo_event id="82" operation="stopped" operation_key="galera-bundle-master_stopped_0"/>

       </trigger>

     </inputs>

   </synapse>

@@ -336,7 +336,7 @@

     </action_set>

     <inputs>

       <trigger>

-        <pseudo_event id="89" operation="demoted" operation_key="galera-bundle_demoted_0"/>

+        <pseudo_event id="90" operation="demoted" operation_key="galera-bundle_demoted_0"/>

       </trigger>

     </inputs>

   </synapse>

-- 

1.8.3.1

From 7a5a29817a694d8dc06eabad71c1e103ee089eef Mon Sep 17 00:00:00 2001

From: Ken Gaillot <kgaillot@redhat.com>

Date: Fri, 4 Sep 2020 17:33:11 -0500

Subject: [PATCH 3/3] Test: scheduler: add test for promotion change

This adds a substantive test for the promotion change.

---

 cts/cts-scheduler.in                               |   1 +

 cts/scheduler/no-promote-on-unrunnable-guest.dot   | 128 ++++

 cts/scheduler/no-promote-on-unrunnable-guest.exp   | 647 +++++++++++++++++

 .../no-promote-on-unrunnable-guest.scores          | 269 +++++++

 .../no-promote-on-unrunnable-guest.summary         | 113 +++

 cts/scheduler/no-promote-on-unrunnable-guest.xml   | 802 +++++++++++++++++++++

 6 files changed, 1960 insertions(+)

 create mode 100644 cts/scheduler/no-promote-on-unrunnable-guest.dot

 create mode 100644 cts/scheduler/no-promote-on-unrunnable-guest.exp

 create mode 100644 cts/scheduler/no-promote-on-unrunnable-guest.scores

 create mode 100644 cts/scheduler/no-promote-on-unrunnable-guest.summary

 create mode 100644 cts/scheduler/no-promote-on-unrunnable-guest.xml

diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in

index ea9c660..c7718fa 100644

--- a/cts/cts-scheduler.in

+++ b/cts/cts-scheduler.in

@@ -520,6 +520,7 @@ TESTS = [

         [ "on_fail_demote3", "Recovery with on-fail=\"demote\" with no promotion" ],

         [ "on_fail_demote4", "Recovery with on-fail=\"demote\" on failed cluster, remote, guest, and bundle nodes" ],

         [ "no_quorum_demote", "Promotable demotion and primitive stop with no-quorum-policy=\"demote\"" ],

+        [ "no-promote-on-unrunnable-guest", "Don't select bundle instance for promotion when container can't run" ],

     ],

     [

         [ "history-1", "Correctly parse stateful-1 resource state" ],

diff --git a/cts/scheduler/no-promote-on-unrunnable-guest.dot b/cts/scheduler/no-promote-on-unrunnable-guest.dot

new file mode 100644

index 0000000..6063640

--- /dev/null

+++ b/cts/scheduler/no-promote-on-unrunnable-guest.dot

@@ -0,0 +1,128 @@

+ digraph "g" {

+"Cancel ovndb_servers_monitor_30000 ovn-dbs-bundle-1" -> "ovndb_servers_promote_0 ovn-dbs-bundle-1" [ style = bold]

+"Cancel ovndb_servers_monitor_30000 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"]

+"ovn-dbs-bundle-0_monitor_30000 controller-0" [ style=dashed color="red" fontcolor="black"]

+"ovn-dbs-bundle-0_start_0 controller-0" -> "ovn-dbs-bundle-0_monitor_30000 controller-0" [ style = dashed]

+"ovn-dbs-bundle-0_start_0 controller-0" -> "ovndb_servers_monitor_30000 ovn-dbs-bundle-0" [ style = dashed]

+"ovn-dbs-bundle-0_start_0 controller-0" -> "ovndb_servers_start_0 ovn-dbs-bundle-0" [ style = dashed]

+"ovn-dbs-bundle-0_start_0 controller-0" [ style=dashed color="red" fontcolor="black"]

+"ovn-dbs-bundle-0_stop_0 controller-0" -> "ovn-dbs-bundle-0_start_0 controller-0" [ style = dashed]

+"ovn-dbs-bundle-0_stop_0 controller-0" -> "ovn-dbs-bundle-podman-0_stop_0 controller-0" [ style = bold]

+"ovn-dbs-bundle-0_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]

+"ovn-dbs-bundle-master_confirmed-post_notify_promoted_0" -> "ovn-dbs-bundle_promoted_0" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-post_notify_promoted_0" -> "ovndb_servers_monitor_10000 ovn-dbs-bundle-1" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_confirmed-post_notify_running_0" -> "ovn-dbs-bundle-master_pre_notify_promote_0" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-post_notify_running_0" -> "ovn-dbs-bundle_running_0" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" -> "ovn-dbs-bundle-master_pre_notify_promote_0" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" -> "ovn-dbs-bundle-master_pre_notify_start_0" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" -> "ovn-dbs-bundle_stopped_0" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" -> "ovndb_servers_monitor_10000 ovn-dbs-bundle-1" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_confirmed-pre_notify_promote_0" -> "ovn-dbs-bundle-master_post_notify_promoted_0" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-pre_notify_promote_0" -> "ovn-dbs-bundle-master_promote_0" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_confirmed-pre_notify_start_0" -> "ovn-dbs-bundle-master_post_notify_running_0" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-pre_notify_start_0" -> "ovn-dbs-bundle-master_start_0" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_confirmed-pre_notify_stop_0" -> "ovn-dbs-bundle-master_post_notify_stopped_0" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-pre_notify_stop_0" -> "ovn-dbs-bundle-master_stop_0" [ style = bold]

+"ovn-dbs-bundle-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_post_notify_promoted_0" -> "ovn-dbs-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]

+"ovn-dbs-bundle-master_post_notify_promoted_0" -> "ovndb_servers_post_notify_promoted_0 ovn-dbs-bundle-1" [ style = bold]

+"ovn-dbs-bundle-master_post_notify_promoted_0" -> "ovndb_servers_post_notify_promoted_0 ovn-dbs-bundle-2" [ style = bold]

+"ovn-dbs-bundle-master_post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_post_notify_running_0" -> "ovn-dbs-bundle-master_confirmed-post_notify_running_0" [ style = bold]

+"ovn-dbs-bundle-master_post_notify_running_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_post_notify_stopped_0" -> "ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" [ style = bold]

+"ovn-dbs-bundle-master_post_notify_stopped_0" -> "ovndb_servers_post_notify_stopped_0 ovn-dbs-bundle-1" [ style = bold]

+"ovn-dbs-bundle-master_post_notify_stopped_0" -> "ovndb_servers_post_notify_stopped_0 ovn-dbs-bundle-2" [ style = bold]

+"ovn-dbs-bundle-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_pre_notify_promote_0" -> "ovn-dbs-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]

+"ovn-dbs-bundle-master_pre_notify_promote_0" -> "ovndb_servers_pre_notify_promote_0 ovn-dbs-bundle-1" [ style = bold]

+"ovn-dbs-bundle-master_pre_notify_promote_0" -> "ovndb_servers_pre_notify_promote_0 ovn-dbs-bundle-2" [ style = bold]

+"ovn-dbs-bundle-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_pre_notify_start_0" -> "ovn-dbs-bundle-master_confirmed-pre_notify_start_0" [ style = bold]

+"ovn-dbs-bundle-master_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_pre_notify_stop_0" -> "ovn-dbs-bundle-master_confirmed-pre_notify_stop_0" [ style = bold]

+"ovn-dbs-bundle-master_pre_notify_stop_0" -> "ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-0" [ style = bold]

+"ovn-dbs-bundle-master_pre_notify_stop_0" -> "ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-1" [ style = bold]

+"ovn-dbs-bundle-master_pre_notify_stop_0" -> "ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-2" [ style = bold]

+"ovn-dbs-bundle-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_promote_0" -> "ovndb_servers_promote_0 ovn-dbs-bundle-1" [ style = bold]

+"ovn-dbs-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_promoted_0" -> "ovn-dbs-bundle-master_post_notify_promoted_0" [ style = bold]

+"ovn-dbs-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_running_0" -> "ovn-dbs-bundle-master_post_notify_running_0" [ style = bold]

+"ovn-dbs-bundle-master_running_0" -> "ovn-dbs-bundle-master_promote_0" [ style = bold]

+"ovn-dbs-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_start_0" -> "ovn-dbs-bundle-master_running_0" [ style = bold]

+"ovn-dbs-bundle-master_start_0" -> "ovndb_servers_start_0 ovn-dbs-bundle-0" [ style = dashed]

+"ovn-dbs-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_stop_0" -> "ovn-dbs-bundle-master_stopped_0" [ style = bold]

+"ovn-dbs-bundle-master_stop_0" -> "ovndb_servers_stop_0 ovn-dbs-bundle-0" [ style = bold]

+"ovn-dbs-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-master_stopped_0" -> "ovn-dbs-bundle-master_post_notify_stopped_0" [ style = bold]

+"ovn-dbs-bundle-master_stopped_0" -> "ovn-dbs-bundle-master_promote_0" [ style = bold]

+"ovn-dbs-bundle-master_stopped_0" -> "ovn-dbs-bundle-master_start_0" [ style = bold]

+"ovn-dbs-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle-podman-0_stop_0 controller-0" -> "ovn-dbs-bundle_stopped_0" [ style = bold]

+"ovn-dbs-bundle-podman-0_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]

+"ovn-dbs-bundle_promote_0" -> "ovn-dbs-bundle-master_promote_0" [ style = bold]

+"ovn-dbs-bundle_promote_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle_running_0" -> "ovn-dbs-bundle_promote_0" [ style = bold]

+"ovn-dbs-bundle_running_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle_stop_0" -> "ovn-dbs-bundle-master_stop_0" [ style = bold]

+"ovn-dbs-bundle_stop_0" -> "ovn-dbs-bundle-podman-0_stop_0 controller-0" [ style = bold]

+"ovn-dbs-bundle_stop_0" -> "ovndb_servers_stop_0 ovn-dbs-bundle-0" [ style = bold]

+"ovn-dbs-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]

+"ovn-dbs-bundle_stopped_0" -> "ovn-dbs-bundle_promote_0" [ style = bold]

+"ovn-dbs-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]

+"ovndb_servers_monitor_10000 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"]

+"ovndb_servers_monitor_30000 ovn-dbs-bundle-0" [ style=dashed color="red" fontcolor="black"]

+"ovndb_servers_post_notify_promoted_0 ovn-dbs-bundle-1" -> "ovn-dbs-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]

+"ovndb_servers_post_notify_promoted_0 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"]

+"ovndb_servers_post_notify_promoted_0 ovn-dbs-bundle-2" -> "ovn-dbs-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]

+"ovndb_servers_post_notify_promoted_0 ovn-dbs-bundle-2" [ style=bold color="green" fontcolor="black"]

+"ovndb_servers_post_notify_stopped_0 ovn-dbs-bundle-1" -> "ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" [ style = bold]

+"ovndb_servers_post_notify_stopped_0 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"]

+"ovndb_servers_post_notify_stopped_0 ovn-dbs-bundle-2" -> "ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" [ style = bold]

+"ovndb_servers_post_notify_stopped_0 ovn-dbs-bundle-2" [ style=bold color="green" fontcolor="black"]

+"ovndb_servers_pre_notify_promote_0 ovn-dbs-bundle-1" -> "ovn-dbs-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]

+"ovndb_servers_pre_notify_promote_0 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"]

+"ovndb_servers_pre_notify_promote_0 ovn-dbs-bundle-2" -> "ovn-dbs-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]

+"ovndb_servers_pre_notify_promote_0 ovn-dbs-bundle-2" [ style=bold color="green" fontcolor="black"]

+"ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-0" -> "ovn-dbs-bundle-master_confirmed-pre_notify_stop_0" [ style = bold]

+"ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-0" [ style=bold color="green" fontcolor="black"]

+"ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-1" -> "ovn-dbs-bundle-master_confirmed-pre_notify_stop_0" [ style = bold]

+"ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"]

+"ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-2" -> "ovn-dbs-bundle-master_confirmed-pre_notify_stop_0" [ style = bold]

+"ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-2" [ style=bold color="green" fontcolor="black"]

+"ovndb_servers_promote_0 ovn-dbs-bundle-1" -> "ovn-dbs-bundle-master_promoted_0" [ style = bold]

+"ovndb_servers_promote_0 ovn-dbs-bundle-1" -> "ovndb_servers_monitor_10000 ovn-dbs-bundle-1" [ style = bold]

+"ovndb_servers_promote_0 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"]

+"ovndb_servers_start_0 ovn-dbs-bundle-0" -> "ovn-dbs-bundle-master_running_0" [ style = dashed]

+"ovndb_servers_start_0 ovn-dbs-bundle-0" -> "ovndb_servers_monitor_30000 ovn-dbs-bundle-0" [ style = dashed]

+"ovndb_servers_start_0 ovn-dbs-bundle-0" [ style=dashed color="red" fontcolor="black"]

+"ovndb_servers_stop_0 ovn-dbs-bundle-0" -> "ovn-dbs-bundle-0_stop_0 controller-0" [ style = bold]

+"ovndb_servers_stop_0 ovn-dbs-bundle-0" -> "ovn-dbs-bundle-master_stopped_0" [ style = bold]

+"ovndb_servers_stop_0 ovn-dbs-bundle-0" -> "ovndb_servers_start_0 ovn-dbs-bundle-0" [ style = dashed]

+"ovndb_servers_stop_0 ovn-dbs-bundle-0" [ style=bold color="green" fontcolor="black"]

+"stonith-fence_ipmilan-5254005e097a_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"]

+"stonith-fence_ipmilan-5254005e097a_start_0 controller-0" -> "stonith-fence_ipmilan-5254005e097a_monitor_60000 controller-0" [ style = bold]

+"stonith-fence_ipmilan-5254005e097a_start_0 controller-0" [ style=bold color="green" fontcolor="black"]

+"stonith-fence_ipmilan-5254005e097a_stop_0 controller-0" -> "stonith-fence_ipmilan-5254005e097a_start_0 controller-0" [ style = bold]

+"stonith-fence_ipmilan-5254005e097a_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]

+"stonith-fence_ipmilan-525400985679_monitor_60000 controller-1" [ style=bold color="green" fontcolor="black"]

+"stonith-fence_ipmilan-525400985679_start_0 controller-1" -> "stonith-fence_ipmilan-525400985679_monitor_60000 controller-1" [ style = bold]

+"stonith-fence_ipmilan-525400985679_start_0 controller-1" [ style=bold color="green" fontcolor="black"]

+"stonith-fence_ipmilan-525400985679_stop_0 controller-1" -> "stonith-fence_ipmilan-525400985679_start_0 controller-1" [ style = bold]

+"stonith-fence_ipmilan-525400985679_stop_0 controller-1" [ style=bold color="green" fontcolor="black"]

+"stonith-fence_ipmilan-525400afe30e_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"]

+"stonith-fence_ipmilan-525400afe30e_start_0 controller-2" -> "stonith-fence_ipmilan-525400afe30e_monitor_60000 controller-2" [ style = bold]

+"stonith-fence_ipmilan-525400afe30e_start_0 controller-2" [ style=bold color="green" fontcolor="black"]

+"stonith-fence_ipmilan-525400afe30e_stop_0 controller-2" -> "stonith-fence_ipmilan-525400afe30e_start_0 controller-2" [ style = bold]

+"stonith-fence_ipmilan-525400afe30e_stop_0 controller-2" [ style=bold color="green" fontcolor="black"]

+}

diff --git a/cts/scheduler/no-promote-on-unrunnable-guest.exp b/cts/scheduler/no-promote-on-unrunnable-guest.exp

new file mode 100644

index 0000000..4417f6e

--- /dev/null

+++ b/cts/scheduler/no-promote-on-unrunnable-guest.exp

@@ -0,0 +1,647 @@

+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">

+  <synapse id="0">

+    <action_set>

+      <rsc_op id="261" operation="notify" operation_key="ovndb_servers_pre_notify_stop_0" internal_operation_key="ovndb_servers:0_pre_notify_stop_0" on_node="ovn-dbs-bundle-0" on_node_uuid="ovn-dbs-bundle-0" router_node="controller-0">

+        <primitive id="ovndb_servers" long-id="ovndb_servers:0" class="ocf" provider="ovn" type="ovndb-servers"/>

+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_active_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="controller-0 controller-1 controller-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource="ovndb_servers:1" CRM_meta_notify_promote_uname="ovn-dbs-bundle-1" CRM_meta_notify_slave_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_slave_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="ovndb_servers:0" CRM_meta_notify_stop_uname="ovn-dbs-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="ovn-dbs-bundle-0" CRM_meta_on_node_uuid="ovn-dbs-bundle-0" CRM_meta_physical_host="controller-0" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000"  inactive_probe_interval="180000" listen_on_master_ip_only="no" manage_northd="yes" master_ip="172.17.1.247" nb_master_port="6641" sb_master_port="6642"/>

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="203" operation="notify" operation_key="ovn-dbs-bundle-master_pre_notify_stop_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="1">

+    <action_set>

+      <rsc_op id="187" operation="stop" operation_key="ovndb_servers_stop_0" internal_operation_key="ovndb_servers:0_stop_0" on_node="ovn-dbs-bundle-0" on_node_uuid="ovn-dbs-bundle-0" router_node="controller-0">

+        <primitive id="ovndb_servers" long-id="ovndb_servers:0" class="ocf" provider="ovn" type="ovndb-servers"/>

+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="true" CRM_meta_notify_active_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_active_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="controller-0 controller-1 controller-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource="ovndb_servers:1" CRM_meta_notify_promote_uname="ovn-dbs-bundle-1" CRM_meta_notify_slave_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_slave_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="ovndb_servers:0" CRM_meta_notify_stop_uname="ovn-dbs-bundle-0" CRM_meta_on_node="ovn-dbs-bundle-0" CRM_meta_on_node_uuid="ovn-dbs-bundle-0" CRM_meta_physical_host="controller-0" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="200000"  inactive_probe_interval="180000" listen_on_master_ip_only="no" manage_northd="yes" master_ip="172.17.1.247" nb_master_port="6641" sb_master_port="6642"/>

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="184" operation="stop" operation_key="ovn-dbs-bundle_stop_0"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="201" operation="stop" operation_key="ovn-dbs-bundle-master_stop_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="2" priority="1000000">

+    <action_set>

+      <rsc_op id="267" operation="notify" operation_key="ovndb_servers_post_notify_promote_0" internal_operation_key="ovndb_servers:1_post_notify_promote_0" on_node="ovn-dbs-bundle-1" on_node_uuid="ovn-dbs-bundle-1" router_node="controller-1">

+        <primitive id="ovndb_servers" long-id="ovndb_servers:1" class="ocf" provider="ovn" type="ovndb-servers"/>

+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_active_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="controller-0 controller-1 controller-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="ovndb_servers:1" CRM_meta_notify_promote_uname="ovn-dbs-bundle-1" CRM_meta_notify_slave_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_slave_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="ovndb_servers:0" CRM_meta_notify_stop_uname="ovn-dbs-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="ovn-dbs-bundle-1" CRM_meta_on_node_uuid="ovn-dbs-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000"  inactive_probe_interval="180000" listen_on_master_ip_only="no" manage_northd="yes" master_ip="172.17.1.247" nb_master_port="6641" sb_master_port="6642"/>

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="211" operation="notify" operation_key="ovn-dbs-bundle-master_post_notify_promoted_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="3">

+    <action_set>

+      <rsc_op id="266" operation="notify" operation_key="ovndb_servers_pre_notify_promote_0" internal_operation_key="ovndb_servers:1_pre_notify_promote_0" on_node="ovn-dbs-bundle-1" on_node_uuid="ovn-dbs-bundle-1" router_node="controller-1">

+        <primitive id="ovndb_servers" long-id="ovndb_servers:1" class="ocf" provider="ovn" type="ovndb-servers"/>

+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_active_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="controller-0 controller-1 controller-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="ovndb_servers:1" CRM_meta_notify_promote_uname="ovn-dbs-bundle-1" CRM_meta_notify_slave_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_slave_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="ovndb_servers:0" CRM_meta_notify_stop_uname="ovn-dbs-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="ovn-dbs-bundle-1" CRM_meta_on_node_uuid="ovn-dbs-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000"  inactive_probe_interval="180000" listen_on_master_ip_only="no" manage_northd="yes" master_ip="172.17.1.247" nb_master_port="6641" sb_master_port="6642"/>

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="209" operation="notify" operation_key="ovn-dbs-bundle-master_pre_notify_promote_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="4" priority="1000000">

+    <action_set>

+      <rsc_op id="263" operation="notify" operation_key="ovndb_servers_post_notify_stop_0" internal_operation_key="ovndb_servers:1_post_notify_stop_0" on_node="ovn-dbs-bundle-1" on_node_uuid="ovn-dbs-bundle-1" router_node="controller-1">

+        <primitive id="ovndb_servers" long-id="ovndb_servers:1" class="ocf" provider="ovn" type="ovndb-servers"/>

+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_active_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="controller-0 controller-1 controller-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource="ovndb_servers:1" CRM_meta_notify_promote_uname="ovn-dbs-bundle-1" CRM_meta_notify_slave_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_slave_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="ovndb_servers:0" CRM_meta_notify_stop_uname="ovn-dbs-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="ovn-dbs-bundle-1" CRM_meta_on_node_uuid="ovn-dbs-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000"  inactive_probe_interval="180000" listen_on_master_ip_only="no" manage_northd="yes" master_ip="172.17.1.247" nb_master_port="6641" sb_master_port="6642"/>

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="205" operation="notify" operation_key="ovn-dbs-bundle-master_post_notify_stopped_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="5">

+    <action_set>

+      <rsc_op id="262" operation="notify" operation_key="ovndb_servers_pre_notify_stop_0" internal_operation_key="ovndb_servers:1_pre_notify_stop_0" on_node="ovn-dbs-bundle-1" on_node_uuid="ovn-dbs-bundle-1" router_node="controller-1">

+        <primitive id="ovndb_servers" long-id="ovndb_servers:1" class="ocf" provider="ovn" type="ovndb-servers"/>

+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_active_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="controller-0 controller-1 controller-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource="ovndb_servers:1" CRM_meta_notify_promote_uname="ovn-dbs-bundle-1" CRM_meta_notify_slave_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_slave_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="ovndb_servers:0" CRM_meta_notify_stop_uname="ovn-dbs-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="ovn-dbs-bundle-1" CRM_meta_on_node_uuid="ovn-dbs-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000"  inactive_probe_interval="180000" listen_on_master_ip_only="no" manage_northd="yes" master_ip="172.17.1.247" nb_master_port="6641" sb_master_port="6642"/>

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="203" operation="notify" operation_key="ovn-dbs-bundle-master_pre_notify_stop_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="6">

+    <action_set>

+      <rsc_op id="192" operation="monitor" operation_key="ovndb_servers_monitor_10000" internal_operation_key="ovndb_servers:1_monitor_10000" on_node="ovn-dbs-bundle-1" on_node_uuid="ovn-dbs-bundle-1" router_node="controller-1">

+        <primitive id="ovndb_servers" long-id="ovndb_servers:1" class="ocf" provider="ovn" type="ovndb-servers"/>

+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="ovn-dbs-bundle-1" CRM_meta_on_node_uuid="ovn-dbs-bundle-1" CRM_meta_op_target_rc="8" CRM_meta_physical_host="controller-1" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Master" CRM_meta_timeout="60000"  inactive_probe_interval="180000" listen_on_master_ip_only="no" manage_northd="yes" master_ip="172.17.1.247" nb_master_port="6641" sb_master_port="6642"/>

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <rsc_op id="191" operation="promote" operation_key="ovndb_servers_promote_0" internal_operation_key="ovndb_servers:1_promote_0" on_node="ovn-dbs-bundle-1" on_node_uuid="ovn-dbs-bundle-1" router_node="controller-1"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="206" operation="notified" operation_key="ovn-dbs-bundle-master_confirmed-post_notify_stopped_0"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="212" operation="notified" operation_key="ovn-dbs-bundle-master_confirmed-post_notify_promoted_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="7">

+    <action_set>

+      <rsc_op id="191" operation="promote" operation_key="ovndb_servers_promote_0" internal_operation_key="ovndb_servers:1_promote_0" on_node="ovn-dbs-bundle-1" on_node_uuid="ovn-dbs-bundle-1" router_node="controller-1">

+        <primitive id="ovndb_servers" long-id="ovndb_servers:1" class="ocf" provider="ovn" type="ovndb-servers"/>

+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="true" CRM_meta_notify_active_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_active_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="controller-0 controller-1 controller-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource="ovndb_servers:1" CRM_meta_notify_promote_uname="ovn-dbs-bundle-1" CRM_meta_notify_slave_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_slave_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="ovndb_servers:0" CRM_meta_notify_stop_uname="ovn-dbs-bundle-0" CRM_meta_on_node="ovn-dbs-bundle-1" CRM_meta_on_node_uuid="ovn-dbs-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="50000"  inactive_probe_interval="180000" listen_on_master_ip_only="no" manage_northd="yes" master_ip="172.17.1.247" nb_master_port="6641" sb_master_port="6642"/>

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <rsc_op id="40" operation="cancel" operation_key="ovndb_servers_monitor_30000" internal_operation_key="ovndb_servers:1_monitor_30000" on_node="ovn-dbs-bundle-1" on_node_uuid="ovn-dbs-bundle-1" router_node="controller-1"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="207" operation="promote" operation_key="ovn-dbs-bundle-master_promote_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="8">

+    <action_set>

+      <rsc_op id="40" operation="cancel" operation_key="ovndb_servers_monitor_30000" internal_operation_key="ovndb_servers:1_monitor_30000" on_node="ovn-dbs-bundle-1" on_node_uuid="ovn-dbs-bundle-1" router_node="controller-1">

+        <primitive id="ovndb_servers" long-id="ovndb_servers:1" class="ocf" provider="ovn" type="ovndb-servers"/>

+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="ovn-dbs-bundle-1" CRM_meta_on_node_uuid="ovn-dbs-bundle-1" CRM_meta_operation="monitor" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Slave" CRM_meta_timeout="60000"  inactive_probe_interval="180000" listen_on_master_ip_only="no" manage_northd="yes" master_ip="172.17.1.247" nb_master_port="6641" sb_master_port="6642"/>

+      </rsc_op>

+    </action_set>

+    <inputs/>

+  </synapse>

+  <synapse id="9" priority="1000000">

+    <action_set>

+      <rsc_op id="269" operation="notify" operation_key="ovndb_servers_post_notify_promote_0" internal_operation_key="ovndb_servers:2_post_notify_promote_0" on_node="ovn-dbs-bundle-2" on_node_uuid="ovn-dbs-bundle-2" router_node="controller-2">

+        <primitive id="ovndb_servers" long-id="ovndb_servers:2" class="ocf" provider="ovn" type="ovndb-servers"/>

+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_active_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="controller-0 controller-1 controller-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="ovndb_servers:1" CRM_meta_notify_promote_uname="ovn-dbs-bundle-1" CRM_meta_notify_slave_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_slave_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="ovndb_servers:0" CRM_meta_notify_stop_uname="ovn-dbs-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="ovn-dbs-bundle-2" CRM_meta_on_node_uuid="ovn-dbs-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000"  inactive_probe_interval="180000" listen_on_master_ip_only="no" manage_northd="yes" master_ip="172.17.1.247" nb_master_port="6641" sb_master_port="6642"/>

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="211" operation="notify" operation_key="ovn-dbs-bundle-master_post_notify_promoted_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="10">

+    <action_set>

+      <rsc_op id="268" operation="notify" operation_key="ovndb_servers_pre_notify_promote_0" internal_operation_key="ovndb_servers:2_pre_notify_promote_0" on_node="ovn-dbs-bundle-2" on_node_uuid="ovn-dbs-bundle-2" router_node="controller-2">

+        <primitive id="ovndb_servers" long-id="ovndb_servers:2" class="ocf" provider="ovn" type="ovndb-servers"/>

+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_active_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="controller-0 controller-1 controller-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="ovndb_servers:1" CRM_meta_notify_promote_uname="ovn-dbs-bundle-1" CRM_meta_notify_slave_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_slave_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="ovndb_servers:0" CRM_meta_notify_stop_uname="ovn-dbs-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="ovn-dbs-bundle-2" CRM_meta_on_node_uuid="ovn-dbs-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000"  inactive_probe_interval="180000" listen_on_master_ip_only="no" manage_northd="yes" master_ip="172.17.1.247" nb_master_port="6641" sb_master_port="6642"/>

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="209" operation="notify" operation_key="ovn-dbs-bundle-master_pre_notify_promote_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="11" priority="1000000">

+    <action_set>

+      <rsc_op id="265" operation="notify" operation_key="ovndb_servers_post_notify_stop_0" internal_operation_key="ovndb_servers:2_post_notify_stop_0" on_node="ovn-dbs-bundle-2" on_node_uuid="ovn-dbs-bundle-2" router_node="controller-2">

+        <primitive id="ovndb_servers" long-id="ovndb_servers:2" class="ocf" provider="ovn" type="ovndb-servers"/>

+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_active_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="controller-0 controller-1 controller-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource="ovndb_servers:1" CRM_meta_notify_promote_uname="ovn-dbs-bundle-1" CRM_meta_notify_slave_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_slave_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="ovndb_servers:0" CRM_meta_notify_stop_uname="ovn-dbs-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="ovn-dbs-bundle-2" CRM_meta_on_node_uuid="ovn-dbs-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000"  inactive_probe_interval="180000" listen_on_master_ip_only="no" manage_northd="yes" master_ip="172.17.1.247" nb_master_port="6641" sb_master_port="6642"/>

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="205" operation="notify" operation_key="ovn-dbs-bundle-master_post_notify_stopped_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="12">

+    <action_set>

+      <rsc_op id="264" operation="notify" operation_key="ovndb_servers_pre_notify_stop_0" internal_operation_key="ovndb_servers:2_pre_notify_stop_0" on_node="ovn-dbs-bundle-2" on_node_uuid="ovn-dbs-bundle-2" router_node="controller-2">

+        <primitive id="ovndb_servers" long-id="ovndb_servers:2" class="ocf" provider="ovn" type="ovndb-servers"/>

+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_active_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="controller-0 controller-1 controller-2 ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource="ovndb_servers:1" CRM_meta_notify_promote_uname="ovn-dbs-bundle-1" CRM_meta_notify_slave_resource="ovndb_servers:0 ovndb_servers:1 ovndb_servers:2" CRM_meta_notify_slave_uname="ovn-dbs-bundle-0 ovn-dbs-bundle-1 ovn-dbs-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="ovndb_servers:0" CRM_meta_notify_stop_uname="ovn-dbs-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="ovn-dbs-bundle-2" CRM_meta_on_node_uuid="ovn-dbs-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000"  inactive_probe_interval="180000" listen_on_master_ip_only="no" manage_northd="yes" master_ip="172.17.1.247" nb_master_port="6641" sb_master_port="6642"/>

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="203" operation="notify" operation_key="ovn-dbs-bundle-master_pre_notify_stop_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="13" priority="1000000">

+    <action_set>

+      <pseudo_event id="212" operation="notified" operation_key="ovn-dbs-bundle-master_confirmed-post_notify_promoted_0">

+        <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="promote" CRM_meta_notify_type="post" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="120000" />

+      </pseudo_event>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="211" operation="notify" operation_key="ovn-dbs-bundle-master_post_notify_promoted_0"/>

+      </trigger>

+      <trigger>

+        <rsc_op id="267" operation="notify" operation_key="ovndb_servers_post_notify_promote_0" internal_operation_key="ovndb_servers:1_post_notify_promote_0" on_node="ovn-dbs-bundle-1" on_node_uuid="ovn-dbs-bundle-1" router_node="controller-1"/>

+      </trigger>

+      <trigger>

+        <rsc_op id="269" operation="notify" operation_key="ovndb_servers_post_notify_promote_0" internal_operation_key="ovndb_servers:2_post_notify_promote_0" on_node="ovn-dbs-bundle-2" on_node_uuid="ovn-dbs-bundle-2" router_node="controller-2"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="14" priority="1000000">

+    <action_set>

+      <pseudo_event id="211" operation="notify" operation_key="ovn-dbs-bundle-master_post_notify_promoted_0">

+        <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="promote" CRM_meta_notify_type="post" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="120000" />

+      </pseudo_event>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="208" operation="promoted" operation_key="ovn-dbs-bundle-master_promoted_0"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="210" operation="notified" operation_key="ovn-dbs-bundle-master_confirmed-pre_notify_promote_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="15">

+    <action_set>

+      <pseudo_event id="210" operation="notified" operation_key="ovn-dbs-bundle-master_confirmed-pre_notify_promote_0">

+        <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="promote" CRM_meta_notify_type="pre" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="120000" />

+      </pseudo_event>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="209" operation="notify" operation_key="ovn-dbs-bundle-master_pre_notify_promote_0"/>

+      </trigger>

+      <trigger>

+        <rsc_op id="266" operation="notify" operation_key="ovndb_servers_pre_notify_promote_0" internal_operation_key="ovndb_servers:1_pre_notify_promote_0" on_node="ovn-dbs-bundle-1" on_node_uuid="ovn-dbs-bundle-1" router_node="controller-1"/>

+      </trigger>

+      <trigger>

+        <rsc_op id="268" operation="notify" operation_key="ovndb_servers_pre_notify_promote_0" internal_operation_key="ovndb_servers:2_pre_notify_promote_0" on_node="ovn-dbs-bundle-2" on_node_uuid="ovn-dbs-bundle-2" router_node="controller-2"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="16">

+    <action_set>

+      <pseudo_event id="209" operation="notify" operation_key="ovn-dbs-bundle-master_pre_notify_promote_0">

+        <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="promote" CRM_meta_notify_type="pre" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="120000" />

+      </pseudo_event>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="200" operation="notified" operation_key="ovn-dbs-bundle-master_confirmed-post_notify_running_0"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="206" operation="notified" operation_key="ovn-dbs-bundle-master_confirmed-post_notify_stopped_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="17" priority="1000000">

+    <action_set>

+      <pseudo_event id="208" operation="promoted" operation_key="ovn-dbs-bundle-master_promoted_0">

+        <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="120000" />

+      </pseudo_event>

+    </action_set>

+    <inputs>

+      <trigger>

+        <rsc_op id="191" operation="promote" operation_key="ovndb_servers_promote_0" internal_operation_key="ovndb_servers:1_promote_0" on_node="ovn-dbs-bundle-1" on_node_uuid="ovn-dbs-bundle-1" router_node="controller-1"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="18">

+    <action_set>

+      <pseudo_event id="207" operation="promote" operation_key="ovn-dbs-bundle-master_promote_0">

+        <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="120000" />

+      </pseudo_event>

+    </action_set>