From 4521f547457af1201442e072d426fdf89de1150e Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Mon, 9 Nov 2020 18:09:54 -0600
Subject: [PATCH 01/13] API: libpe_status: add pe_rsc_params() and new
 pe_resource_t member

Instead of a single parameters table for a resource, this allows the
possibility of a parameter table per node for the resource, since
rule-based parameters may evaluate differently on different nodes.
---
 include/crm/pengine/complex.h  |  5 +++-
 include/crm/pengine/pe_types.h |  8 +++++-
 lib/pengine/complex.c          | 61 +++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 71 insertions(+), 3 deletions(-)

diff --git a/include/crm/pengine/complex.h b/include/crm/pengine/complex.h
index effa44f..1d010f4 100644
--- a/include/crm/pengine/complex.h
+++ b/include/crm/pengine/complex.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2004-2019 the Pacemaker project contributors
+ * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
@@ -19,6 +19,9 @@ extern "C" {
 #include <crm/pengine/pe_types.h>   // pe_node_t, pe_resource_t, etc.
 
 extern resource_object_functions_t resource_class_functions[];
+
+GHashTable *pe_rsc_params(pe_resource_t *rsc, pe_node_t *node,
+                          pe_working_set_t *data_set);
 void get_meta_attributes(GHashTable * meta_hash, pe_resource_t *rsc,
                          pe_node_t *node, pe_working_set_t *data_set);
 void get_rsc_attributes(GHashTable *meta_hash, pe_resource_t *rsc,
diff --git a/include/crm/pengine/pe_types.h b/include/crm/pengine/pe_types.h
index 59d5ce8..5529714 100644
--- a/include/crm/pengine/pe_types.h
+++ b/include/crm/pengine/pe_types.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2004-2020 the Pacemaker project contributors
+ * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
@@ -371,6 +371,12 @@ struct pe_resource_s {
     pe_node_t *lock_node;       // Resource is shutdown-locked to this node
     time_t lock_time;           // When shutdown lock started
 
+    /* Resource parameters may have node-attribute-based rules, which means the
+     * values can vary by node. This table is a cache of parameter name/value
+     * tables for each node (as needed). Use pe_rsc_params() to get the table
+     * for a given node.
+     */
+    GHashTable *parameter_cache; // Key = node name, value = parameters table
 #if ENABLE_VERSIONED_ATTRS
     xmlNode *versioned_parameters;
 #endif
diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
index 5f484ad..7037ca1 100644
--- a/lib/pengine/complex.c
+++ b/lib/pengine/complex.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2004-2020 the Pacemaker project contributors
+ * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
@@ -435,6 +435,62 @@ detect_promotable(pe_resource_t *rsc)
     return FALSE;
 }
 
+static void
+free_params_table(gpointer data)
+{
+    g_hash_table_destroy((GHashTable *) data);
+}
+
+/*!
+ * \brief Get a table of resource parameters
+ *
+ * \param[in] rsc       Resource to query
+ * \param[in] node      Node for evaluating rules (NULL for defaults)
+ * \param[in] data_set  Cluster working set
+ *
+ * \return Hash table containing resource parameter names and values
+ *         (or NULL if \p rsc or \p data_set is NULL)
+ * \note The returned table will be destroyed when the resource is freed, so
+ *       callers should not destroy it.
+ */
+GHashTable *
+pe_rsc_params(pe_resource_t *rsc, pe_node_t *node, pe_working_set_t *data_set)
+{
+    GHashTable *params_on_node = NULL;
+
+    /* A NULL node is used to request the resource's default parameters
+     * (not evaluated for node), but we always want something non-NULL
+     * as a hash table key.
+     */
+    const char *node_name = "";
+
+    // Sanity check
+    if ((rsc == NULL) || (data_set == NULL)) {
+        return NULL;
+    }
+    if ((node != NULL) && (node->details->uname != NULL)) {
+        node_name = node->details->uname;
+    }
+
+    // Find the parameter table for given node
+    if (rsc->parameter_cache == NULL) {
+        rsc->parameter_cache = g_hash_table_new_full(crm_strcase_hash,
+                                                     crm_strcase_equal, free,
+                                                     free_params_table);
+    } else {
+        params_on_node = g_hash_table_lookup(rsc->parameter_cache, node_name);
+    }
+
+    // If none exists yet, create one with parameters evaluated for node
+    if (params_on_node == NULL) {
+        params_on_node = crm_str_table_new();
+        get_rsc_attributes(params_on_node, rsc, node, data_set);
+        g_hash_table_insert(rsc->parameter_cache, strdup(node_name),
+                            params_on_node);
+    }
+    return params_on_node;
+}
+
 gboolean
 common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc,
               pe_resource_t * parent, pe_working_set_t * data_set)
@@ -869,6 +925,9 @@ common_free(pe_resource_t * rsc)
     if (rsc->parameters != NULL) {
         g_hash_table_destroy(rsc->parameters);
     }
+    if (rsc->parameter_cache != NULL) {
+        g_hash_table_destroy(rsc->parameter_cache);
+    }
 #if ENABLE_VERSIONED_ATTRS
     if (rsc->versioned_parameters != NULL) {
         free_xml(rsc->versioned_parameters);
-- 
1.8.3.1


From d5075f64c5fff1f037ee8dbca2ad6268bce15681 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Wed, 11 Nov 2020 13:03:50 -0600
Subject: [PATCH 02/13] API: libpe_status: ignore ->parameter() resource object
 function's create argument

This uses the new resource parameter function to implement ->parameter().
That means the parameter table will always be created if not already existent.

->parameter() is not called internally.
---
 lib/pengine/native.c | 28 ++++------------------------
 1 file changed, 4 insertions(+), 24 deletions(-)

diff --git a/lib/pengine/native.c b/lib/pengine/native.c
index 193be17..95c3da9 100644
--- a/lib/pengine/native.c
+++ b/lib/pengine/native.c
@@ -312,48 +312,28 @@ native_find_rsc(pe_resource_t * rsc, const char *id, const pe_node_t *on_node,
     return NULL;
 }
 
+// create is ignored
 char *
 native_parameter(pe_resource_t * rsc, pe_node_t * node, gboolean create, const char *name,
                  pe_working_set_t * data_set)
 {
     char *value_copy = NULL;
     const char *value = NULL;
-    GHashTable *hash = NULL;
-    GHashTable *local_hash = NULL;
+    GHashTable *params = NULL;
 
     CRM_CHECK(rsc != NULL, return NULL);
     CRM_CHECK(name != NULL && strlen(name) != 0, return NULL);
 
     pe_rsc_trace(rsc, "Looking up %s in %s", name, rsc->id);
-
-    if (create || g_hash_table_size(rsc->parameters) == 0) {
-        if (node != NULL) {
-            pe_rsc_trace(rsc, "Creating hash with node %s", node->details->uname);
-        } else {
-            pe_rsc_trace(rsc, "Creating default hash");
-        }
-
-        local_hash = crm_str_table_new();
-
-        get_rsc_attributes(local_hash, rsc, node, data_set);
-
-        hash = local_hash;
-    } else {
-        hash = rsc->parameters;
-    }
-
-    value = g_hash_table_lookup(hash, name);
+    params = pe_rsc_params(rsc, node, data_set);
+    value = g_hash_table_lookup(params, name);
     if (value == NULL) {
         /* try meta attributes instead */
         value = g_hash_table_lookup(rsc->meta, name);
     }
-
     if (value != NULL) {
         value_copy = strdup(value);
     }
-    if (local_hash != NULL) {
-        g_hash_table_destroy(local_hash);
-    }
     return value_copy;
 }
 
-- 
1.8.3.1


From 7089c19d1a1a79dd353ade0002ba6ed3321145ca Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Mon, 9 Nov 2020 18:15:56 -0600
Subject: [PATCH 03/13] Refactor: fencer: use new resource parameters function

---
 daemons/fenced/pacemaker-fenced.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c
index 69c29a7..5390d66 100644
--- a/daemons/fenced/pacemaker-fenced.c
+++ b/daemons/fenced/pacemaker-fenced.c
@@ -643,6 +643,7 @@ static void cib_device_update(pe_resource_t *rsc, pe_working_set_t *data_set)
         /* Our node is allowed, so update the device information */
         int rc;
         xmlNode *data;
+        GHashTable *rsc_params = NULL;
         GHashTableIter gIter;
         stonith_key_value_t *params = NULL;
 
@@ -651,12 +652,12 @@ static void cib_device_update(pe_resource_t *rsc, pe_working_set_t *data_set)
         const char *rsc_provides = NULL;
 
         crm_debug("Device %s is allowed on %s: score=%d", rsc->id, stonith_our_uname, node->weight);
-        get_rsc_attributes(rsc->parameters, rsc, node, data_set);
+        rsc_params = pe_rsc_params(rsc, node, data_set);
         get_meta_attributes(rsc->meta, rsc, node, data_set);
 
         rsc_provides = g_hash_table_lookup(rsc->meta, PCMK_STONITH_PROVIDES);
 
-        g_hash_table_iter_init(&gIter, rsc->parameters);
+        g_hash_table_iter_init(&gIter, rsc_params);
         while (g_hash_table_iter_next(&gIter, (gpointer *) & name, (gpointer *) & value)) {
             if (!name || !value) {
                 continue;
-- 
1.8.3.1


From ae0a2f26891b30a7bcf09467dac461a17d071cd9 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Mon, 9 Nov 2020 18:35:08 -0600
Subject: [PATCH 04/13] Refactor: scheduler: use new resource parameter
 function when getting fence action timeout

This means that the fence device's active node is used to evaluate the
parameters, which will be of use if we ever support rules in fence device
configuration.
---
 lib/pengine/utils.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
index b07afbe..831f890 100644
--- a/lib/pengine/utils.c
+++ b/lib/pengine/utils.c
@@ -1099,12 +1099,12 @@ unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * contai
      */
     if (pcmk_is_set(pcmk_get_ra_caps(rsc_rule_data.standard),
                     pcmk_ra_cap_fence_params)
-            && (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)
-                || is_probe)
-            && action->rsc->parameters) {
+        && (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)
+            || is_probe)) {
 
-        value = g_hash_table_lookup(action->rsc->parameters,
-                                    "pcmk_monitor_timeout");
+        GHashTable *params = pe_rsc_params(action->rsc, action->node, data_set);
+
+        value = g_hash_table_lookup(params, "pcmk_monitor_timeout");
 
         if (value) {
             crm_trace("\t%s: Setting timeout to pcmk_monitor_timeout '%s', "
-- 
1.8.3.1


From 25cf7f5e0f6e3a22000fcaa4f4492fe1c086252b Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Fri, 13 Nov 2020 09:27:43 -0600
Subject: [PATCH 05/13] Refactor: scheduler: use new resource parameter
 function when replacing bundle #uname

---
 include/crm/pengine/internal.h      |  8 +++++---
 lib/pacemaker/pcmk_sched_allocate.c |  2 +-
 lib/pacemaker/pcmk_sched_bundle.c   | 14 ++++++++++++--
 lib/pengine/bundle.c                | 12 ++++++++----
 lib/pengine/pe_digest.c             |  2 +-
 lib/pengine/unpack.c                |  2 +-
 6 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
index 1e5aee1..9f4e28a 100644
--- a/include/crm/pengine/internal.h
+++ b/include/crm/pengine/internal.h
@@ -545,9 +545,11 @@ int pe__common_output_text(pcmk__output_t *out, pe_resource_t * rsc, const char
 int pe__common_output_html(pcmk__output_t *out, pe_resource_t * rsc, const char *name, pe_node_t *node, long options);
 pe_resource_t *pe__find_bundle_replica(const pe_resource_t *bundle,
                                        const pe_node_t *node);
-bool pe__bundle_needs_remote_name(pe_resource_t *rsc);
-const char *pe__add_bundle_remote_name(pe_resource_t *rsc, xmlNode *xml,
-                                       const char *field);
+bool pe__bundle_needs_remote_name(pe_resource_t *rsc,
+                                  pe_working_set_t *data_set);
+const char *pe__add_bundle_remote_name(pe_resource_t *rsc,
+                                       pe_working_set_t *data_set,
+                                       xmlNode *xml, const char *field);
 const char *pe_node_attribute_calculated(const pe_node_t *node,
                                          const char *name,
                                          const pe_resource_t *rsc);
diff --git a/lib/pacemaker/pcmk_sched_allocate.c b/lib/pacemaker/pcmk_sched_allocate.c
index a0fb5ab..22f284a 100644
--- a/lib/pacemaker/pcmk_sched_allocate.c
+++ b/lib/pacemaker/pcmk_sched_allocate.c
@@ -470,7 +470,7 @@ check_actions_for(xmlNode * rsc_entry, pe_resource_t * rsc, pe_node_t * node, pe
              * has changed, clear any fail count so they can be retried fresh.
              */
 
-            if (pe__bundle_needs_remote_name(rsc)) {
+            if (pe__bundle_needs_remote_name(rsc, data_set)) {
                 /* We haven't allocated resources to nodes yet, so if the
                  * REMOTE_CONTAINER_HACK is used, we may calculate the digest
                  * based on the literal "#uname" value rather than the properly
diff --git a/lib/pacemaker/pcmk_sched_bundle.c b/lib/pacemaker/pcmk_sched_bundle.c
index ac9219c..4f41b70 100644
--- a/lib/pacemaker/pcmk_sched_bundle.c
+++ b/lib/pacemaker/pcmk_sched_bundle.c
@@ -911,7 +911,7 @@ pcmk__bundle_expand(pe_resource_t *rsc, pe_working_set_t * data_set)
 
         CRM_ASSERT(replica);
         if (replica->remote && replica->container
-            && pe__bundle_needs_remote_name(replica->remote)) {
+            && pe__bundle_needs_remote_name(replica->remote, data_set)) {
 
             /* REMOTE_CONTAINER_HACK: Allow remote nodes to run containers that
              * run pacemaker-remoted inside, without needing a separate IP for
@@ -923,12 +923,22 @@ pcmk__bundle_expand(pe_resource_t *rsc, pe_working_set_t * data_set)
                                                replica->remote->xml, LOG_ERR);
             const char *calculated_addr = NULL;
 
+            // Replace the value in replica->remote->xml (if appropriate)
             calculated_addr = pe__add_bundle_remote_name(replica->remote,
+                                                         data_set,
                                                          nvpair, "value");
             if (calculated_addr) {
+                /* Since this is for the bundle as a resource, and not any
+                 * particular action, replace the value in the default
+                 * parameters (not evaluated for node). action2xml() will grab
+                 * it from there to replace it in node-evaluated parameters.
+                 */
+                GHashTable *params = pe_rsc_params(replica->remote,
+                                                   NULL, data_set);
+
                 crm_trace("Set address for bundle connection %s to bundle host %s",
                           replica->remote->id, calculated_addr);
-                g_hash_table_replace(replica->remote->parameters,
+                g_hash_table_replace(params,
                                      strdup(XML_RSC_ATTR_REMOTE_RA_ADDR),
                                      strdup(calculated_addr));
             } else {
diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
index 7b326e9..615a35a 100644
--- a/lib/pengine/bundle.c
+++ b/lib/pengine/bundle.c
@@ -948,29 +948,33 @@ replica_for_remote(pe_resource_t *remote)
 }
 
 bool
-pe__bundle_needs_remote_name(pe_resource_t *rsc)
+pe__bundle_needs_remote_name(pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     const char *value;
+    GHashTable *params = NULL;
 
     if (rsc == NULL) {
         return false;
     }
 
-    value = g_hash_table_lookup(rsc->parameters, XML_RSC_ATTR_REMOTE_RA_ADDR);
+    // Use NULL node since pcmk__bundle_expand() uses that to set value
+    params = pe_rsc_params(rsc, NULL, data_set);
+    value = g_hash_table_lookup(params, XML_RSC_ATTR_REMOTE_RA_ADDR);
 
     return pcmk__str_eq(value, "#uname", pcmk__str_casei)
            && xml_contains_remote_node(rsc->xml);
 }
 
 const char *
-pe__add_bundle_remote_name(pe_resource_t *rsc, xmlNode *xml, const char *field)
+pe__add_bundle_remote_name(pe_resource_t *rsc, pe_working_set_t *data_set,
+                           xmlNode *xml, const char *field)
 {
     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 
     pe_node_t *node = NULL;
     pe__bundle_replica_t *replica = NULL;
 
-    if (!pe__bundle_needs_remote_name(rsc)) {
+    if (!pe__bundle_needs_remote_name(rsc, data_set)) {
         return NULL;
     }
 
diff --git a/lib/pengine/pe_digest.c b/lib/pengine/pe_digest.c
index f55c896..f6e41e9 100644
--- a/lib/pengine/pe_digest.c
+++ b/lib/pengine/pe_digest.c
@@ -147,7 +147,7 @@ calculate_main_digest(op_digest_cache_t *data, pe_resource_t *rsc,
     /* REMOTE_CONTAINER_HACK: Allow Pacemaker Remote nodes to run containers
      * that themselves are Pacemaker Remote nodes
      */
-    if (pe__add_bundle_remote_name(rsc, data->params_all,
+    if (pe__add_bundle_remote_name(rsc, data_set, data->params_all,
                                    XML_RSC_ATTR_REMOTE_RA_ADDR)) {
         crm_trace("Set address for bundle connection %s (on %s)",
                   rsc->id, node->details->uname);
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index a15bb92..281bc88 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -3182,7 +3182,7 @@ should_clear_for_param_change(xmlNode *xml_op, const char *task,
 {
     if (!strcmp(task, "start") || !strcmp(task, "monitor")) {
 
-        if (pe__bundle_needs_remote_name(rsc)) {
+        if (pe__bundle_needs_remote_name(rsc, data_set)) {
             /* We haven't allocated resources yet, so we can't reliably
              * substitute addr parameters for the REMOTE_CONTAINER_HACK.
              * When that's needed, defer the check until later.
-- 
1.8.3.1


From 992b2edfe573a2bfd510090b37a8a1b355ad3c44 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 10 Nov 2020 15:32:29 -0600
Subject: [PATCH 06/13] Refactor: scheduler: use new resource parameter
 function when creating graph

---
 lib/pacemaker/pcmk_sched_graph.c | 30 +++++++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/lib/pacemaker/pcmk_sched_graph.c b/lib/pacemaker/pcmk_sched_graph.c
index c012d23..f0d1f47 100644
--- a/lib/pacemaker/pcmk_sched_graph.c
+++ b/lib/pacemaker/pcmk_sched_graph.c
@@ -1210,11 +1210,29 @@ action2xml(pe_action_t * action, gboolean as_input, pe_working_set_t *data_set)
 
     g_hash_table_foreach(action->extra, hash2field, args_xml);
     if (action->rsc != NULL && action->node) {
-        GHashTable *p = crm_str_table_new();
+        // Get the resource instance attributes, evaluated properly for node
+        GHashTable *params = pe_rsc_params(action->rsc, action->node, data_set);
 
-        get_rsc_attributes(p, action->rsc, action->node, data_set);
-        g_hash_table_foreach(p, hash2smartfield, args_xml);
-        g_hash_table_destroy(p);
+        /* REMOTE_CONTAINER_HACK: If this is a remote connection resource with
+         * addr="#uname", pull the actual value from the parameters evaluated
+         * without a node (which was put there earlier in stage8() when the
+         * bundle's expand() method was called).
+         */
+        const char *remote_addr = g_hash_table_lookup(params,
+                                                      XML_RSC_ATTR_REMOTE_RA_ADDR);
+
+        if (pcmk__str_eq(remote_addr, "#uname", pcmk__str_none)) {
+            GHashTable *base = pe_rsc_params(action->rsc, NULL, data_set);
+
+            remote_addr = g_hash_table_lookup(base,
+                                              XML_RSC_ATTR_REMOTE_RA_ADDR);
+            if (remote_addr != NULL) {
+                g_hash_table_insert(params, strdup(XML_RSC_ATTR_REMOTE_RA_ADDR),
+                                    strdup(remote_addr));
+            }
+        }
+
+        g_hash_table_foreach(params, hash2smartfield, args_xml);
 
 #if ENABLE_VERSIONED_ATTRS
         {
@@ -1230,7 +1248,9 @@ action2xml(pe_action_t * action, gboolean as_input, pe_working_set_t *data_set)
 #endif
 
     } else if(action->rsc && action->rsc->variant <= pe_native) {
-        g_hash_table_foreach(action->rsc->parameters, hash2smartfield, args_xml);
+        GHashTable *params = pe_rsc_params(action->rsc, NULL, data_set);
+
+        g_hash_table_foreach(params, hash2smartfield, args_xml);
 
 #if ENABLE_VERSIONED_ATTRS
         if (xml_has_children(action->rsc->versioned_parameters)) {
-- 
1.8.3.1


From 94316455ceead6f466e901d38e421f6116cf22d3 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Mon, 9 Nov 2020 16:04:23 -0600
Subject: [PATCH 07/13] Low: scheduler: calculate secure digest properly for
 node attribute rules

6830621 corrected secure digest calculation in most cases, but did not work
properly when non-sensitive parameters depended on node attribute-based rules
(since it used rsc->parameters, which is not evaluated for node).

This fixes it by using the new resource parameters function (which is the
equivalent of what calculate_main_digest() already was doing, so it is now
exposed earlier for use by both digest functions).
---
 lib/pengine/pe_digest.c | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/lib/pengine/pe_digest.c b/lib/pengine/pe_digest.c
index f6e41e9..2066a53 100644
--- a/lib/pengine/pe_digest.c
+++ b/lib/pengine/pe_digest.c
@@ -124,6 +124,7 @@ append_all_versioned_params(pe_resource_t *rsc, pe_node_t *node,
  * \param[out]    data         Digest cache entry to modify
  * \param[in]     rsc          Resource that action was for
  * \param[in]     node         Node action was performed on
+ * \param[in]     params       Resource parameters evaluated for node
  * \param[in]     task         Name of action performed
  * \param[in,out] interval_ms  Action's interval (will be reset if in overrides)
  * \param[in]     xml_op       XML of operation in CIB status (if available)
@@ -133,14 +134,12 @@ append_all_versioned_params(pe_resource_t *rsc, pe_node_t *node,
  */
 static void
 calculate_main_digest(op_digest_cache_t *data, pe_resource_t *rsc,
-                      pe_node_t *node, const char *task, guint *interval_ms,
+                      pe_node_t *node, GHashTable *params,
+                      const char *task, guint *interval_ms,
                       xmlNode *xml_op, const char *op_version,
                       GHashTable *overrides, pe_working_set_t *data_set)
 {
     pe_action_t *action = NULL;
-    GHashTable *local_rsc_params = crm_str_table_new();
-
-    get_rsc_attributes(local_rsc_params, rsc, node, data_set);
 
     data->params_all = create_xml_node(NULL, XML_TAG_PARAMS);
 
@@ -174,7 +173,7 @@ calculate_main_digest(op_digest_cache_t *data, pe_resource_t *rsc,
     if (overrides != NULL) {
         g_hash_table_foreach(overrides, hash2field, data->params_all);
     }
-    g_hash_table_foreach(local_rsc_params, hash2field, data->params_all);
+    g_hash_table_foreach(params, hash2field, data->params_all);
     g_hash_table_foreach(action->extra, hash2field, data->params_all);
     g_hash_table_foreach(action->meta, hash2metafield, data->params_all);
 
@@ -184,7 +183,6 @@ calculate_main_digest(op_digest_cache_t *data, pe_resource_t *rsc,
 
     pcmk__filter_op_for_digest(data->params_all);
 
-    g_hash_table_destroy(local_rsc_params);
     pe_free_action(action);
 
     data->digest_all_calc = calculate_operation_digest(data->params_all,
@@ -204,14 +202,15 @@ is_fence_param(xmlAttrPtr attr, void *user_data)
  *
  * \param[out] data        Digest cache entry to modify
  * \param[in]  rsc         Resource that action was for
+ * \param[in]  params      Resource parameters evaluated for node
  * \param[in]  xml_op      XML of operation in CIB status (if available)
  * \param[in]  op_version  CRM feature set to use for digest calculation
  * \param[in]  overrides   Key/value hash table to override resource parameters
  */
 static void
 calculate_secure_digest(op_digest_cache_t *data, pe_resource_t *rsc,
-                        xmlNode *xml_op, const char *op_version,
-                        GHashTable *overrides)
+                        GHashTable *params, xmlNode *xml_op,
+                        const char *op_version, GHashTable *overrides)
 {
     const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     const char *secure_list = NULL;
@@ -222,16 +221,12 @@ calculate_secure_digest(op_digest_cache_t *data, pe_resource_t *rsc,
         secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE);
     }
 
-    /* The controller doesn't create a digest of *all* non-sensitive
-     * parameters, only those listed in resource agent meta-data. The
-     * equivalent here is rsc->parameters.
-     */
     data->params_secure = create_xml_node(NULL, XML_TAG_PARAMS);
     if (overrides != NULL) {
         g_hash_table_foreach(overrides, hash2field, data->params_secure);
     }
 
-    g_hash_table_foreach(rsc->parameters, hash2field, data->params_secure);
+    g_hash_table_foreach(params, hash2field, data->params_secure);
     if (secure_list != NULL) {
         pcmk__xe_remove_matching_attrs(data->params_secure, attr_not_in_string,
                                        (void *) secure_list);
@@ -328,6 +323,7 @@ pe__calculate_digests(pe_resource_t *rsc, const char *task, guint *interval_ms,
 {
     op_digest_cache_t *data = calloc(1, sizeof(op_digest_cache_t));
     const char *op_version = CRM_FEATURE_SET;
+    GHashTable *params = NULL;
 
     if (data == NULL) {
         return NULL;
@@ -336,10 +332,12 @@ pe__calculate_digests(pe_resource_t *rsc, const char *task, guint *interval_ms,
         op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
     }
 
-    calculate_main_digest(data, rsc, node, task, interval_ms, xml_op,
+    params = pe_rsc_params(rsc, node, data_set);
+    calculate_main_digest(data, rsc, node, params, task, interval_ms, xml_op,
                           op_version, overrides, data_set);
     if (calc_secure) {
-        calculate_secure_digest(data, rsc, xml_op, op_version, overrides);
+        calculate_secure_digest(data, rsc, params, xml_op, op_version,
+                                overrides);
     }
     calculate_restart_digest(data, xml_op, op_version);
     return data;
-- 
1.8.3.1


From f546d0125b6f39fae744f43dad752089648e3f1f Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 10 Nov 2020 16:01:32 -0600
Subject: [PATCH 08/13] Fix: tools: respect rules when showing node attributes
 in crm_mon

Previously, crm_mon checked rsc->parameters for ocf:pacemaker:ping parameter
values. However that is not evaluated for node attribute rules. It also called
get_rsc_attributes() for all resources unnecessarily, since that's part of
common_unpack().

Now, use pe_rsc_params() to get the right values per node.
---
 tools/crm_mon.h         |  1 -
 tools/crm_mon_print.c   | 28 ++++++++++++++--------------
 tools/crm_mon_runtime.c | 15 +--------------
 3 files changed, 15 insertions(+), 29 deletions(-)

diff --git a/tools/crm_mon.h b/tools/crm_mon.h
index 143e8d8..f746507 100644
--- a/tools/crm_mon.h
+++ b/tools/crm_mon.h
@@ -109,7 +109,6 @@ int print_html_status(pcmk__output_t *out, pe_working_set_t *data_set,
 
 GList *append_attr_list(GList *attr_list, char *name);
 void blank_screen(void);
-void crm_mon_get_parameters(pe_resource_t *rsc, pe_working_set_t *data_set);
 unsigned int get_resource_display_options(unsigned int mon_ops);
 
 void crm_mon_register_messages(pcmk__output_t *out);
diff --git a/tools/crm_mon_print.c b/tools/crm_mon_print.c
index cc3efb0..8ae11bf 100644
--- a/tools/crm_mon_print.c
+++ b/tools/crm_mon_print.c
@@ -38,7 +38,8 @@ static int print_rsc_history(pcmk__output_t *out, pe_working_set_t *data_set,
 static int print_node_history(pcmk__output_t *out, pe_working_set_t *data_set,
                               pe_node_t *node, xmlNode *node_state, gboolean operations,
                               unsigned int mon_ops, GListPtr only_node, GListPtr only_rsc);
-static gboolean add_extra_info(pcmk__output_t *out, pe_node_t * node, GListPtr rsc_list,
+static gboolean add_extra_info(pcmk__output_t *out, pe_node_t *node,
+                               GListPtr rsc_list, pe_working_set_t *data_set,
                                const char *attrname, int *expected_score);
 static void print_node_attribute(gpointer name, gpointer user_data);
 static int print_node_summary(pcmk__output_t *out, pe_working_set_t * data_set,
@@ -330,7 +331,8 @@ print_node_history(pcmk__output_t *out, pe_working_set_t *data_set,
  */
 static gboolean
 add_extra_info(pcmk__output_t *out, pe_node_t *node, GListPtr rsc_list,
-               const char *attrname, int *expected_score)
+               pe_working_set_t *data_set, const char *attrname,
+               int *expected_score)
 {
     GListPtr gIter = NULL;
 
@@ -338,9 +340,11 @@ add_extra_info(pcmk__output_t *out, pe_node_t *node, GListPtr rsc_list,
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
         const char *type = g_hash_table_lookup(rsc->meta, "type");
         const char *name = NULL;
+        GHashTable *params = NULL;
 
         if (rsc->children != NULL) {
-            if (add_extra_info(out, node, rsc->children, attrname, expected_score)) {
+            if (add_extra_info(out, node, rsc->children, data_set, attrname,
+                               expected_score)) {
                 return TRUE;
             }
         }
@@ -349,7 +353,8 @@ add_extra_info(pcmk__output_t *out, pe_node_t *node, GListPtr rsc_list,
             continue;
         }
 
-        name = g_hash_table_lookup(rsc->parameters, "name");
+        params = pe_rsc_params(rsc, node, data_set);
+        name = g_hash_table_lookup(params, "name");
 
         if (name == NULL) {
             name = "pingd";
@@ -359,8 +364,8 @@ add_extra_info(pcmk__output_t *out, pe_node_t *node, GListPtr rsc_list,
         if (pcmk__str_eq(name, attrname, pcmk__str_casei)) {
             int host_list_num = 0;
             /* int value = crm_parse_int(attrvalue, "0"); */
-            const char *hosts = g_hash_table_lookup(rsc->parameters, "host_list");
-            const char *multiplier = g_hash_table_lookup(rsc->parameters, "multiplier");
+            const char *hosts = g_hash_table_lookup(params, "host_list");
+            const char *multiplier = g_hash_table_lookup(params, "multiplier");
 
             if (hosts) {
                 char **host_list = g_strsplit(hosts, " ", 0);
@@ -381,6 +386,7 @@ add_extra_info(pcmk__output_t *out, pe_node_t *node, GListPtr rsc_list,
 struct mon_attr_data {
     pcmk__output_t *out;
     pe_node_t *node;
+    pe_working_set_t *data_set;
 };
 
 static void
@@ -394,7 +400,7 @@ print_node_attribute(gpointer name, gpointer user_data)
     value = pe_node_attribute_raw(data->node, name);
 
     add_extra = add_extra_info(data->out, data->node, data->node->details->running_rsc,
-                               name, &expected_score);
+                               data->data_set, name, &expected_score);
 
     /* Print attribute name and value */
     data->out->message(data->out, "node-attribute", name, value, add_extra,
@@ -547,19 +553,13 @@ print_node_attributes(pcmk__output_t *out, pe_working_set_t *data_set,
     GListPtr gIter = NULL;
     int rc = pcmk_rc_no_output;
 
-    /* Unpack all resource parameters (it would be more efficient to do this
-     * only when needed for the first time in add_extra_info())
-     */
-    for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
-        crm_mon_get_parameters(gIter->data, data_set);
-    }
-
     /* Display each node's attributes */
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         struct mon_attr_data data;
 
         data.out = out;
         data.node = (pe_node_t *) gIter->data;
+        data.data_set = data_set;
 
         if (data.node && data.node->details && data.node->details->online) {
             GList *attr_list = NULL;
diff --git a/tools/crm_mon_runtime.c b/tools/crm_mon_runtime.c
index ce31559..43152ce 100644
--- a/tools/crm_mon_runtime.c
+++ b/tools/crm_mon_runtime.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2019 the Pacemaker project contributors
+ * Copyright 2019-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
@@ -66,19 +66,6 @@ append_attr_list(GList *attr_list, char *name)
     return g_list_insert_sorted(attr_list, name, compare_attribute);
 }
 
-void
-crm_mon_get_parameters(pe_resource_t *rsc, pe_working_set_t * data_set)
-{
-    get_rsc_attributes(rsc->parameters, rsc, NULL, data_set);
-    if(rsc->children) {
-        GListPtr gIter = NULL;
-
-        for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
-            crm_mon_get_parameters(gIter->data, data_set);
-        }
-    }
-}
-
 /*!
  * \internal
  * \brief Return resource display options corresponding to command-line choices
-- 
1.8.3.1


From 397ad868d464a0ffd14ed527a97010e141ac60b3 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Wed, 11 Nov 2020 12:59:34 -0600
Subject: [PATCH 09/13] Fix: scheduler: multiple issues with value-source in
 location rules

Delay creating rule match data for location constraints until the node is
known, and evaluate the rules using that node (using the new resource
parameter function). This fixes multiple issues:

* Previously, match data using resource parameters was created only when
  rsc-pattern was specified and a resource positively matched. This meant that
  a node attribute rule expression with a value-source of "param" or "meta"
  would only work in that case, and not when rsc was specified instead,
  or when rsc-pattern was specified with an inverted match ("!pattern"),
  or when a rule was used in a constraint with a resource set.

* Previously, with rsc-pattern, the match data used the resource's default
  parameters (not evaluated by node). This meant that a location rule that used
  a node attribute expression with a value-source of "param" could not be
  used with a resource parameter that itself was determined by a rule
  using a node attribute expression.
---
 lib/pacemaker/pcmk_sched_constraints.c | 40 ++++++++++++++++++----------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/lib/pacemaker/pcmk_sched_constraints.c b/lib/pacemaker/pcmk_sched_constraints.c
index 0029ad7..92b9740 100644
--- a/lib/pacemaker/pcmk_sched_constraints.c
+++ b/lib/pacemaker/pcmk_sched_constraints.c
@@ -47,7 +47,7 @@ static pe__location_t *generate_location_rule(pe_resource_t *rsc,
                                               const char *discovery,
                                               crm_time_t *next_change,
                                               pe_working_set_t *data_set,
-                                              pe_match_data_t *match_data);
+                                              pe_re_match_data_t *match_data);
 static void unpack_location(xmlNode *xml_obj, pe_working_set_t *data_set);
 static void unpack_rsc_colocation(xmlNode *xml_obj, pe_working_set_t *data_set);
 
@@ -714,7 +714,7 @@ tag_to_set(xmlNode * xml_obj, xmlNode ** rsc_set, const char * attr,
 static void unpack_rsc_location(xmlNode *xml_obj, pe_resource_t *rsc_lh,
                                 const char *role, const char *score,
                                 pe_working_set_t *data_set,
-                                pe_match_data_t *match_data);
+                                pe_re_match_data_t *match_data);
 
 static void
 unpack_simple_location(xmlNode *xml_obj, pe_working_set_t *data_set)
@@ -769,13 +769,9 @@ unpack_simple_location(xmlNode *xml_obj, pe_working_set_t *data_set)
                                                 .nregs = nregs,
                                                 .pmatch = pmatch
                                                };
-                pe_match_data_t match_data = {
-                                                .re = &re_match_data,
-                                                .params = r->parameters,
-                                                .meta = r->meta,
-                                             };
+
                 crm_debug("'%s' matched '%s' for %s", r->id, value, id);
-                unpack_rsc_location(xml_obj, r, NULL, NULL, data_set, &match_data);
+                unpack_rsc_location(xml_obj, r, NULL, NULL, data_set, &re_match_data);
 
             } else if (invert && (status != 0)) {
                 crm_debug("'%s' is an inverted match of '%s' for %s", r->id, value, id);
@@ -796,7 +792,7 @@ unpack_simple_location(xmlNode *xml_obj, pe_working_set_t *data_set)
 static void
 unpack_rsc_location(xmlNode *xml_obj, pe_resource_t *rsc_lh, const char *role,
                     const char *score, pe_working_set_t *data_set,
-                    pe_match_data_t *match_data)
+                    pe_re_match_data_t *re_match_data)
 {
     pe__location_t *location = NULL;
     const char *id_lh = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
@@ -836,7 +832,7 @@ unpack_rsc_location(xmlNode *xml_obj, pe_resource_t *rsc_lh, const char *role,
             empty = FALSE;
             crm_trace("Unpacking %s/%s", id, ID(rule_xml));
             generate_location_rule(rsc_lh, rule_xml, discovery, next_change,
-                                   data_set, match_data);
+                                   data_set, re_match_data);
         }
 
         if (empty) {
@@ -1067,7 +1063,8 @@ get_node_score(const char *rule, const char *score, gboolean raw, pe_node_t * no
 static pe__location_t *
 generate_location_rule(pe_resource_t *rsc, xmlNode *rule_xml,
                        const char *discovery, crm_time_t *next_change,
-                       pe_working_set_t *data_set, pe_match_data_t *match_data)
+                       pe_working_set_t *data_set,
+                       pe_re_match_data_t *re_match_data)
 {
     const char *rule_id = NULL;
     const char *score = NULL;
@@ -1113,14 +1110,14 @@ generate_location_rule(pe_resource_t *rsc, xmlNode *rule_xml,
         return NULL;
     }
 
-    if (match_data && match_data->re && match_data->re->nregs > 0 && match_data->re->pmatch[0].rm_so != -1) {
-        if (raw_score == FALSE) {
-            char *result = pe_expand_re_matches(score, match_data->re);
+    if ((re_match_data != NULL) && (re_match_data->nregs > 0)
+        && (re_match_data->pmatch[0].rm_so != -1) && !raw_score) {
 
-            if (result) {
-                score = (const char *) result;
-                score_allocated = TRUE;
-            }
+        char *result = pe_expand_re_matches(score, re_match_data);
+
+        if (result != NULL) {
+            score = result;
+            score_allocated = TRUE;
         }
     }
 
@@ -1148,9 +1145,14 @@ generate_location_rule(pe_resource_t *rsc, xmlNode *rule_xml,
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         int score_f = 0;
         pe_node_t *node = (pe_node_t *) gIter->data;
+        pe_match_data_t match_data = {
+            .re = re_match_data,
+            .params = pe_rsc_params(rsc, node, data_set),
+            .meta = rsc->meta,
+        };
 
         accept = pe_test_rule(rule_xml, node->details->attrs, RSC_ROLE_UNKNOWN,
-                              data_set->now, next_change, match_data);
+                              data_set->now, next_change, &match_data);
 
         crm_trace("Rule %s %s on %s", ID(rule_xml), accept ? "passed" : "failed",
                   node->details->uname);
-- 
1.8.3.1


From 40383ae2e2796b1bcbdb86bdd9cdc93c117eb69f Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Thu, 12 Nov 2020 15:32:47 -0600
Subject: [PATCH 10/13] Test: scheduler: add regression test for rules using
 value-source

---
 cts/cts-scheduler.in               |   1 +
 cts/scheduler/value-source.dot     |  29 +++
 cts/scheduler/value-source.exp     | 200 ++++++++++++++++
 cts/scheduler/value-source.scores  |  47 ++++
 cts/scheduler/value-source.summary |  60 +++++
 cts/scheduler/value-source.xml     | 463 +++++++++++++++++++++++++++++++++++++
 6 files changed, 800 insertions(+)
 create mode 100644 cts/scheduler/value-source.dot
 create mode 100644 cts/scheduler/value-source.exp
 create mode 100644 cts/scheduler/value-source.scores
 create mode 100644 cts/scheduler/value-source.summary
 create mode 100644 cts/scheduler/value-source.xml

diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in
index 23e6a91..939f8c8 100644
--- a/cts/cts-scheduler.in
+++ b/cts/cts-scheduler.in
@@ -145,6 +145,7 @@ TESTS = [
         [ "location-date-rules-1", "Use location constraints with ineffective date-based rules" ],
         [ "location-date-rules-2", "Use location constraints with effective date-based rules" ],
         [ "nvpair-date-rules-1", "Use nvpair blocks with a variety of date-based rules" ],
+        [ "value-source", "Use location constraints with node attribute expressions using value-source" ],
         [ "rule-dbl-as-auto-number-match",
           "Floating-point rule values default to number comparison: match" ],
         [ "rule-dbl-as-auto-number-no-match",
diff --git a/cts/scheduler/value-source.dot b/cts/scheduler/value-source.dot
new file mode 100644
index 0000000..dfb61e9
--- /dev/null
+++ b/cts/scheduler/value-source.dot
@@ -0,0 +1,29 @@
+ digraph "g" {
+"Fencing_monitor_120000 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"Fencing_start_0 rhel7-1" -> "Fencing_monitor_120000 rhel7-1" [ style = bold]
+"Fencing_start_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"insane-rsc_monitor_10000 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"insane-rsc_start_0 rhel7-4" -> "insane-rsc_monitor_10000 rhel7-4" [ style = bold]
+"insane-rsc_start_0 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"invert-match_monitor_10000 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"invert-match_start_0 rhel7-1" -> "invert-match_monitor_10000 rhel7-1" [ style = bold]
+"invert-match_start_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"meta-rsc_monitor_10000 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"meta-rsc_start_0 rhel7-5" -> "meta-rsc_monitor_10000 rhel7-5" [ style = bold]
+"meta-rsc_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"rsc1_monitor_10000 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc1_start_0 rhel7-4" -> "rsc1_monitor_10000 rhel7-4" [ style = bold]
+"rsc1_start_0 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc2_monitor_10000 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"rsc2_start_0 rhel7-5" -> "rsc2_monitor_10000 rhel7-5" [ style = bold]
+"rsc2_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"set-rsc1_monitor_10000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"set-rsc1_start_0 rhel7-3" -> "set-rsc1_monitor_10000 rhel7-3" [ style = bold]
+"set-rsc1_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"set-rsc2_monitor_10000 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"set-rsc2_start_0 rhel7-4" -> "set-rsc2_monitor_10000 rhel7-4" [ style = bold]
+"set-rsc2_start_0 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"single-rsc_monitor_10000 rhel7-2" [ style=bold color="green" fontcolor="black"]
+"single-rsc_start_0 rhel7-2" -> "single-rsc_monitor_10000 rhel7-2" [ style = bold]
+"single-rsc_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/cts/scheduler/value-source.exp b/cts/scheduler/value-source.exp
new file mode 100644
index 0000000..4bf469f
--- /dev/null
+++ b/cts/scheduler/value-source.exp
@@ -0,0 +1,200 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="1"  transition_id="0">
+  <synapse id="0">
+    <action_set>
+      <rsc_op id="2" operation="monitor" operation_key="Fencing_monitor_120000" on_node="rhel7-1" on_node_uuid="1">
+        <primitive id="Fencing" class="stonith" type="fence_xvm"/>
+        <attributes CRM_meta_interval="120000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="120000"  key_file="/etc/pacemaker/fence_xvm.key" multicast_address="239.255.100.100" pcmk_host_list="rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="1" operation="start" operation_key="Fencing_start_0" on_node="rhel7-1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="1">
+    <action_set>
+      <rsc_op id="1" operation="start" operation_key="Fencing_start_0" on_node="rhel7-1" on_node_uuid="1">
+        <primitive id="Fencing" class="stonith" type="fence_xvm"/>
+        <attributes CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="90000"  key_file="/etc/pacemaker/fence_xvm.key" multicast_address="239.255.100.100" pcmk_host_list="rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5"/>
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="2">
+    <action_set>
+      <rsc_op id="4" operation="monitor" operation_key="rsc1_monitor_10000" on_node="rhel7-4" on_node_uuid="4">
+        <primitive id="rsc1" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_timeout="20000"  fake="300"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="3" operation="start" operation_key="rsc1_start_0" on_node="rhel7-4" on_node_uuid="4"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="3">
+    <action_set>
+      <rsc_op id="3" operation="start" operation_key="rsc1_start_0" on_node="rhel7-4" on_node_uuid="4">
+        <primitive id="rsc1" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_name="start" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_timeout="20000"  fake="300"/>
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="4">
+    <action_set>
+      <rsc_op id="6" operation="monitor" operation_key="rsc2_monitor_10000" on_node="rhel7-5" on_node_uuid="5">
+        <primitive id="rsc2" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-5" CRM_meta_on_node_uuid="5" CRM_meta_timeout="20000"  fake="400"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="5" operation="start" operation_key="rsc2_start_0" on_node="rhel7-5" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="5">
+    <action_set>
+      <rsc_op id="5" operation="start" operation_key="rsc2_start_0" on_node="rhel7-5" on_node_uuid="5">
+        <primitive id="rsc2" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_name="start" CRM_meta_on_node="rhel7-5" CRM_meta_on_node_uuid="5" CRM_meta_timeout="20000"  fake="400"/>
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="6">
+    <action_set>
+      <rsc_op id="8" operation="monitor" operation_key="invert-match_monitor_10000" on_node="rhel7-1" on_node_uuid="1">
+        <primitive id="invert-match" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000"  fake="200"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="7" operation="start" operation_key="invert-match_start_0" on_node="rhel7-1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="7">
+    <action_set>
+      <rsc_op id="7" operation="start" operation_key="invert-match_start_0" on_node="rhel7-1" on_node_uuid="1">
+        <primitive id="invert-match" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_name="start" CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000"  fake="200"/>
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="8">
+    <action_set>
+      <rsc_op id="10" operation="monitor" operation_key="single-rsc_monitor_10000" on_node="rhel7-2" on_node_uuid="2">
+        <primitive id="single-rsc" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000"  fake="200"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="9" operation="start" operation_key="single-rsc_start_0" on_node="rhel7-2" on_node_uuid="2"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="9">
+    <action_set>
+      <rsc_op id="9" operation="start" operation_key="single-rsc_start_0" on_node="rhel7-2" on_node_uuid="2">
+        <primitive id="single-rsc" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_name="start" CRM_meta_on_node="rhel7-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000"  fake="200"/>
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="10">
+    <action_set>
+      <rsc_op id="12" operation="monitor" operation_key="set-rsc1_monitor_10000" on_node="rhel7-3" on_node_uuid="3">
+        <primitive id="set-rsc1" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000"  fake="300"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="11" operation="start" operation_key="set-rsc1_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="11">
+    <action_set>
+      <rsc_op id="11" operation="start" operation_key="set-rsc1_start_0" on_node="rhel7-3" on_node_uuid="3">
+        <primitive id="set-rsc1" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_name="start" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000"  fake="300"/>
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="12">
+    <action_set>
+      <rsc_op id="14" operation="monitor" operation_key="set-rsc2_monitor_10000" on_node="rhel7-4" on_node_uuid="4">
+        <primitive id="set-rsc2" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_timeout="20000"  fake="400"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="13" operation="start" operation_key="set-rsc2_start_0" on_node="rhel7-4" on_node_uuid="4"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="13">
+    <action_set>
+      <rsc_op id="13" operation="start" operation_key="set-rsc2_start_0" on_node="rhel7-4" on_node_uuid="4">
+        <primitive id="set-rsc2" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_name="start" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_timeout="20000"  fake="400"/>
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="14">
+    <action_set>
+      <rsc_op id="16" operation="monitor" operation_key="meta-rsc_monitor_10000" on_node="rhel7-5" on_node_uuid="5">
+        <primitive id="meta-rsc" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-5" CRM_meta_on_node_uuid="5" CRM_meta_timeout="20000"  fake="500"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="15" operation="start" operation_key="meta-rsc_start_0" on_node="rhel7-5" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="15">
+    <action_set>
+      <rsc_op id="15" operation="start" operation_key="meta-rsc_start_0" on_node="rhel7-5" on_node_uuid="5">
+        <primitive id="meta-rsc" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_name="start" CRM_meta_on_node="rhel7-5" CRM_meta_on_node_uuid="5" CRM_meta_timeout="20000"  fake="500"/>
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="16">
+    <action_set>
+      <rsc_op id="18" operation="monitor" operation_key="insane-rsc_monitor_10000" on_node="rhel7-4" on_node_uuid="4">
+        <primitive id="insane-rsc" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_timeout="20000"  fake="400"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="17" operation="start" operation_key="insane-rsc_start_0" on_node="rhel7-4" on_node_uuid="4"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="17">
+    <action_set>
+      <rsc_op id="17" operation="start" operation_key="insane-rsc_start_0" on_node="rhel7-4" on_node_uuid="4">
+        <primitive id="insane-rsc" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_name="start" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_timeout="20000"  fake="400"/>
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+</transition_graph>
diff --git a/cts/scheduler/value-source.scores b/cts/scheduler/value-source.scores
new file mode 100644
index 0000000..1f781c4
--- /dev/null
+++ b/cts/scheduler/value-source.scores
@@ -0,0 +1,47 @@
+Allocation scores:
+Using the original execution date of: 2020-11-12 21:28:08Z
+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-2: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0
+pcmk__native_allocate: insane-rsc allocation score on rhel7-1: 0
+pcmk__native_allocate: insane-rsc allocation score on rhel7-2: 0
+pcmk__native_allocate: insane-rsc allocation score on rhel7-3: 0
+pcmk__native_allocate: insane-rsc allocation score on rhel7-4: INFINITY
+pcmk__native_allocate: insane-rsc allocation score on rhel7-5: 0
+pcmk__native_allocate: invert-match allocation score on rhel7-1: INFINITY
+pcmk__native_allocate: invert-match allocation score on rhel7-2: 0
+pcmk__native_allocate: invert-match allocation score on rhel7-3: 0
+pcmk__native_allocate: invert-match allocation score on rhel7-4: 0
+pcmk__native_allocate: invert-match allocation score on rhel7-5: 0
+pcmk__native_allocate: meta-rsc allocation score on rhel7-1: 0
+pcmk__native_allocate: meta-rsc allocation score on rhel7-2: 0
+pcmk__native_allocate: meta-rsc allocation score on rhel7-3: 0
+pcmk__native_allocate: meta-rsc allocation score on rhel7-4: INFINITY
+pcmk__native_allocate: meta-rsc allocation score on rhel7-5: INFINITY
+pcmk__native_allocate: rsc1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1 allocation score on rhel7-3: 0
+pcmk__native_allocate: rsc1 allocation score on rhel7-4: INFINITY
+pcmk__native_allocate: rsc1 allocation score on rhel7-5: INFINITY
+pcmk__native_allocate: rsc2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2 allocation score on rhel7-3: 0
+pcmk__native_allocate: rsc2 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc2 allocation score on rhel7-5: INFINITY
+pcmk__native_allocate: set-rsc1 allocation score on rhel7-1: 0
+pcmk__native_allocate: set-rsc1 allocation score on rhel7-2: 0
+pcmk__native_allocate: set-rsc1 allocation score on rhel7-3: INFINITY
+pcmk__native_allocate: set-rsc1 allocation score on rhel7-4: 0
+pcmk__native_allocate: set-rsc1 allocation score on rhel7-5: 0
+pcmk__native_allocate: set-rsc2 allocation score on rhel7-1: 0
+pcmk__native_allocate: set-rsc2 allocation score on rhel7-2: 0
+pcmk__native_allocate: set-rsc2 allocation score on rhel7-3: 0
+pcmk__native_allocate: set-rsc2 allocation score on rhel7-4: INFINITY
+pcmk__native_allocate: set-rsc2 allocation score on rhel7-5: 0
+pcmk__native_allocate: single-rsc allocation score on rhel7-1: 0
+pcmk__native_allocate: single-rsc allocation score on rhel7-2: INFINITY
+pcmk__native_allocate: single-rsc allocation score on rhel7-3: 0
+pcmk__native_allocate: single-rsc allocation score on rhel7-4: 0
+pcmk__native_allocate: single-rsc allocation score on rhel7-5: 0
diff --git a/cts/scheduler/value-source.summary b/cts/scheduler/value-source.summary
new file mode 100644
index 0000000..a9b0392
--- /dev/null
+++ b/cts/scheduler/value-source.summary
@@ -0,0 +1,60 @@
+Using the original execution date of: 2020-11-12 21:28:08Z
+
+Current cluster status:
+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing	(stonith:fence_xvm):	 Stopped
+ rsc1	(ocf::pacemaker:Dummy):	 Stopped
+ rsc2	(ocf::pacemaker:Dummy):	 Stopped
+ invert-match	(ocf::pacemaker:Dummy):	 Stopped
+ single-rsc	(ocf::pacemaker:Dummy):	 Stopped
+ set-rsc1	(ocf::pacemaker:Dummy):	 Stopped
+ set-rsc2	(ocf::pacemaker:Dummy):	 Stopped
+ meta-rsc	(ocf::pacemaker:Dummy):	 Stopped
+ insane-rsc	(ocf::pacemaker:Dummy):	 Stopped
+
+Transition Summary:
+ * Start      Fencing          ( rhel7-1 )  
+ * Start      rsc1             ( rhel7-4 )  
+ * Start      rsc2             ( rhel7-5 )  
+ * Start      invert-match     ( rhel7-1 )  
+ * Start      single-rsc       ( rhel7-2 )  
+ * Start      set-rsc1         ( rhel7-3 )  
+ * Start      set-rsc2         ( rhel7-4 )  
+ * Start      meta-rsc         ( rhel7-5 )  
+ * Start      insane-rsc       ( rhel7-4 )  
+
+Executing cluster transition:
+ * Resource action: Fencing         start on rhel7-1
+ * Resource action: rsc1            start on rhel7-4
+ * Resource action: rsc2            start on rhel7-5
+ * Resource action: invert-match    start on rhel7-1
+ * Resource action: single-rsc      start on rhel7-2
+ * Resource action: set-rsc1        start on rhel7-3
+ * Resource action: set-rsc2        start on rhel7-4
+ * Resource action: meta-rsc        start on rhel7-5
+ * Resource action: insane-rsc      start on rhel7-4
+ * Resource action: Fencing         monitor=120000 on rhel7-1
+ * Resource action: rsc1            monitor=10000 on rhel7-4
+ * Resource action: rsc2            monitor=10000 on rhel7-5
+ * Resource action: invert-match    monitor=10000 on rhel7-1
+ * Resource action: single-rsc      monitor=10000 on rhel7-2
+ * Resource action: set-rsc1        monitor=10000 on rhel7-3
+ * Resource action: set-rsc2        monitor=10000 on rhel7-4
+ * Resource action: meta-rsc        monitor=10000 on rhel7-5
+ * Resource action: insane-rsc      monitor=10000 on rhel7-4
+Using the original execution date of: 2020-11-12 21:28:08Z
+
+Revised cluster status:
+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing	(stonith:fence_xvm):	 Started rhel7-1
+ rsc1	(ocf::pacemaker:Dummy):	 Started rhel7-4
+ rsc2	(ocf::pacemaker:Dummy):	 Started rhel7-5
+ invert-match	(ocf::pacemaker:Dummy):	 Started rhel7-1
+ single-rsc	(ocf::pacemaker:Dummy):	 Started rhel7-2
+ set-rsc1	(ocf::pacemaker:Dummy):	 Started rhel7-3
+ set-rsc2	(ocf::pacemaker:Dummy):	 Started rhel7-4
+ meta-rsc	(ocf::pacemaker:Dummy):	 Started rhel7-5
+ insane-rsc	(ocf::pacemaker:Dummy):	 Started rhel7-4
+
diff --git a/cts/scheduler/value-source.xml b/cts/scheduler/value-source.xml
new file mode 100644
index 0000000..95e7d57
--- /dev/null
+++ b/cts/scheduler/value-source.xml
@@ -0,0 +1,463 @@
+<cib crm_feature_set="3.6.3" validate-with="pacemaker-3.5" epoch="96" num_updates="0" admin_epoch="1" cib-last-written="Thu Nov 12 15:28:08 2020" update-origin="rhel7-2" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="3" execution-date="1605216488">
+  <!-- This is a multiple test of location constraints with node attribute
+       expressions using a value-source of "param" or "meta". The non-fencing
+       resources are all Dummy resources, and their resource parameter "fake"
+       is used in the rules, along with a node attribute also named "fake".
+
+       There are the following 5 nodes with these node attribute values:
+
+        rhel7-1 fake=100 rack=1
+        rhel7-2 fake=200 rack=1
+        rhel7-3 fake=300 rack=1
+        rhel7-4 fake=400 rack=2
+        rhel7-5 fake=500 rack=2
+
+       A location constraint with a rsc-pattern of "^rsc[0-9]$" matches the
+       following resources, and uses a +INFINITY expression where
+       node fake > resource fake:
+         rsc1 fake=300
+         rsc2 fake=400
+       Thus rsc1 should prefer node rhel7-4 or rhel7-5, and rsc2 should
+       prefer rhel7-5.
+
+       A location constraint with an inverted rsc-pattern of "!rsc" matches
+       the following resource, and uses a +INFINITY expression where
+       node fake < resource fake:
+         invert-match fake=200
+       Thus invert-match should prefer rhel7-1.
+
+       A location constraint with rsc="single-rsc" uses a +INFINITY expression
+       where node fake == resource fake, and the resource is:
+         single-rsc fake=200
+       Thus single-rsc should prefer rhel7-2.
+
+       A location constraint with a resource set uses a +INFINITY expression
+       where node fake == resource fake, and the resources in the set are:
+         set-rsc1 fake=300
+         set-rsc2 fake=400
+       Thus set-rsc1 should prefer rhel7-3, and set-rsc2 should prefer rhel7-4.
+
+       To test value-source="meta", a location constraint with a rsc-pattern of
+       "meta-" uses a +INFINITY expression where node rack == resource meta rack,
+       and there is a resource:
+         meta-rsc fake=500 rack=2
+       Thus meta-rsc should prefer rhel7-4 or rhel7-5.
+
+       Finally there is a configuration that is useful solely for verifying
+       that the code can handle it. A location constraint with rsc="insane-rsc"
+       uses a +INFINITY expression where node fake == resource fake. However
+       the resource parameter itself uses a rule with a node attribute
+       expression such that it is 200 on rhel7-1, 400 on rhel7-4, and 100
+       everywhere else. Thus insane-rsc will prefer rhel7-4.
+    -->
+  <configuration>
+    <crm_config>
+      <cluster_property_set id="cib-bootstrap-options">
+        <nvpair id="cts-stonith-enabled" name="stonith-enabled" value="1"/>
+        <nvpair id="cts-start-failure-is-fatal" name="start-failure-is-fatal" value="false"/>
+        <nvpair id="cts-pe-input-series-max" name="pe-input-series-max" value="5000"/>
+        <nvpair id="cts-shutdown-escalation" name="shutdown-escalation" value="5min"/>
+        <nvpair id="cts-batch-limit" name="batch-limit" value="10"/>
+        <nvpair id="cts-dc-deadtime" name="dc-deadtime" value="5s"/>
+        <nvpair id="cts-no-quorum-policy" name="no-quorum-policy" value="stop"/>
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.5-650.0cdd837.git.el7_9-0cdd837"/>
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="mycluster"/>
+        <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1604951742"/>
+      </cluster_property_set>
+    </crm_config>
+    <nodes>
+      <node id="1" uname="rhel7-1">
+        <instance_attributes id="nodes-1">
+          <nvpair id="nodes-1-fake" name="fake" value="100"/>
+          <nvpair id="nodes-1-rack" name="rack" value="1"/>
+        </instance_attributes>
+      </node>
+      <node id="2" uname="rhel7-2">
+        <instance_attributes id="nodes-2">
+          <nvpair id="nodes-2-fake" name="fake" value="200"/>
+          <nvpair id="nodes-2-rack" name="rack" value="1"/>
+        </instance_attributes>
+      </node>
+      <node id="3" uname="rhel7-3">
+        <instance_attributes id="nodes-3">
+          <nvpair id="nodes-3-fake" name="fake" value="300"/>
+          <nvpair id="nodes-3-rack" name="rack" value="1"/>
+        </instance_attributes>
+      </node>
+      <node id="4" uname="rhel7-4">
+        <instance_attributes id="nodes-4">
+          <nvpair id="nodes-4-fake" name="fake" value="400"/>
+          <nvpair id="nodes-4-rack" name="rack" value="2"/>
+        </instance_attributes>
+      </node>
+      <node id="5" uname="rhel7-5">
+        <instance_attributes id="nodes-5">
+          <nvpair id="nodes-5-fake" name="fake" value="500"/>
+          <nvpair id="nodes-5-rack" name="rack" value="2"/>
+        </instance_attributes>
+      </node>
+    </nodes>
+    <resources>
+      <primitive class="stonith" id="Fencing" type="fence_xvm">
+        <instance_attributes id="Fencing-params">
+          <nvpair id="Fencing-key_file" name="key_file" value="/etc/pacemaker/fence_xvm.key"/>
+          <nvpair id="Fencing-multicast_address" name="multicast_address" value="239.255.100.100"/>
+          <nvpair id="Fencing-pcmk_host_list" name="pcmk_host_list" value="rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5"/>
+        </instance_attributes>
+        <operations>
+          <op id="Fencing-monitor-120s" interval="120s" name="monitor" timeout="120s"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="rsc1" provider="pacemaker" type="Dummy">
+        <instance_attributes id="rsc1-instance_attributes" score="10">
+          <nvpair id="rsc1-instance_attributes-fake" name="fake" value="300"/>
+        </instance_attributes>
+        <operations>
+          <op id="rsc1-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
+          <op id="rsc1-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
+          <op id="rsc1-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
+          <op id="rsc1-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
+          <op id="rsc1-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+          <op id="rsc1-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="rsc2" provider="pacemaker" type="Dummy">
+        <instance_attributes id="rsc2-instance_attributes" score="10">
+          <nvpair id="rsc2-instance_attributes-fake" name="fake" value="400"/>
+        </instance_attributes>
+        <operations>
+          <op id="rsc2-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
+          <op id="rsc2-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
+          <op id="rsc2-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
+          <op id="rsc2-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
+          <op id="rsc2-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+          <op id="rsc2-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="invert-match" provider="pacemaker" type="Dummy">
+        <instance_attributes id="invert-match-instance_attributes" score="10">
+          <nvpair id="invert-match-instance_attributes-fake" name="fake" value="200"/>
+        </instance_attributes>
+        <operations>
+          <op id="invert-match-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
+          <op id="invert-match-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
+          <op id="invert-match-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
+          <op id="invert-match-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
+          <op id="invert-match-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+          <op id="invert-match-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="single-rsc" provider="pacemaker" type="Dummy">
+        <instance_attributes id="single-rsc-instance_attributes" score="10">
+          <nvpair id="single-rsc-instance_attributes-fake" name="fake" value="200"/>
+        </instance_attributes>
+        <operations>
+          <op id="single-rsc-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
+          <op id="single-rsc-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
+          <op id="single-rsc-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
+          <op id="single-rsc-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
+          <op id="single-rsc-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+          <op id="single-rsc-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="set-rsc1" provider="pacemaker" type="Dummy">
+        <instance_attributes id="set-rsc1-instance_attributes" score="10">
+          <nvpair id="set-rsc1-instance_attributes-fake" name="fake" value="300"/>
+        </instance_attributes>
+        <operations>
+          <op id="set-rsc1-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
+          <op id="set-rsc1-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
+          <op id="set-rsc1-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
+          <op id="set-rsc1-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
+          <op id="set-rsc1-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+          <op id="set-rsc1-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="set-rsc2" provider="pacemaker" type="Dummy">
+        <instance_attributes id="set-rsc2-instance_attributes" score="10">
+          <nvpair id="set-rsc2-instance_attributes-fake" name="fake" value="400"/>
+        </instance_attributes>
+        <operations>
+          <op id="set-rsc2-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
+          <op id="set-rsc2-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
+          <op id="set-rsc2-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
+          <op id="set-rsc2-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
+          <op id="set-rsc2-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+          <op id="set-rsc2-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="meta-rsc" provider="pacemaker" type="Dummy">
+        <meta_attributes id="meta-rsc-meta_attributes">
+          <nvpair id="meta-rsc-meta_attributes-rack" name="rack" value="2"/>
+        </meta_attributes>
+        <instance_attributes id="meta-rsc-instance_attributes" score="10">
+          <nvpair id="meta-rsc-instance_attributes-fake" name="fake" value="500"/>
+        </instance_attributes>
+        <operations>
+          <op id="meta-rsc-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
+          <op id="meta-rsc-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
+          <op id="meta-rsc-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
+          <op id="meta-rsc-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
+          <op id="meta-rsc-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+          <op id="meta-rsc-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="insane-rsc" provider="pacemaker" type="Dummy">
+        <instance_attributes id="insane-rsc-instance_attributes" score="10">
+          <nvpair id="insane-rsc-instance_attributes-fake" name="fake" value="100"/>
+        </instance_attributes>
+        <instance_attributes id="insane-rsc-instance_attributes2" score="20">
+          <rule id="insane-rsc-rule1" score="INFINITY">
+            <expression id="insane-rsc-rule1-expr1" attribute="#uname" operation="eq" value="rhel7-1"/>
+          </rule>
+          <nvpair id="insane-rsc-instance_attributes2-fake" name="fake" value="200"/>
+        </instance_attributes>
+        <instance_attributes id="insane-rsc-instance_attributes3" score="20">
+          <rule id="insane-rsc-rule2" score="INFINITY">
+            <expression id="insane-rsc-rule2-expr1" attribute="#uname" operation="eq" value="rhel7-4"/>
+          </rule>
+          <nvpair id="insane-rsc-instance_attributes23fake" name="fake" value="400"/>
+        </instance_attributes>
+        <operations>
+          <op id="insane-rsc-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
+          <op id="insane-rsc-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
+          <op id="insane-rsc-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
+          <op id="insane-rsc-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
+          <op id="insane-rsc-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+          <op id="insane-rsc-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+        </operations>
+      </primitive>
+    </resources>
+    <constraints>
+      <rsc_location id="location-rscN" rsc-pattern="^rsc[0-9]$">
+        <rule id="location-rule1" score="INFINITY">
+          <expression id="location-rule1-expr1" attribute="fake" operation="gt" value="fake" value-source="param"/>
+        </rule>
+      </rsc_location>
+      <rsc_location id="location-not-rsc" rsc-pattern="!rsc">
+        <rule id="location-rule2" score="INFINITY">
+          <expression id="location-rule2-expr1" attribute="fake" operation="lt" value="fake" value-source="param"/>
+        </rule>
+      </rsc_location>
+      <rsc_location id="location-single-rsc" rsc="single-rsc">
+        <rule id="location-rule3" score="INFINITY">
+          <expression id="location-rule3-expr1" attribute="fake" operation="eq" value="fake" value-source="param"/>
+        </rule>
+      </rsc_location>
+      <rsc_location id="location-set">
+        <resource_set id="resource-set1">
+          <resource_ref id="set-rsc1"/>
+          <resource_ref id="set-rsc2"/>
+        </resource_set>
+        <rule id="location-rule4" score="INFINITY">
+          <expression id="location-rule4-expr1" attribute="fake" operation="eq" value="fake" value-source="param"/>
+        </rule>
+      </rsc_location>
+      <rsc_location id="location-meta" rsc-pattern="meta-">
+        <rule id="location-rule5" score="INFINITY">
+          <expression id="location-rule5-expr1" attribute="rack" operation="eq" value="rack" value-source="meta"/>
+        </rule>
+      </rsc_location>
+      <rsc_location id="location-insane-rsc" rsc="insane-rsc">
+        <rule id="location-rule6" score="INFINITY">
+          <expression id="location-rule6-expr1" attribute="fake" operation="eq" value="fake" value-source="param"/>
+        </rule>
+      </rsc_location>
+    </constraints>
+    <fencing-topology/>
+    <op_defaults>
+      <meta_attributes id="cts-op_defaults-meta">
+        <nvpair id="cts-op_defaults-timeout" name="timeout" value="90s"/>
+      </meta_attributes>
+    </op_defaults>
+    <alerts>
+      <alert id="alert-1" path="/var/lib/pacemaker/notify.sh">
+        <recipient id="alert-1-recipient-1" value="/run/crm/alert.log"/>
+      </alert>
+    </alerts>
+    <rsc_defaults>
+      <meta_attributes id="rsc_defaults-options">
+        <nvpair id="rsc_defaults-options-target-role" name="target-role" value="Started"/>
+      </meta_attributes>
+    </rsc_defaults>
+  </configuration>
+  <status>
+    <node_state id="1" uname="rhel7-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="1">
+        <lrm_resources>
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.6.3" transition-key="10:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;10:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-1" call-id="63" rc-code="0" op-status="0" interval="0" last-rc-change="1605216481" last-run="1605216481" exec-time="1" queue-time="0" op-digest="422e90c96b7732222706b322138f43c8"/>
+            <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="2:15:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;2:15:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-1" call-id="49" rc-code="0" op-status="0" interval="120000" last-rc-change="1605212720" exec-time="75" queue-time="0" op-digest="75eb18cbd607b3b4911f723ab1c89388"/>
+          </lrm_resource>
+          <lrm_resource id="meta-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="meta-rsc_last_0" operation_key="meta-rsc_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="8:23:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;8:23:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-1" call-id="57" rc-code="7" op-status="0" interval="0" last-rc-change="1605215239" last-run="1605215239" exec-time="34" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="2:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;2:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-1" call-id="13" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="21" queue-time="0" op-digest="740d07f67c20fca411b03996e601119e" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="740d07f67c20fca411b03996e601119e"/>
+          </lrm_resource>
+          <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="3:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;3:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-1" call-id="17" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="26" queue-time="0" op-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84"/>
+          </lrm_resource>
+          <lrm_resource id="invert-match" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="invert-match_last_0" operation_key="invert-match_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.6.3" transition-key="13:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;13:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-1" call-id="67" rc-code="0" op-status="0" interval="0" last-rc-change="1605216481" last-run="1605216481" exec-time="59" queue-time="0" op-digest="1bd01c2c51e0e8a776ac0e84d5715d70" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1bd01c2c51e0e8a776ac0e84d5715d70"/>
+            <lrm_rsc_op id="invert-match_monitor_10000" operation_key="invert-match_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="8:15:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;8:15:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-1" call-id="51" rc-code="0" op-status="0" interval="10000" last-rc-change="1605212720" exec-time="52" queue-time="3" op-digest="53e49b6c90e7e71546c273e0f220a321" op-secure-params="  passwd  " op-secure-digest="1bd01c2c51e0e8a776ac0e84d5715d70"/>
+          </lrm_resource>
+          <lrm_resource id="single-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="single-rsc_last_0" operation_key="single-rsc_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="5:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;5:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-1" call-id="29" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="25" queue-time="0" op-digest="1bd01c2c51e0e8a776ac0e84d5715d70" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1bd01c2c51e0e8a776ac0e84d5715d70"/>
+          </lrm_resource>
+          <lrm_resource id="insane-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="insane-rsc_last_0" operation_key="insane-rsc_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="9:29:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;9:29:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-1" call-id="61" rc-code="7" op-status="0" interval="0" last-rc-change="1605216014" last-run="1605216014" exec-time="33" queue-time="0" op-digest="1905d2bf96a4b1c41f38dff8c7b84308" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1905d2bf96a4b1c41f38dff8c7b84308"/>
+          </lrm_resource>
+          <lrm_resource id="set-rsc1" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="set-rsc1_last_0" operation_key="set-rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="6:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;6:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-1" call-id="33" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="20" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="set-rsc2" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="set-rsc2_last_0" operation_key="set-rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="7:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;7:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-1" call-id="37" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="20" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+    <node_state id="2" uname="rhel7-2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="2">
+        <lrm_resources>
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="8:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;8:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-2" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="5" queue-time="0" op-digest="422e90c96b7732222706b322138f43c8"/>
+          </lrm_resource>
+          <lrm_resource id="meta-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="meta-rsc_last_0" operation_key="meta-rsc_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="23:27:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;23:27:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-2" call-id="57" rc-code="0" op-status="0" interval="0" last-rc-change="1605215447" last-run="1605215447" exec-time="24" queue-time="0" op-digest="4269017c9755b64d3c5687024693ab8d"/>
+          </lrm_resource>
+          <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="9:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;9:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-2" call-id="9" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="24" queue-time="0" op-digest="740d07f67c20fca411b03996e601119e" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="740d07f67c20fca411b03996e601119e"/>
+          </lrm_resource>
+          <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="10:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;10:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-2" call-id="13" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="35" queue-time="0" op-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84"/>
+          </lrm_resource>
+          <lrm_resource id="invert-match" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="invert-match_last_0" operation_key="invert-match_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="11:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;11:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-2" call-id="17" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="29" queue-time="0" op-digest="1bd01c2c51e0e8a776ac0e84d5715d70" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1bd01c2c51e0e8a776ac0e84d5715d70"/>
+          </lrm_resource>
+          <lrm_resource id="single-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="single-rsc_last_0" operation_key="single-rsc_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.6.3" transition-key="14:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;14:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-2" call-id="73" rc-code="0" op-status="0" interval="0" last-rc-change="1605216481" last-run="1605216481" exec-time="70" queue-time="0" op-digest="1bd01c2c51e0e8a776ac0e84d5715d70" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1bd01c2c51e0e8a776ac0e84d5715d70"/>
+            <lrm_rsc_op id="single-rsc_monitor_10000" operation_key="single-rsc_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="10:15:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;10:15:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-2" call-id="40" rc-code="0" op-status="0" interval="10000" last-rc-change="1605212720" exec-time="32" queue-time="0" op-digest="53e49b6c90e7e71546c273e0f220a321" op-secure-params="  passwd  " op-secure-digest="1bd01c2c51e0e8a776ac0e84d5715d70"/>
+          </lrm_resource>
+          <lrm_resource id="insane-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="insane-rsc_last_0" operation_key="insane-rsc_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.6.3" transition-key="26:32:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;26:32:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-2" call-id="69" rc-code="0" op-status="0" interval="0" last-rc-change="1605216258" last-run="1605216258" exec-time="27" queue-time="0" op-digest="1905d2bf96a4b1c41f38dff8c7b84308" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1905d2bf96a4b1c41f38dff8c7b84308"/>
+            <lrm_rsc_op id="insane-rsc_monitor_10000" operation_key="insane-rsc_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="31:29:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;31:29:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-2" call-id="66" rc-code="0" op-status="0" interval="10000" last-rc-change="1605216014" exec-time="21" queue-time="0" op-digest="b87d7fef533f90972e41fe89986ae9ff" op-secure-params="  passwd  " op-secure-digest="1905d2bf96a4b1c41f38dff8c7b84308"/>
+          </lrm_resource>
+          <lrm_resource id="set-rsc1" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="set-rsc1_last_0" operation_key="set-rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="13:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;13:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-2" call-id="25" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="28" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="set-rsc2" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="set-rsc2_last_0" operation_key="set-rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="14:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;14:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-2" call-id="29" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="18" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+    <node_state id="5" uname="rhel7-5" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="5">
+        <lrm_resources>
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="29:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;29:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-5" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="1" queue-time="0" op-digest="422e90c96b7732222706b322138f43c8"/>
+          </lrm_resource>
+          <lrm_resource id="meta-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="meta-rsc_last_0" operation_key="meta-rsc_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.6.3" transition-key="17:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;17:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-5" call-id="72" rc-code="0" op-status="0" interval="0" last-rc-change="1605216481" last-run="1605216481" exec-time="78" queue-time="0" op-digest="4269017c9755b64d3c5687024693ab8d" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="4269017c9755b64d3c5687024693ab8d"/>
+            <lrm_rsc_op id="meta-rsc_monitor_10000" operation_key="meta-rsc_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="25:27:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;25:27:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-5" call-id="62" rc-code="0" op-status="0" interval="10000" last-rc-change="1605215447" exec-time="20" queue-time="0" op-digest="ce835727a24a9c27be80f53be9716835" op-secure-params="  passwd  " op-secure-digest="4269017c9755b64d3c5687024693ab8d"/>
+          </lrm_resource>
+          <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="30:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;30:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-5" call-id="9" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="27" queue-time="0" op-digest="740d07f67c20fca411b03996e601119e" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="740d07f67c20fca411b03996e601119e"/>
+          </lrm_resource>
+          <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.6.3" transition-key="12:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;12:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-5" call-id="69" rc-code="0" op-status="0" interval="0" last-rc-change="1605216481" last-run="1605216481" exec-time="85" queue-time="0" op-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84"/>
+            <lrm_rsc_op id="rsc2_monitor_10000" operation_key="rsc2_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="14:18:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;14:18:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-5" call-id="50" rc-code="0" op-status="0" interval="10000" last-rc-change="1605214609" exec-time="19" queue-time="0" op-digest="86322d62158bad207944738f0a1bac13" op-secure-params="  passwd  " op-secure-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84"/>
+          </lrm_resource>
+          <lrm_resource id="invert-match" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="invert-match_last_0" operation_key="invert-match_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="32:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;32:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-5" call-id="17" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="29" queue-time="0" op-digest="1bd01c2c51e0e8a776ac0e84d5715d70" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1bd01c2c51e0e8a776ac0e84d5715d70"/>
+          </lrm_resource>
+          <lrm_resource id="single-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="single-rsc_last_0" operation_key="single-rsc_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="33:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;33:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-5" call-id="21" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="28" queue-time="0" op-digest="1bd01c2c51e0e8a776ac0e84d5715d70" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1bd01c2c51e0e8a776ac0e84d5715d70"/>
+          </lrm_resource>
+          <lrm_resource id="insane-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="insane-rsc_last_0" operation_key="insane-rsc_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="13:29:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;13:29:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-5" call-id="67" rc-code="7" op-status="0" interval="0" last-rc-change="1605216014" last-run="1605216014" exec-time="32" queue-time="0" op-digest="1905d2bf96a4b1c41f38dff8c7b84308" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1905d2bf96a4b1c41f38dff8c7b84308"/>
+          </lrm_resource>
+          <lrm_resource id="set-rsc1" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="set-rsc1_last_0" operation_key="set-rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="34:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;34:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-5" call-id="25" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="35" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="set-rsc2" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="set-rsc2_last_0" operation_key="set-rsc2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="20:19:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;20:19:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-5" call-id="53" rc-code="0" op-status="0" interval="0" last-rc-change="1605214850" last-run="1605214850" exec-time="22" queue-time="0" op-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+    <node_state id="3" uname="rhel7-3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="3">
+        <lrm_resources>
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="15:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;15:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-3" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="9" queue-time="0" op-digest="422e90c96b7732222706b322138f43c8"/>
+          </lrm_resource>
+          <lrm_resource id="meta-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="meta-rsc_last_0" operation_key="meta-rsc_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="10:23:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;10:23:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-3" call-id="51" rc-code="7" op-status="0" interval="0" last-rc-change="1605215239" last-run="1605215239" exec-time="68" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="16:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;16:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-3" call-id="9" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="32" queue-time="0" op-digest="740d07f67c20fca411b03996e601119e" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="740d07f67c20fca411b03996e601119e"/>
+          </lrm_resource>
+          <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="17:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;17:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-3" call-id="13" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="20" queue-time="0" op-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84"/>
+          </lrm_resource>
+          <lrm_resource id="invert-match" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="invert-match_last_0" operation_key="invert-match_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="18:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;18:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-3" call-id="17" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="33" queue-time="0" op-digest="1bd01c2c51e0e8a776ac0e84d5715d70" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1bd01c2c51e0e8a776ac0e84d5715d70"/>
+          </lrm_resource>
+          <lrm_resource id="single-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="single-rsc_last_0" operation_key="single-rsc_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="19:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;19:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-3" call-id="21" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="31" queue-time="0" op-digest="1bd01c2c51e0e8a776ac0e84d5715d70" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1bd01c2c51e0e8a776ac0e84d5715d70"/>
+          </lrm_resource>
+          <lrm_resource id="insane-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="insane-rsc_last_0" operation_key="insane-rsc_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="11:29:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;11:29:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-3" call-id="55" rc-code="7" op-status="0" interval="0" last-rc-change="1605216014" last-run="1605216014" exec-time="46" queue-time="0" op-digest="1905d2bf96a4b1c41f38dff8c7b84308" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1905d2bf96a4b1c41f38dff8c7b84308"/>
+          </lrm_resource>
+          <lrm_resource id="set-rsc1" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="set-rsc1_last_0" operation_key="set-rsc1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.6.3" transition-key="15:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;15:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-3" call-id="57" rc-code="0" op-status="0" interval="0" last-rc-change="1605216481" last-run="1605216481" exec-time="110" queue-time="0" op-digest="740d07f67c20fca411b03996e601119e" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="740d07f67c20fca411b03996e601119e"/>
+            <lrm_rsc_op id="set-rsc1_monitor_10000" operation_key="set-rsc1_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="12:15:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;12:15:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-3" call-id="46" rc-code="0" op-status="0" interval="10000" last-rc-change="1605212720" exec-time="34" queue-time="0" op-digest="72d8b1ceea915fac5b98d7c16b3cb81a" op-secure-params="  passwd  " op-secure-digest="740d07f67c20fca411b03996e601119e"/>
+          </lrm_resource>
+          <lrm_resource id="set-rsc2" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="set-rsc2_last_0" operation_key="set-rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="21:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;21:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-3" call-id="29" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+    <node_state id="4" uname="rhel7-4" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="4">
+        <lrm_resources>
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="22:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;22:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-4" call-id="10" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="7" queue-time="0" op-digest="422e90c96b7732222706b322138f43c8"/>
+          </lrm_resource>
+          <lrm_resource id="meta-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="meta-rsc_last_0" operation_key="meta-rsc_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="11:23:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;11:23:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-4" call-id="70" rc-code="7" op-status="0" interval="0" last-rc-change="1605215239" last-run="1605215239" exec-time="36" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.6.3" transition-key="11:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;11:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-4" call-id="80" rc-code="0" op-status="0" interval="0" last-rc-change="1605216481" last-run="1605216481" exec-time="77" queue-time="0" op-digest="740d07f67c20fca411b03996e601119e" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="740d07f67c20fca411b03996e601119e"/>
+            <lrm_rsc_op id="rsc1_monitor_10000" operation_key="rsc1_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="4:15:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;4:15:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-4" call-id="54" rc-code="0" op-status="0" interval="10000" last-rc-change="1605212720" exec-time="56" queue-time="0" op-digest="72d8b1ceea915fac5b98d7c16b3cb81a" op-secure-params="  passwd  " op-secure-digest="740d07f67c20fca411b03996e601119e"/>
+          </lrm_resource>
+          <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="12:18:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;12:18:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-4" call-id="60" rc-code="0" op-status="0" interval="0" last-rc-change="1605214609" last-run="1605214609" exec-time="29" queue-time="0" op-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84"/>
+          </lrm_resource>
+          <lrm_resource id="invert-match" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="invert-match_last_0" operation_key="invert-match_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="25:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;25:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-4" call-id="22" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="69" queue-time="0" op-digest="1bd01c2c51e0e8a776ac0e84d5715d70" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1bd01c2c51e0e8a776ac0e84d5715d70"/>
+          </lrm_resource>
+          <lrm_resource id="single-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="single-rsc_last_0" operation_key="single-rsc_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="26:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;26:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-4" call-id="26" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="39" queue-time="0" op-digest="1bd01c2c51e0e8a776ac0e84d5715d70" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="1bd01c2c51e0e8a776ac0e84d5715d70"/>
+          </lrm_resource>
+          <lrm_resource id="insane-rsc" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="insane-rsc_last_0" operation_key="insane-rsc_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.6.3" transition-key="18:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;18:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-4" call-id="88" rc-code="0" op-status="0" interval="0" last-rc-change="1605216481" last-run="1605216481" exec-time="39" queue-time="0" op-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84"/>
+            <lrm_rsc_op id="insane-rsc_monitor_10000" operation_key="insane-rsc_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.6.3" transition-key="28:32:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;28:32:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-4" call-id="77" rc-code="0" op-status="0" interval="10000" last-rc-change="1605216258" exec-time="21" queue-time="0" op-digest="86322d62158bad207944738f0a1bac13" op-secure-params="  passwd  " op-secure-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84"/>
+          </lrm_resource>
+          <lrm_resource id="set-rsc1" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="set-rsc1_last_0" operation_key="set-rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="27:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:7;27:0:7:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-4" call-id="30" rc-code="7" op-status="0" interval="0" last-rc-change="1605205770" last-run="1605205770" exec-time="28" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="set-rsc2" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="set-rsc2_last_0" operation_key="set-rsc2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.6.3" transition-key="16:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;16:33:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-4" call-id="83" rc-code="0" op-status="0" interval="0" last-rc-change="1605216481" last-run="1605216481" exec-time="71" queue-time="0" op-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84" op-force-restart="  envfile op_sleep passwd state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params="  passwd  " op-secure-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84"/>
+            <lrm_rsc_op id="set-rsc2_monitor_10000" operation_key="set-rsc2_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.6.3" transition-key="22:19:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" transition-magic="0:0;22:19:0:2b51cdb9-fb8b-43dd-905e-e7c0472fdaca" exit-reason="" on_node="rhel7-4" call-id="65" rc-code="0" op-status="0" interval="10000" last-rc-change="1605214850" exec-time="19" queue-time="0" op-digest="86322d62158bad207944738f0a1bac13" op-secure-params="  passwd  " op-secure-digest="e4a0cc96ccaaaa02bea1e0be9c4dee84"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+  </status>
+</cib>
-- 
1.8.3.1


From f678813f9054e2eb80a04796b8d35214269cd352 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Mon, 9 Nov 2020 18:13:04 -0600
Subject: [PATCH 11/13] API: libpe_status: deprecate pe_resource_t parameters
 member

Instead, pe_rsc_params() should be used.

Also, define the parameters member using such a pe_rsc_params() call.
---
 include/crm/pengine/pe_types.h |  2 +-
 lib/pengine/complex.c          | 17 ++++++++++-------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/include/crm/pengine/pe_types.h b/include/crm/pengine/pe_types.h
index 5529714..1416cee 100644
--- a/include/crm/pengine/pe_types.h
+++ b/include/crm/pengine/pe_types.h
@@ -358,7 +358,7 @@ struct pe_resource_s {
     enum rsc_role_e next_role;
 
     GHashTable *meta;
-    GHashTable *parameters;
+    GHashTable *parameters; //! \deprecated Use pe_rsc_params() instead
     GHashTable *utilization;
 
     GListPtr children;          /* pe_resource_t*   */
diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
index 7037ca1..60199c7 100644
--- a/lib/pengine/complex.c
+++ b/lib/pengine/complex.c
@@ -557,8 +557,6 @@ common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc,
         return FALSE;
     }
 
-    (*rsc)->parameters = crm_str_table_new();
-
 #if ENABLE_VERSIONED_ATTRS
     (*rsc)->versioned_parameters = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS);
 #endif
@@ -584,7 +582,7 @@ common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc,
     pe_rsc_trace((*rsc), "Unpacking resource...");
 
     get_meta_attributes((*rsc)->meta, *rsc, NULL, data_set);
-    get_rsc_attributes((*rsc)->parameters, *rsc, NULL, data_set);
+    (*rsc)->parameters = pe_rsc_params(*rsc, NULL, data_set); // \deprecated
 #if ENABLE_VERSIONED_ATTRS
     pe_get_versioned_attributes((*rsc)->versioned_parameters, *rsc, NULL, data_set);
 #endif
@@ -808,7 +806,15 @@ common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc,
     }
 
     if (remote_node) {
-        value = g_hash_table_lookup((*rsc)->parameters, XML_REMOTE_ATTR_RECONNECT_INTERVAL);
+        GHashTable *params = pe_rsc_params(*rsc, NULL, data_set);
+
+        /* Grabbing the value now means that any rules based on node attributes
+         * will evaluate to false, so such rules should not be used with
+         * reconnect_interval.
+         *
+         * @TODO Evaluate per node before using
+         */
+        value = g_hash_table_lookup(params, XML_REMOTE_ATTR_RECONNECT_INTERVAL);
         if (value) {
             /* reconnect delay works by setting failure_timeout and preventing the
              * connection from starting until the failure is cleared. */
@@ -922,9 +928,6 @@ common_free(pe_resource_t * rsc)
     g_list_free(rsc->rsc_tickets);
     g_list_free(rsc->dangling_migrations);
 
-    if (rsc->parameters != NULL) {
-        g_hash_table_destroy(rsc->parameters);
-    }
     if (rsc->parameter_cache != NULL) {
         g_hash_table_destroy(rsc->parameter_cache);
     }
-- 
1.8.3.1


From 267d0cc44e94d6963fe13974f83b2fc845b0c431 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 17 Nov 2020 15:45:35 -0600
Subject: [PATCH 12/13] Test: execd: use new resource parameter function in
 cts-exec-helper

---
 daemons/execd/cts-exec-helper.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/daemons/execd/cts-exec-helper.c b/daemons/execd/cts-exec-helper.c
index 8be8e18..423b54d 100644
--- a/daemons/execd/cts-exec-helper.c
+++ b/daemons/execd/cts-exec-helper.c
@@ -482,13 +482,12 @@ generate_params(void)
         goto param_gen_bail;
     }
 
-    params = crm_str_table_new();
+    params = pe_rsc_params(rsc, NULL, data_set);
     meta = crm_str_table_new();
 
-    get_rsc_attributes(params, rsc, NULL, data_set);
     get_meta_attributes(meta, rsc, NULL, data_set);
 
-    if (params) {
+    if (params != NULL) {
         char *key = NULL;
         char *value = NULL;
 
@@ -496,7 +495,6 @@ generate_params(void)
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             options.params = lrmd_key_value_add(options.params, key, value);
         }
-        g_hash_table_destroy(params);
     }
 
     if (meta) {
-- 
1.8.3.1


From b10c86bf785beccef147b702366607cc4ead0f79 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 17 Nov 2020 16:01:12 -0600
Subject: [PATCH 13/13] Refactor: tools: use new resource parameter function in
 crm_resource

---
 tools/crm_resource.c         | 14 +++++++++-----
 tools/crm_resource_print.c   | 10 ++++++++--
 tools/crm_resource_runtime.c | 32 +++++++++++---------------------
 3 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index b028c40..9663e68 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1902,6 +1902,7 @@ main(int argc, char **argv)
             unsigned int count = 0;
             GHashTable *params = NULL;
             pe_node_t *current = pe__find_active_on(rsc, &count, NULL);
+            bool free_params = true;
 
             if (count > 1) {
                 out->err(out, "%s is active on more than one node,"
@@ -1909,23 +1910,26 @@ main(int argc, char **argv)
                 current = NULL;
             }
 
-            params = crm_str_table_new();
+            crm_debug("Looking up %s in %s", options.prop_name, rsc->id);
 
             if (pcmk__str_eq(options.attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
-                get_rsc_attributes(params, rsc, current, data_set);
+                params = pe_rsc_params(rsc, current, data_set);
+                free_params = false;
 
             } else if (pcmk__str_eq(options.attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
-                /* No need to redirect to the parent */
+                params = crm_str_table_new();
                 get_meta_attributes(params, rsc, current, data_set);
 
             } else {
+                params = crm_str_table_new();
                 pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_UTILIZATION, NULL, params,
                                            NULL, FALSE, data_set);
             }
 
-            crm_debug("Looking up %s in %s", options.prop_name, rsc->id);
             rc = out->message(out, "attribute-list", rsc, options.prop_name, params);
-            g_hash_table_destroy(params);
+            if (free_params) {
+                g_hash_table_destroy(params);
+            }
             break;
         }
 
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
index 89d6172..398fef0 100644
--- a/tools/crm_resource_print.c
+++ b/tools/crm_resource_print.c
@@ -134,8 +134,11 @@ attribute_list_default(pcmk__output_t *out, va_list args) {
     char *attr = va_arg(args, char *);
     GHashTable *params = va_arg(args, GHashTable *);
 
-    const char *value = g_hash_table_lookup(params, attr);
+    const char *value = NULL;
 
+    if (params != NULL) {
+        value = g_hash_table_lookup(params, attr);
+    }
     if (value != NULL) {
         out->begin_list(out, NULL, NULL, "Attributes");
         out->list_item(out, attr, "%s", value);
@@ -154,8 +157,11 @@ attribute_list_text(pcmk__output_t *out, va_list args) {
     char *attr = va_arg(args, char *);
     GHashTable *params = va_arg(args, GHashTable *);
 
-    const char *value = g_hash_table_lookup(params, attr);
+    const char *value = NULL;
 
+    if (params != NULL) {
+        value = g_hash_table_lookup(params, attr);
+    }
     if (value != NULL) {
         out->info(out, "%s", value);
     } else {
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index e0804fc..9ff9e96 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -916,40 +916,29 @@ cli_resource_fail(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
 }
 
 static GHashTable *
-generate_resource_params(pe_resource_t * rsc, pe_working_set_t * data_set)
+generate_resource_params(pe_resource_t *rsc, pe_node_t *node,
+                         pe_working_set_t *data_set)
 {
     GHashTable *params = NULL;
     GHashTable *meta = NULL;
     GHashTable *combined = NULL;
     GHashTableIter iter;
+    char *key = NULL;
+    char *value = NULL;
 
-    if (!rsc) {
-        crm_err("Resource does not exist in config");
-        return NULL;
-    }
-
-    params = crm_str_table_new();
-    meta = crm_str_table_new();
     combined = crm_str_table_new();
 
-    get_rsc_attributes(params, rsc, NULL /* TODO: Pass in local node */ , data_set);
-    get_meta_attributes(meta, rsc, NULL /* TODO: Pass in local node */ , data_set);
-
-    if (params) {
-        char *key = NULL;
-        char *value = NULL;
-
+    params = pe_rsc_params(rsc, node, data_set);
+    if (params != NULL) {
         g_hash_table_iter_init(&iter, params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             g_hash_table_insert(combined, strdup(key), strdup(value));
         }
-        g_hash_table_destroy(params);
     }
 
-    if (meta) {
-        char *key = NULL;
-        char *value = NULL;
-
+    meta = crm_str_table_new();
+    get_meta_attributes(meta, rsc, node, data_set);
+    if (meta != NULL) {
         g_hash_table_iter_init(&iter, meta);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             char *crm_name = crm_meta_name(key);
@@ -1827,7 +1816,8 @@ cli_resource_execute(pcmk__output_t *out, pe_resource_t *rsc,
     rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
     rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
 
-    params = generate_resource_params(rsc, data_set);
+    params = generate_resource_params(rsc, NULL /* @TODO use local node */,
+                                      data_set);
 
     if (timeout_ms == 0) {
         timeout_ms = pe_get_configured_timeout(rsc, action, data_set);
-- 
1.8.3.1