Blob Blame History Raw
From 08d3e77237eb3b9f4600b4a8a4c5153e928ca3ce Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Fri, 7 Aug 2020 13:07:50 -0400
Subject: [PATCH 01/19] Feature: tools: Add the beginnings of formatted output
 to crm_resource.

This just adds the command line options, the output object, hooks it
up for use if --version is given, and uses it for printing error
messages.
---
 tools/crm_resource.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 47 insertions(+), 6 deletions(-)

diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index acaddc0..9700618 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -154,6 +154,7 @@ gboolean why_cb(const gchar *option_name, const gchar *optarg, gpointer data, GE
 
 bool BE_QUIET = FALSE;
 static crm_exit_t exit_code = CRM_EX_OK;
+static pcmk__output_t *out = NULL;
 
 // Things that should be cleaned up on exit
 static GError *error = NULL;
@@ -166,15 +167,32 @@ static pe_working_set_t *data_set = NULL;
 
 #define INDENT "                                    "
 
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
 // Clean up and exit
 static crm_exit_t
 bye(crm_exit_t ec)
 {
     if (error != NULL) {
-        fprintf(stderr, "%s: %s\n", g_get_prgname(), error->message);
+        if (out != NULL) {
+            out->err(out, "%s: %s", g_get_prgname(), error->message);
+        } else {
+            fprintf(stderr, "%s: %s\n", g_get_prgname(), error->message);
+        }
+
         g_clear_error(&error);
     }
 
+    if (out != NULL) {
+        out->finish(out, ec, true, NULL);
+        pcmk__output_free(out);
+    }
+
     if (cib_conn != NULL) {
         cib_t *save_cib_conn = cib_conn;
 
@@ -1428,7 +1446,7 @@ validate_cmdline(crm_exit_t *exit_code)
 }
 
 static GOptionContext *
-build_arg_context(pcmk__common_args_t *args) {
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
     GOptionContext *context = NULL;
 
     GOptionEntry extra_prog_entries[] = {
@@ -1471,7 +1489,7 @@ build_arg_context(pcmk__common_args_t *args) {
                               "had failed permanently and has been repaired by an administrator):\n\n"
                               "\t# crm_resource --resource myResource --cleanup --node aNode\n\n";
 
-    context = pcmk__build_arg_context(args, NULL, NULL, NULL);
+    context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
     g_option_context_set_description(context, description);
 
     /* Add the -Q option, which cannot be part of the globally supported options
@@ -1504,9 +1522,11 @@ main(int argc, char **argv)
 
     pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
     GOptionContext *context = NULL;
+    GOptionGroup *output_group = NULL;
     gchar **processed_args = NULL;
 
-    context = build_arg_context(args);
+    context = build_arg_context(args, &output_group);
+    pcmk__register_formats(output_group, formats);
     crm_log_cli_init("crm_resource");
 
     processed_args = pcmk__cmdline_preproc(argv, "GINSTdginpstuv");
@@ -1520,6 +1540,14 @@ main(int argc, char **argv)
         crm_bump_log_level(argc, argv);
     }
 
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        fprintf(stderr, "Error creating output format %s: %s\n",
+                args->output_ty, pcmk_rc_str(rc));
+        exit_code = CRM_EX_ERROR;
+        goto done;
+    }
+
     options.resource_verbose = args->verbosity;
     BE_QUIET = args->quiet;
 
@@ -1593,9 +1621,22 @@ main(int argc, char **argv)
         goto done;
     }
 
+    if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
+        /* Kind of a hack to display XML lists using a real tag instead of <list>.  This just
+         * saves from having to write custom messages to build the lists around all these things
+         */
+        if (options.rsc_cmd == cmd_list_resources || options.rsc_cmd == cmd_query_xml ||
+            options.rsc_cmd == cmd_query_raw_xml || options.rsc_cmd == cmd_list_active_ops ||
+            options.rsc_cmd == cmd_list_all_ops) {
+            pcmk__force_args(context, &error, "%s --xml-simple-list --xml-substitute", g_get_prgname());
+        } else {
+            pcmk__force_args(context, &error, "%s --xml-substitute", g_get_prgname());
+        }
+    }
+
     if (args->version) {
-        /* FIXME:  When crm_resource is converted to use formatted output, this can go. */
-        pcmk__cli_help('v', CRM_EX_USAGE);
+        out->version(out, false);
+        goto done;
     }
 
     if (optind > argc) {
-- 
1.8.3.1


From 38f09e048e662c0e1814cbde6ecee633cc9560d5 Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Wed, 26 Aug 2020 16:32:41 -0400
Subject: [PATCH 02/19] Refactor: tools: Pass a pcmk__output_t object around
 crm_resource.

Basically, any function that does any printing is eventually going to
need this object.  Adding it all at once here should make for more easy
to understand patches later.
---
 tools/crm_resource.c         | 109 +++++++++---------
 tools/crm_resource.h         | 110 ++++++++++--------
 tools/crm_resource_ban.c     |  19 ++--
 tools/crm_resource_print.c   |  38 ++++---
 tools/crm_resource_runtime.c | 266 +++++++++++++++++++++++--------------------
 5 files changed, 288 insertions(+), 254 deletions(-)

diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index 9700618..7a661a4 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -918,7 +918,8 @@ why_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **er
 }
 
 static int
-ban_or_move(pe_resource_t *rsc, const char *move_lifetime, crm_exit_t *exit_code)
+ban_or_move(pcmk__output_t *out, pe_resource_t *rsc, const char *move_lifetime,
+            crm_exit_t *exit_code)
 {
     int rc = pcmk_rc_ok;
     pe_node_t *current = NULL;
@@ -929,7 +930,7 @@ ban_or_move(pe_resource_t *rsc, const char *move_lifetime, crm_exit_t *exit_code
     current = pe__find_active_requires(rsc, &nactive);
 
     if (nactive == 1) {
-        rc = cli_resource_ban(options.rsc_id, current->details->uname, move_lifetime, NULL,
+        rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime, NULL,
                               cib_conn, options.cib_options, options.promoted_role_only);
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
@@ -948,7 +949,7 @@ ban_or_move(pe_resource_t *rsc, const char *move_lifetime, crm_exit_t *exit_code
         }
 
         if(count == 1 && current) {
-            rc = cli_resource_ban(options.rsc_id, current->details->uname, move_lifetime, NULL,
+            rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime, NULL,
                                   cib_conn, options.cib_options, options.promoted_role_only);
 
         } else {
@@ -977,7 +978,7 @@ ban_or_move(pe_resource_t *rsc, const char *move_lifetime, crm_exit_t *exit_code
 }
 
 static void
-cleanup(pe_resource_t *rsc)
+cleanup(pcmk__output_t *out, pe_resource_t *rsc)
 {
     int rc = pcmk_rc_ok;
 
@@ -987,12 +988,12 @@ cleanup(pe_resource_t *rsc)
 
     crm_debug("Erasing failures of %s (%s requested) on %s",
               rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
-    rc = cli_resource_delete(controld_api, options.host_uname, rsc, options.operation,
+    rc = cli_resource_delete(out, controld_api, options.host_uname, rsc, options.operation,
                              options.interval_spec, TRUE, data_set, options.force);
 
     if ((rc == pcmk_rc_ok) && !BE_QUIET) {
         // Show any reasons why resource might stay stopped
-        cli_resource_check(cib_conn, rsc);
+        cli_resource_check(out, cib_conn, rsc);
     }
 
     if (rc == pcmk_rc_ok) {
@@ -1001,7 +1002,7 @@ cleanup(pe_resource_t *rsc)
 }
 
 static int
-clear_constraints(xmlNodePtr *cib_xml_copy)
+clear_constraints(pcmk__output_t *out, xmlNodePtr *cib_xml_copy)
 {
     GListPtr before = NULL;
     GListPtr after = NULL;
@@ -1089,7 +1090,7 @@ delete()
 }
 
 static int
-list_agents(const char *agent_spec, crm_exit_t *exit_code)
+list_agents(pcmk__output_t *out, const char *agent_spec, crm_exit_t *exit_code)
 {
     int rc = pcmk_rc_ok;
     lrmd_list_t *list = NULL;
@@ -1126,7 +1127,7 @@ list_agents(const char *agent_spec, crm_exit_t *exit_code)
 }
 
 static int
-list_providers(const char *agent_spec, crm_exit_t *exit_code)
+list_providers(pcmk__output_t *out, const char *agent_spec, crm_exit_t *exit_code)
 {
     int rc;
     const char *text = NULL;
@@ -1177,7 +1178,7 @@ list_providers(const char *agent_spec, crm_exit_t *exit_code)
 }
 
 static int
-list_raw()
+list_raw(pcmk__output_t *out)
 {
     int rc = pcmk_rc_ok;
     int found = 0;
@@ -1187,7 +1188,7 @@ list_raw()
         pe_resource_t *rsc = (pe_resource_t *) lpc->data;
 
         found++;
-        cli_resource_print_raw(rsc);
+        cli_resource_print_raw(out, rsc);
     }
 
     if (found == 0) {
@@ -1199,7 +1200,7 @@ list_raw()
 }
 
 static void
-list_stacks_and_constraints(pe_resource_t *rsc, bool recursive)
+list_stacks_and_constraints(pcmk__output_t *out, pe_resource_t *rsc, bool recursive)
 {
     GListPtr lpc = NULL;
     xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS,
@@ -1216,10 +1217,10 @@ list_stacks_and_constraints(pe_resource_t *rsc, bool recursive)
         pe__clear_resource_flags(r, pe_rsc_allocating);
     }
 
-    cli_resource_print_colocation(rsc, TRUE, recursive, 1);
+    cli_resource_print_colocation(out, rsc, TRUE, recursive, 1);
 
     fprintf(stdout, "* %s\n", rsc->id);
-    cli_resource_print_location(rsc, NULL);
+    cli_resource_print_location(out, rsc, NULL);
 
     for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
         pe_resource_t *r = (pe_resource_t *) lpc->data;
@@ -1227,7 +1228,7 @@ list_stacks_and_constraints(pe_resource_t *rsc, bool recursive)
         pe__clear_resource_flags(r, pe_rsc_allocating);
     }
 
-    cli_resource_print_colocation(rsc, FALSE, recursive, 1);
+    cli_resource_print_colocation(out, rsc, FALSE, recursive, 1);
 }
 
 static int
@@ -1262,7 +1263,7 @@ populate_working_set(xmlNodePtr *cib_xml_copy)
 }
 
 static int
-refresh()
+refresh(pcmk__output_t *out)
 {
     int rc = pcmk_rc_ok;
     const char *router_node = options.host_uname;
@@ -1307,7 +1308,7 @@ refresh()
 }
 
 static void
-refresh_resource(pe_resource_t *rsc)
+refresh_resource(pcmk__output_t *out, pe_resource_t *rsc)
 {
     int rc = pcmk_rc_ok;
 
@@ -1317,12 +1318,12 @@ refresh_resource(pe_resource_t *rsc)
 
     crm_debug("Re-checking the state of %s (%s requested) on %s",
               rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
-    rc = cli_resource_delete(controld_api, options.host_uname, rsc, NULL, 0, FALSE,
-                             data_set, options.force);
+    rc = cli_resource_delete(out, controld_api, options.host_uname, rsc, NULL,
+                             0, FALSE, data_set, options.force);
 
     if ((rc == pcmk_rc_ok) && !BE_QUIET) {
         // Show any reasons why resource might stay stopped
-        cli_resource_check(cib_conn, rsc);
+        cli_resource_check(out, cib_conn, rsc);
     }
 
     if (rc == pcmk_rc_ok) {
@@ -1364,7 +1365,7 @@ set_property()
 }
 
 static int
-show_metadata(const char *agent_spec, crm_exit_t *exit_code)
+show_metadata(pcmk__output_t *out, const char *agent_spec, crm_exit_t *exit_code)
 {
     int rc = pcmk_rc_ok;
     char *standard = NULL;
@@ -1438,7 +1439,7 @@ validate_cmdline(crm_exit_t *exit_code)
         if (options.validate_options == NULL) {
             options.validate_options = crm_str_table_new();
         }
-        *exit_code = cli_resource_execute_from_params("test", options.v_class, options.v_provider, options.v_agent,
+        *exit_code = cli_resource_execute_from_params(out, "test", options.v_class, options.v_provider, options.v_agent,
                                                       "validate-all", options.validate_options,
                                                       options.override_params, options.timeout_ms,
                                                       options.resource_verbose, options.force);
@@ -1737,25 +1738,25 @@ main(int argc, char **argv)
     switch (options.rsc_cmd) {
         case cmd_list_resources:
             rc = pcmk_rc_ok;
-            cli_resource_print_list(data_set, FALSE);
+            cli_resource_print_list(out, data_set, FALSE);
             break;
 
         case cmd_list_instances:
-            rc = list_raw();
+            rc = list_raw(out);
             break;
 
         case cmd_list_standards:
         case cmd_list_providers:
         case cmd_list_alternatives:
-            rc = list_providers(options.agent_spec, &exit_code);
+            rc = list_providers(out, options.agent_spec, &exit_code);
             break;
 
         case cmd_list_agents:
-            rc = list_agents(options.agent_spec, &exit_code);
+            rc = list_agents(out, options.agent_spec, &exit_code);
             break;
 
         case cmd_metadata:
-            rc = show_metadata(options.agent_spec, &exit_code);
+            rc = show_metadata(out, options.agent_spec, &exit_code);
             break;
 
         case cmd_restart:
@@ -1764,7 +1765,7 @@ main(int argc, char **argv)
              * update the working set multiple times, so it needs to use its own
              * copy.
              */
-            rc = cli_resource_restart(rsc, options.host_uname,
+            rc = cli_resource_restart(out, rsc, options.host_uname,
                                       options.move_lifetime, options.timeout_ms,
                                       cib_conn, options.cib_options,
                                       options.promoted_role_only,
@@ -1772,11 +1773,11 @@ main(int argc, char **argv)
             break;
 
         case cmd_wait:
-            rc = wait_till_stable(options.timeout_ms, cib_conn);
+            rc = wait_till_stable(out, options.timeout_ms, cib_conn);
             break;
 
         case cmd_execute_agent:
-            exit_code = cli_resource_execute(rsc, options.rsc_id,
+            exit_code = cli_resource_execute(out, rsc, options.rsc_id,
                                              options.operation,
                                              options.override_params,
                                              options.timeout_ms, cib_conn,
@@ -1785,11 +1786,11 @@ main(int argc, char **argv)
             break;
 
         case cmd_colocations:
-            list_stacks_and_constraints(rsc, false);
+            list_stacks_and_constraints(out, rsc, false);
             break;
 
         case cmd_colocations_deep:
-            list_stacks_and_constraints(rsc, true);
+            list_stacks_and_constraints(out, rsc, true);
             break;
 
         case cmd_cts:
@@ -1798,13 +1799,13 @@ main(int argc, char **argv)
                  lpc = lpc->next) {
 
                 rsc = (pe_resource_t *) lpc->data;
-                cli_resource_print_cts(rsc);
+                cli_resource_print_cts(out, rsc);
             }
-            cli_resource_print_cts_constraints(data_set);
+            cli_resource_print_cts_constraints(out, data_set);
             break;
 
         case cmd_fail:
-            rc = cli_resource_fail(controld_api, options.host_uname,
+            rc = cli_resource_fail(out, controld_api, options.host_uname,
                                    options.rsc_id, data_set);
             if (rc == pcmk_rc_ok) {
                 start_mainloop(controld_api);
@@ -1812,28 +1813,28 @@ main(int argc, char **argv)
             break;
 
         case cmd_list_active_ops:
-            rc = cli_resource_print_operations(options.rsc_id,
+            rc = cli_resource_print_operations(out, options.rsc_id,
                                                options.host_uname, TRUE,
                                                data_set);
             break;
 
         case cmd_list_all_ops:
-            rc = cli_resource_print_operations(options.rsc_id,
+            rc = cli_resource_print_operations(out, options.rsc_id,
                                                options.host_uname, FALSE,
                                                data_set);
             break;
 
         case cmd_locate:
-            cli_resource_search(rsc, options.rsc_id, data_set);
+            cli_resource_search(out, rsc, options.rsc_id, data_set);
             rc = pcmk_rc_ok;
             break;
 
         case cmd_query_xml:
-            rc = cli_resource_print(rsc, data_set, TRUE);
+            rc = cli_resource_print(out, rsc, data_set, TRUE);
             break;
 
         case cmd_query_raw_xml:
-            rc = cli_resource_print(rsc, data_set, FALSE);
+            rc = cli_resource_print(out, rsc, data_set, FALSE);
             break;
 
         case cmd_why:
@@ -1847,20 +1848,20 @@ main(int argc, char **argv)
                         goto done;
                     }
                 }
-                cli_resource_why(cib_conn, data_set->resources, rsc, dest);
+                cli_resource_why(out, cib_conn, data_set->resources, rsc, dest);
                 rc = pcmk_rc_ok;
             }
             break;
 
         case cmd_clear:
-            rc = clear_constraints(&cib_xml_copy);
+            rc = clear_constraints(out, &cib_xml_copy);
             break;
 
         case cmd_move:
             if (options.host_uname == NULL) {
-                rc = ban_or_move(rsc, options.move_lifetime, &exit_code);
+                rc = ban_or_move(out, rsc, options.move_lifetime, &exit_code);
             } else {
-                rc = cli_resource_move(rsc, options.rsc_id, options.host_uname,
+                rc = cli_resource_move(out, rsc, options.rsc_id, options.host_uname,
                                        options.move_lifetime, cib_conn,
                                        options.cib_options, data_set,
                                        options.promoted_role_only,
@@ -1870,7 +1871,7 @@ main(int argc, char **argv)
 
         case cmd_ban:
             if (options.host_uname == NULL) {
-                rc = ban_or_move(rsc, options.move_lifetime, &exit_code);
+                rc = ban_or_move(out, rsc, options.move_lifetime, &exit_code);
             } else {
                 pe_node_t *dest = pe_find_node(data_set->nodes,
                                                options.host_uname);
@@ -1879,7 +1880,7 @@ main(int argc, char **argv)
                     rc = pcmk_rc_node_unknown;
                     goto done;
                 }
-                rc = cli_resource_ban(options.rsc_id, dest->details->uname,
+                rc = cli_resource_ban(out, options.rsc_id, dest->details->uname,
                                       options.move_lifetime, NULL, cib_conn,
                                       options.cib_options,
                                       options.promoted_role_only);
@@ -1887,7 +1888,7 @@ main(int argc, char **argv)
             break;
 
         case cmd_get_property:
-            rc = cli_resource_print_property(rsc, options.prop_name, data_set);
+            rc = cli_resource_print_property(out, rsc, options.prop_name, data_set);
             break;
 
         case cmd_set_property:
@@ -1895,7 +1896,7 @@ main(int argc, char **argv)
             break;
 
         case cmd_get_param:
-            rc = cli_resource_print_attribute(rsc, options.prop_name,
+            rc = cli_resource_print_attribute(out, rsc, options.prop_name,
                                               options.attr_set_type, data_set);
             break;
 
@@ -1908,7 +1909,7 @@ main(int argc, char **argv)
             }
 
             /* coverity[var_deref_model] False positive */
-            rc = cli_resource_update_attribute(rsc, options.rsc_id,
+            rc = cli_resource_update_attribute(out, rsc, options.rsc_id,
                                                options.prop_set,
                                                options.attr_set_type,
                                                options.prop_id,
@@ -1921,7 +1922,7 @@ main(int argc, char **argv)
 
         case cmd_delete_param:
             /* coverity[var_deref_model] False positive */
-            rc = cli_resource_delete_attribute(rsc, options.rsc_id,
+            rc = cli_resource_delete_attribute(out, rsc, options.rsc_id,
                                                options.prop_set,
                                                options.attr_set_type,
                                                options.prop_id,
@@ -1932,22 +1933,22 @@ main(int argc, char **argv)
 
         case cmd_cleanup:
             if (rsc == NULL) {
-                rc = cli_cleanup_all(controld_api, options.host_uname,
+                rc = cli_cleanup_all(out, controld_api, options.host_uname,
                                      options.operation, options.interval_spec,
                                      data_set);
                 if (rc == pcmk_rc_ok) {
                     start_mainloop(controld_api);
                 }
             } else {
-                cleanup(rsc);
+                cleanup(out, rsc);
             }
             break;
 
         case cmd_refresh:
             if (rsc == NULL) {
-                rc = refresh();
+                rc = refresh(out);
             } else {
-                refresh_resource(rsc);
+                refresh_resource(out, rsc);
             }
             break;
 
diff --git a/tools/crm_resource.h b/tools/crm_resource.h
index e979ef8..bf99f24 100644
--- a/tools/crm_resource.h
+++ b/tools/crm_resource.h
@@ -25,75 +25,85 @@
 extern bool BE_QUIET;
 
 /* ban */
-int cli_resource_prefer(const char *rsc_id, const char *host, const char *move_lifetime,
-                        cib_t * cib_conn, int cib_options, gboolean promoted_role_only);
-int cli_resource_ban(const char *rsc_id, const char *host, const char *move_lifetime,
-                     GListPtr allnodes, cib_t * cib_conn, int cib_options,
-                     gboolean promoted_role_only);
+int cli_resource_prefer(pcmk__output_t *out, const char *rsc_id, const char *host,
+                        const char *move_lifetime, cib_t * cib_conn, int cib_options,
+                        gboolean promoted_role_only);
+int cli_resource_ban(pcmk__output_t *out, const char *rsc_id, const char *host,
+                     const char *move_lifetime, GListPtr allnodes, cib_t * cib_conn,
+                     int cib_options, gboolean promoted_role_only);
 int cli_resource_clear(const char *rsc_id, const char *host, GListPtr allnodes,
                        cib_t * cib_conn, int cib_options, bool clear_ban_constraints, gboolean force);
 int cli_resource_clear_all_expired(xmlNode *root, cib_t *cib_conn, int cib_options,
                                    const char *rsc, const char *node, gboolean promoted_role_only);
 
 /* print */
-void cli_resource_print_cts(pe_resource_t * rsc);
-void cli_resource_print_raw(pe_resource_t * rsc);
-void cli_resource_print_cts_constraints(pe_working_set_t * data_set);
-void cli_resource_print_location(pe_resource_t * rsc, const char *prefix);
-void cli_resource_print_colocation(pe_resource_t * rsc, bool dependents, bool recursive, int offset);
+void cli_resource_print_cts(pcmk__output_t *out, pe_resource_t * rsc);
+void cli_resource_print_raw(pcmk__output_t *out, pe_resource_t * rsc);
+void cli_resource_print_cts_constraints(pcmk__output_t *out, pe_working_set_t * data_set);
+void cli_resource_print_location(pcmk__output_t *out, pe_resource_t * rsc,
+                                 const char *prefix);
+void cli_resource_print_colocation(pcmk__output_t *out, pe_resource_t * rsc,
+                                   bool dependents, bool recursive, int offset);
 
-int cli_resource_print(pe_resource_t *rsc, pe_working_set_t *data_set,
+int cli_resource_print(pcmk__output_t *out, pe_resource_t *rsc, pe_working_set_t *data_set,
                        bool expanded);
-int cli_resource_print_list(pe_working_set_t * data_set, bool raw);
-int cli_resource_print_attribute(pe_resource_t *rsc, const char *attr, const char *attr_set_type,
+int cli_resource_print_list(pcmk__output_t *out, pe_working_set_t * data_set, bool raw);
+int cli_resource_print_attribute(pcmk__output_t *out, pe_resource_t *rsc,
+                                 const char *attr, const char *attr_set_type,
                                  pe_working_set_t *data_set);
-int cli_resource_print_property(pe_resource_t *rsc, const char *attr,
+int cli_resource_print_property(pcmk__output_t *out, pe_resource_t *rsc, const char *attr,
                                 pe_working_set_t *data_set);
-int cli_resource_print_operations(const char *rsc_id, const char *host_uname, bool active, pe_working_set_t * data_set);
+int cli_resource_print_operations(pcmk__output_t *out, const char *rsc_id,
+                                  const char *host_uname, bool active,
+                                  pe_working_set_t * data_set);
 
 /* runtime */
-void cli_resource_check(cib_t * cib, pe_resource_t *rsc);
-int cli_resource_fail(pcmk_ipc_api_t *controld_api,
+void cli_resource_check(pcmk__output_t *out, cib_t * cib, pe_resource_t *rsc);
+int cli_resource_fail(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
                       const char *host_uname, const char *rsc_id,
                       pe_working_set_t *data_set);
-int cli_resource_search(pe_resource_t *rsc, const char *requested_name,
-                        pe_working_set_t *data_set);
-int cli_resource_delete(pcmk_ipc_api_t *controld_api,
+int cli_resource_search(pcmk__output_t *out, pe_resource_t *rsc,
+                        const char *requested_name, pe_working_set_t *data_set);
+int cli_resource_delete(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
                         const char *host_uname, pe_resource_t *rsc,
                         const char *operation, const char *interval_spec,
                         bool just_failures, pe_working_set_t *data_set,
                         gboolean force);
-int cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
-                    const char *operation, const char *interval_spec,
-                    pe_working_set_t *data_set);
-int cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_lifetime,
-                         int timeout_ms, cib_t *cib, int cib_options,
-                         gboolean promoted_role_only, gboolean force);
-int cli_resource_move(pe_resource_t *rsc, const char *rsc_id,
-                      const char *host_name, const char *move_lifetime,
-                      cib_t *cib, int cib_options, pe_working_set_t *data_set,
+int cli_cleanup_all(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
+                    const char *node_name, const char *operation,
+                    const char *interval_spec, pe_working_set_t *data_set);
+int cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
+                         const char *move_lifetime, int timeout_ms, cib_t *cib,
+                         int cib_options, gboolean promoted_role_only, gboolean force);
+int cli_resource_move(pcmk__output_t *out, pe_resource_t *rsc, const char *rsc_id,
+                      const char *host_name, const char *move_lifetime, cib_t *cib,
+                      int cib_options, pe_working_set_t *data_set,
                       gboolean promoted_role_only, gboolean force);
-crm_exit_t cli_resource_execute_from_params(const char *rsc_name, const char *rsc_class,
-                                            const char *rsc_prov, const char *rsc_type,
-                                            const char *rsc_action, GHashTable *params,
-                                            GHashTable *override_hash, int timeout_ms,
-                                            int resource_verbose, gboolean force);
-crm_exit_t cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
-                                const char *rsc_action, GHashTable *override_hash,
-                                int timeout_ms, cib_t *cib, pe_working_set_t *data_set,
-                                int resource_verbose, gboolean force);
+crm_exit_t cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
+                                            const char *rsc_class, const char *rsc_prov,
+                                            const char *rsc_type, const char *rsc_action,
+                                            GHashTable *params, GHashTable *override_hash,
+                                            int timeout_ms, int resource_verbose,
+                                            gboolean force);
+crm_exit_t cli_resource_execute(pcmk__output_t *out, pe_resource_t *rsc,
+                                const char *requested_name, const char *rsc_action,
+                                GHashTable *override_hash, int timeout_ms, cib_t *cib,
+                                pe_working_set_t *data_set, int resource_verbose,
+                                gboolean force);
 
-int cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
-                                  const char *attr_set, const char *attr_set_type,
-                                  const char *attr_id, const char *attr_name,
-                                  const char *attr_value, gboolean recursive, cib_t *cib,
-                                  int cib_options, pe_working_set_t *data_set, gboolean force);
-int cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name,
-                                  const char *attr_set, const char *attr_set_type,
-                                  const char *attr_id, const char *attr_name, cib_t *cib,
-                                  int cib_options, pe_working_set_t *data_set, gboolean force);
+int cli_resource_update_attribute(pcmk__output_t *out, pe_resource_t *rsc,
+                                  const char *requested_name, const char *attr_set,
+                                  const char *attr_set_type, const char *attr_id,
+                                  const char *attr_name, const char *attr_value,
+                                  gboolean recursive, cib_t *cib, int cib_options,
+                                  pe_working_set_t *data_set, gboolean force);
+int cli_resource_delete_attribute(pcmk__output_t *out, pe_resource_t *rsc,
+                                  const char *requested_name, const char *attr_set,
+                                  const char *attr_set_type, const char *attr_id,
+                                  const char *attr_name, cib_t *cib, int cib_options,
+                                  pe_working_set_t *data_set, gboolean force);
 
 int update_working_set_xml(pe_working_set_t *data_set, xmlNode **xml);
-int wait_till_stable(int timeout_ms, cib_t * cib);
-void cli_resource_why(cib_t *cib_conn, GListPtr resources, pe_resource_t *rsc,
-                      pe_node_t *node);
+int wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib);
+void cli_resource_why(pcmk__output_t *out, cib_t *cib_conn, GListPtr resources,
+                      pe_resource_t *rsc, pe_node_t *node);
diff --git a/tools/crm_resource_ban.c b/tools/crm_resource_ban.c
index e055797..95e5a17 100644
--- a/tools/crm_resource_ban.c
+++ b/tools/crm_resource_ban.c
@@ -12,7 +12,7 @@
 #define XPATH_MAX 1024
 
 static char *
-parse_cli_lifetime(const char *move_lifetime)
+parse_cli_lifetime(pcmk__output_t *out, const char *move_lifetime)
 {
     char *later_s = NULL;
     crm_time_t *now = NULL;
@@ -58,9 +58,9 @@ parse_cli_lifetime(const char *move_lifetime)
 
 // \return Standard Pacemaker return code
 int
-cli_resource_ban(const char *rsc_id, const char *host, const char *move_lifetime,
-                 GListPtr allnodes, cib_t * cib_conn, int cib_options,
-                 gboolean promoted_role_only)
+cli_resource_ban(pcmk__output_t *out, const char *rsc_id, const char *host,
+                 const char *move_lifetime, GListPtr allnodes, cib_t * cib_conn,
+                 int cib_options, gboolean promoted_role_only)
 {
     char *later_s = NULL;
     int rc = pcmk_rc_ok;
@@ -72,13 +72,13 @@ cli_resource_ban(const char *rsc_id, const char *host, const char *move_lifetime
         for(; n && rc == pcmk_rc_ok; n = n->next) {
             pe_node_t *target = n->data;
 
-            rc = cli_resource_ban(rsc_id, target->details->uname, move_lifetime,
+            rc = cli_resource_ban(out, rsc_id, target->details->uname, move_lifetime,
                                   NULL, cib_conn, cib_options, promoted_role_only);
         }
         return rc;
     }
 
-    later_s = parse_cli_lifetime(move_lifetime);
+    later_s = parse_cli_lifetime(out, move_lifetime);
     if(move_lifetime && later_s == NULL) {
         return EINVAL;
     }
@@ -143,10 +143,11 @@ cli_resource_ban(const char *rsc_id, const char *host, const char *move_lifetime
 
 // \return Standard Pacemaker return code
 int
-cli_resource_prefer(const char *rsc_id, const char *host, const char *move_lifetime,
-                    cib_t * cib_conn, int cib_options, gboolean promoted_role_only)
+cli_resource_prefer(pcmk__output_t *out,const char *rsc_id, const char *host,
+                    const char *move_lifetime, cib_t * cib_conn, int cib_options,
+                    gboolean promoted_role_only)
 {
-    char *later_s = parse_cli_lifetime(move_lifetime);
+    char *later_s = parse_cli_lifetime(out, move_lifetime);
     int rc = pcmk_rc_ok;
     xmlNode *location = NULL;
     xmlNode *fragment = NULL;
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
index 1dbc2e2..de1c608 100644
--- a/tools/crm_resource_print.c
+++ b/tools/crm_resource_print.c
@@ -12,7 +12,7 @@
 
 #define cons_string(x) x?x:"NA"
 void
-cli_resource_print_cts_constraints(pe_working_set_t * data_set)
+cli_resource_print_cts_constraints(pcmk__output_t *out, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     xmlNode *lifetime = NULL;
@@ -49,7 +49,7 @@ cli_resource_print_cts_constraints(pe_working_set_t * data_set)
 }
 
 void
-cli_resource_print_cts(pe_resource_t * rsc)
+cli_resource_print_cts(pcmk__output_t *out, pe_resource_t * rsc)
 {
     GListPtr lpc = NULL;
     const char *host = NULL;
@@ -78,13 +78,13 @@ cli_resource_print_cts(pe_resource_t * rsc)
     for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
         pe_resource_t *child = (pe_resource_t *) lpc->data;
 
-        cli_resource_print_cts(child);
+        cli_resource_print_cts(out, child);
     }
 }
 
 
 void
-cli_resource_print_raw(pe_resource_t * rsc)
+cli_resource_print_raw(pcmk__output_t *out, pe_resource_t * rsc)
 {
     GListPtr lpc = NULL;
     GListPtr children = rsc->children;
@@ -96,13 +96,13 @@ cli_resource_print_raw(pe_resource_t * rsc)
     for (lpc = children; lpc != NULL; lpc = lpc->next) {
         pe_resource_t *child = (pe_resource_t *) lpc->data;
 
-        cli_resource_print_raw(child);
+        cli_resource_print_raw(out, child);
     }
 }
 
 // \return Standard Pacemaker return code
 int
-cli_resource_print_list(pe_working_set_t * data_set, bool raw)
+cli_resource_print_list(pcmk__output_t *out, pe_working_set_t * data_set, bool raw)
 {
     int found = 0;
 
@@ -130,8 +130,9 @@ cli_resource_print_list(pe_working_set_t * data_set, bool raw)
 
 // \return Standard Pacemaker return code
 int
-cli_resource_print_operations(const char *rsc_id, const char *host_uname, bool active,
-                         pe_working_set_t * data_set)
+cli_resource_print_operations(pcmk__output_t *out, const char *rsc_id,
+                              const char *host_uname, bool active,
+                              pe_working_set_t * data_set)
 {
     pe_resource_t *rsc = NULL;
     int opts = pe_print_printf | pe_print_rsconly | pe_print_suppres_nl | pe_print_pending;
@@ -172,7 +173,7 @@ cli_resource_print_operations(const char *rsc_id, const char *host_uname, bool a
 }
 
 void
-cli_resource_print_location(pe_resource_t * rsc, const char *prefix)
+cli_resource_print_location(pcmk__output_t *out, pe_resource_t * rsc, const char *prefix)
 {
     GListPtr lpc = NULL;
     GListPtr list = rsc->rsc_location;
@@ -199,7 +200,8 @@ cli_resource_print_location(pe_resource_t * rsc, const char *prefix)
 }
 
 void
-cli_resource_print_colocation(pe_resource_t * rsc, bool dependents, bool recursive, int offset)
+cli_resource_print_colocation(pcmk__output_t *out, pe_resource_t * rsc,
+                              bool dependents, bool recursive, int offset)
 {
     char *prefix = NULL;
     GListPtr lpc = NULL;
@@ -239,7 +241,7 @@ cli_resource_print_colocation(pe_resource_t * rsc, bool dependents, bool recursi
         }
 
         if (dependents && recursive) {
-            cli_resource_print_colocation(peer, dependents, recursive, offset + 1);
+            cli_resource_print_colocation(out, peer, dependents, recursive, offset + 1);
         }
 
         score = score2char(cons->score);
@@ -251,11 +253,11 @@ cli_resource_print_colocation(pe_resource_t * rsc, bool dependents, bool recursi
             fprintf(stdout, "%s%-*s (score=%s, id=%s)\n", prefix, 80 - (4 * offset),
                     peer->id, score, cons->id);
         }
-        cli_resource_print_location(peer, prefix);
+        cli_resource_print_location(out, peer, prefix);
         free(score);
 
         if (!dependents && recursive) {
-            cli_resource_print_colocation(peer, dependents, recursive, offset + 1);
+            cli_resource_print_colocation(out, peer, dependents, recursive, offset + 1);
         }
     }
     free(prefix);
@@ -263,7 +265,8 @@ cli_resource_print_colocation(pe_resource_t * rsc, bool dependents, bool recursi
 
 // \return Standard Pacemaker return code
 int
-cli_resource_print(pe_resource_t *rsc, pe_working_set_t *data_set, bool expanded)
+cli_resource_print(pcmk__output_t *out, pe_resource_t *rsc,
+                   pe_working_set_t *data_set, bool expanded)
 {
     char *rsc_xml = NULL;
     int opts = pe_print_printf | pe_print_pending;
@@ -279,8 +282,8 @@ cli_resource_print(pe_resource_t *rsc, pe_working_set_t *data_set, bool expanded
 
 // \return Standard Pacemaker return code
 int
-cli_resource_print_attribute(pe_resource_t *rsc, const char *attr, const char *attr_set_type,
-                             pe_working_set_t * data_set)
+cli_resource_print_attribute(pcmk__output_t *out, pe_resource_t *rsc, const char *attr,
+                             const char *attr_set_type, pe_working_set_t * data_set)
 {
     int rc = ENXIO;
     unsigned int count = 0;
@@ -324,7 +327,8 @@ cli_resource_print_attribute(pe_resource_t *rsc, const char *attr, const char *a
 
 // \return Standard Pacemaker return code
 int
-cli_resource_print_property(pe_resource_t *rsc, const char *attr, pe_working_set_t * data_set)
+cli_resource_print_property(pcmk__output_t *out, pe_resource_t *rsc,
+                            const char *attr, pe_working_set_t * data_set)
 {
     const char *value = crm_element_value(rsc->xml, attr);
 
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index d133219..42d33bd 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -13,7 +13,8 @@
 #include <crm/common/xml_internal.h>
 
 static int
-do_find_resource(const char *rsc, pe_resource_t * the_rsc, pe_working_set_t * data_set)
+do_find_resource(pcmk__output_t *out, const char *rsc, pe_resource_t * the_rsc,
+                 pe_working_set_t * data_set)
 {
     int found = 0;
     GListPtr lpc = NULL;
@@ -43,7 +44,7 @@ do_find_resource(const char *rsc, pe_resource_t * the_rsc, pe_working_set_t * da
 }
 
 int
-cli_resource_search(pe_resource_t *rsc, const char *requested_name,
+cli_resource_search(pcmk__output_t *out, pe_resource_t *rsc, const char *requested_name,
                     pe_working_set_t *data_set)
 {
     int found = 0;
@@ -51,7 +52,7 @@ cli_resource_search(pe_resource_t *rsc, const char *requested_name,
 
     if (pe_rsc_is_clone(rsc)) {
         for (GListPtr iter = rsc->children; iter != NULL; iter = iter->next) {
-            found += do_find_resource(requested_name, iter->data, data_set);
+            found += do_find_resource(out, requested_name, iter->data, data_set);
         }
 
     /* The anonymous clone children's common ID is supplied */
@@ -62,11 +63,11 @@ cli_resource_search(pe_resource_t *rsc, const char *requested_name,
                && !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {
 
         for (GListPtr iter = parent->children; iter; iter = iter->next) {
-            found += do_find_resource(requested_name, iter->data, data_set);
+            found += do_find_resource(out, requested_name, iter->data, data_set);
         }
 
     } else {
-        found += do_find_resource(requested_name, rsc, data_set);
+        found += do_find_resource(out, requested_name, rsc, data_set);
     }
 
     return found;
@@ -76,8 +77,9 @@ cli_resource_search(pe_resource_t *rsc, const char *requested_name,
 
 // \return Standard Pacemaker return code
 static int
-find_resource_attr(cib_t * the_cib, const char *attr, const char *rsc, const char *attr_set_type,
-                   const char *set_name, const char *attr_id, const char *attr_name, char **value)
+find_resource_attr(pcmk__output_t *out, cib_t * the_cib, const char *attr,
+                   const char *rsc, const char *attr_set_type, const char *set_name,
+                   const char *attr_id, const char *attr_name, char **value)
 {
     int offset = 0;
     int rc = pcmk_rc_ok;
@@ -156,9 +158,11 @@ find_resource_attr(cib_t * the_cib, const char *attr, const char *rsc, const cha
 
 /* PRIVATE. Use the find_matching_attr_resources instead. */
 static void
-find_matching_attr_resources_recursive(GList/* <pe_resource_t*> */ ** result, pe_resource_t * rsc, const char * rsc_id, 
-                                       const char * attr_set, const char * attr_set_type, const char * attr_id,
-                                       const char * attr_name, cib_t * cib, const char * cmd, int depth)
+find_matching_attr_resources_recursive(pcmk__output_t *out, GList/* <pe_resource_t*> */ ** result,
+                                       pe_resource_t * rsc, const char * rsc_id,
+                                       const char * attr_set, const char * attr_set_type,
+                                       const char * attr_id, const char * attr_name,
+                                       cib_t * cib, const char * cmd, int depth)
 {
     int rc = pcmk_rc_ok;
     char *lookup_id = clone_strip(rsc->id);
@@ -166,7 +170,7 @@ find_matching_attr_resources_recursive(GList/* <pe_resource_t*> */ ** result, pe
 
     /* visit the children */
     for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
-        find_matching_attr_resources_recursive(result, (pe_resource_t*)gIter->data,
+        find_matching_attr_resources_recursive(out, result, (pe_resource_t*)gIter->data,
                                                rsc_id, attr_set, attr_set_type,
                                                attr_id, attr_name, cib, cmd, depth+1);
         /* do it only once for clones */
@@ -175,7 +179,8 @@ find_matching_attr_resources_recursive(GList/* <pe_resource_t*> */ ** result, pe
         }
     }
 
-    rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);
+    rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
+                            attr_set, attr_id, attr_name, &local_attr_id);
     /* Post-order traversal. 
      * The root is always on the list and it is the last item. */
     if((0 == depth) || (pcmk_rc_ok == rc)) {
@@ -190,9 +195,11 @@ find_matching_attr_resources_recursive(GList/* <pe_resource_t*> */ ** result, pe
 
 /* The result is a linearized pre-ordered tree of resources. */
 static GList/*<pe_resource_t*>*/ *
-find_matching_attr_resources(pe_resource_t * rsc, const char * rsc_id, const char * attr_set,
+find_matching_attr_resources(pcmk__output_t *out, pe_resource_t * rsc,
+                             const char * rsc_id, const char * attr_set,
                              const char * attr_set_type, const char * attr_id,
-                             const char * attr_name, cib_t * cib, const char * cmd, gboolean force)
+                             const char * attr_name, cib_t * cib, const char * cmd,
+                             gboolean force)
 {
     int rc = pcmk_rc_ok;
     char *lookup_id = NULL;
@@ -207,7 +214,8 @@ find_matching_attr_resources(pe_resource_t * rsc, const char * rsc_id, const cha
     if(rsc->parent && pe_clone == rsc->parent->variant) {
         int rc = pcmk_rc_ok;
         char *local_attr_id = NULL;
-        rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);
+        rc = find_resource_attr(out, cib, XML_ATTR_ID, rsc_id, attr_set_type,
+                                attr_set, attr_id, attr_name, &local_attr_id);
         free(local_attr_id);
 
         if(rc != pcmk_rc_ok) {
@@ -222,7 +230,8 @@ find_matching_attr_resources(pe_resource_t * rsc, const char * rsc_id, const cha
 
         if(child->variant == pe_native) {
             lookup_id = clone_strip(child->id); /* Could be a cloned group! */
-            rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);
+            rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
+                                    attr_set, attr_id, attr_name, &local_attr_id);
 
             if(rc == pcmk_rc_ok) {
                 rsc = child;
@@ -237,7 +246,7 @@ find_matching_attr_resources(pe_resource_t * rsc, const char * rsc_id, const cha
         return g_list_append(result, rsc);
     }
     /* If the resource is a group ==> children inherit the attribute if defined. */
-    find_matching_attr_resources_recursive(&result, rsc, rsc_id, attr_set,
+    find_matching_attr_resources_recursive(out, &result, rsc, rsc_id, attr_set,
                                            attr_set_type, attr_id, attr_name,
                                            cib, cmd, 0);
     return result;
@@ -245,11 +254,12 @@ find_matching_attr_resources(pe_resource_t * rsc, const char * rsc_id, const cha
 
 // \return Standard Pacemaker return code
 int
-cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
-                              const char *attr_set, const char *attr_set_type,
-                              const char *attr_id, const char *attr_name,
-                              const char *attr_value, gboolean recursive, cib_t *cib,
-                              int cib_options, pe_working_set_t *data_set, gboolean force)
+cli_resource_update_attribute(pcmk__output_t *out, pe_resource_t *rsc,
+                              const char *requested_name, const char *attr_set,
+                              const char *attr_set_type, const char *attr_id,
+                              const char *attr_name, const char *attr_value,
+                              gboolean recursive, cib_t *cib, int cib_options,
+                              pe_working_set_t *data_set, gboolean force)
 {
     int rc = pcmk_rc_ok;
     static bool need_init = TRUE;
@@ -263,13 +273,13 @@ cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
     if(attr_id == NULL
        && force == FALSE
        && find_resource_attr(
-           cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL) == EINVAL) {
+           out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL) == EINVAL) {
         printf("\n");
     }
 
     if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
         if (force == FALSE) {
-            rc = find_resource_attr(cib, XML_ATTR_ID, uber_parent(rsc)->id,
+            rc = find_resource_attr(out, cib, XML_ATTR_ID, uber_parent(rsc)->id,
                                     XML_TAG_META_SETS, attr_set, attr_id,
                                     attr_name, &local_attr_id);
             if (rc == pcmk_rc_ok && BE_QUIET == FALSE) {
@@ -286,7 +296,7 @@ cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
         resources = g_list_append(resources, rsc);
 
     } else {
-        resources = find_matching_attr_resources(rsc, requested_name, attr_set, attr_set_type,
+        resources = find_matching_attr_resources(out, rsc, requested_name, attr_set, attr_set_type,
                                                  attr_id, attr_name, cib, "update", force);
     }
 
@@ -306,8 +316,8 @@ cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
         attr_id = common_attr_id;
 
         lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
-        rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name,
-                                &local_attr_id);
+        rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
+                                attr_set, attr_id, attr_name, &local_attr_id);
 
         if (rc == pcmk_rc_ok) {
             crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
@@ -387,7 +397,7 @@ cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
                 if (cons->score > 0 && !pcmk_is_set(peer->flags, pe_rsc_allocating)) {
                     /* Don't get into colocation loops */
                     crm_debug("Setting %s=%s for dependent resource %s", attr_name, attr_value, peer->id);
-                    cli_resource_update_attribute(peer, peer->id, NULL, attr_set_type,
+                    cli_resource_update_attribute(out, peer, peer->id, NULL, attr_set_type,
                                                   NULL, attr_name, attr_value, recursive,
                                                   cib, cib_options, data_set, force);
                 }
@@ -400,10 +410,11 @@ cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
 
 // \return Standard Pacemaker return code
 int
-cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name,
-                              const char *attr_set, const char *attr_set_type,
-                              const char *attr_id, const char *attr_name, cib_t *cib,
-                              int cib_options, pe_working_set_t *data_set, gboolean force)
+cli_resource_delete_attribute(pcmk__output_t *out, pe_resource_t *rsc,
+                              const char *requested_name, const char *attr_set,
+                              const char *attr_set_type, const char *attr_id,
+                              const char *attr_name, cib_t *cib, int cib_options,
+                              pe_working_set_t *data_set, gboolean force)
 {
     int rc = pcmk_rc_ok;
     GList/*<pe_resource_t*>*/ *resources = NULL;
@@ -411,12 +422,12 @@ cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name,
     if(attr_id == NULL
        && force == FALSE
        && find_resource_attr(
-           cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL) == EINVAL) {
+           out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL) == EINVAL) {
         printf("\n");
     }
 
     if(pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
-        resources = find_matching_attr_resources(rsc, requested_name, attr_set, attr_set_type,
+        resources = find_matching_attr_resources(out, rsc, requested_name, attr_set, attr_set_type,
                                                  attr_id, attr_name, cib, "delete", force);
     } else {
         resources = g_list_append(resources, rsc);
@@ -430,8 +441,8 @@ cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name,
         rsc = (pe_resource_t*)gIter->data;
 
         lookup_id = clone_strip(rsc->id);
-        rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name,
-                                &local_attr_id);
+        rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
+                                attr_set, attr_id, attr_name, &local_attr_id);
 
         if (rc == ENXIO) {
             free(lookup_id);
@@ -471,9 +482,8 @@ cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name,
 
 // \return Standard Pacemaker return code
 static int
-send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
-                const char *host_uname, const char *rsc_id,
-                pe_working_set_t *data_set)
+send_lrm_rsc_op(pcmk__output_t *out, pcmk_ipc_api_t *controld_api, bool do_fail_resource,
+                const char *host_uname, const char *rsc_id, pe_working_set_t *data_set)
 {
     const char *router_node = host_uname;
     const char *rsc_api_id = NULL;
@@ -568,8 +578,9 @@ rsc_fail_name(pe_resource_t *rsc)
 
 // \return Standard Pacemaker return code
 static int
-clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
-                  const char *rsc_id, pe_working_set_t *data_set)
+clear_rsc_history(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
+                  const char *host_uname, const char *rsc_id,
+                  pe_working_set_t *data_set)
 {
     int rc = pcmk_rc_ok;
 
@@ -578,7 +589,7 @@ clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
      * single operation, we might wind up with a wrong idea of the current
      * resource state, and we might not re-probe the resource.
      */
-    rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, data_set);
+    rc = send_lrm_rsc_op(out, controld_api, false, host_uname, rsc_id, data_set);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
@@ -594,8 +605,8 @@ clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
 
 // \return Standard Pacemaker return code
 static int
-clear_rsc_failures(pcmk_ipc_api_t *controld_api, const char *node_name,
-                   const char *rsc_id, const char *operation,
+clear_rsc_failures(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
+                   const char *node_name, const char *rsc_id, const char *operation,
                    const char *interval_spec, pe_working_set_t *data_set)
 {
     int rc = pcmk_rc_ok;
@@ -667,7 +678,7 @@ clear_rsc_failures(pcmk_ipc_api_t *controld_api, const char *node_name,
     g_hash_table_iter_init(&iter, rscs);
     while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
         crm_debug("Erasing failures of %s on %s", failed_id, node_name);
-        rc = clear_rsc_history(controld_api, node_name, failed_id, data_set);
+        rc = clear_rsc_history(out, controld_api, node_name, failed_id, data_set);
         if (rc != pcmk_rc_ok) {
             return rc;
         }
@@ -697,8 +708,8 @@ clear_rsc_fail_attrs(pe_resource_t *rsc, const char *operation,
 
 // \return Standard Pacemaker return code
 int
-cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
-                    pe_resource_t *rsc, const char *operation,
+cli_resource_delete(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
+                    const char *host_uname, pe_resource_t *rsc, const char *operation,
                     const char *interval_spec, bool just_failures,
                     pe_working_set_t *data_set, gboolean force)
 {
@@ -714,7 +725,7 @@ cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
         for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
             pe_resource_t *child = (pe_resource_t *) lpc->data;
 
-            rc = cli_resource_delete(controld_api, host_uname, child, operation,
+            rc = cli_resource_delete(out, controld_api, host_uname, child, operation,
                                      interval_spec, just_failures, data_set,
                                      force);
             if (rc != pcmk_rc_ok) {
@@ -749,7 +760,7 @@ cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
             node = (pe_node_t *) lpc->data;
 
             if (node->details->online) {
-                rc = cli_resource_delete(controld_api, node->details->uname,
+                rc = cli_resource_delete(out, controld_api, node->details->uname,
                                          rsc, operation, interval_spec,
                                          just_failures, data_set, force);
             }
@@ -791,10 +802,10 @@ cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
     }
 
     if (just_failures) {
-        rc = clear_rsc_failures(controld_api, host_uname, rsc->id, operation,
+        rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
                                 interval_spec, data_set);
     } else {
-        rc = clear_rsc_history(controld_api, host_uname, rsc->id, data_set);
+        rc = clear_rsc_history(out, controld_api, host_uname, rsc->id, data_set);
     }
     if (rc != pcmk_rc_ok) {
         printf("Cleaned %s failures on %s, but unable to clean history: %s\n",
@@ -807,9 +818,9 @@ cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
 
 // \return Standard Pacemaker return code
 int
-cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
-                const char *operation, const char *interval_spec,
-                pe_working_set_t *data_set)
+cli_cleanup_all(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
+                const char *node_name, const char *operation,
+                const char *interval_spec, pe_working_set_t *data_set)
 {
     int rc = pcmk_rc_ok;
     int attr_options = pcmk__node_attr_none;
@@ -842,7 +853,7 @@ cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
     }
 
     if (node_name) {
-        rc = clear_rsc_failures(controld_api, node_name, NULL,
+        rc = clear_rsc_failures(out, controld_api, node_name, NULL,
                                 operation, interval_spec, data_set);
         if (rc != pcmk_rc_ok) {
             printf("Cleaned all resource failures on %s, but unable to clean history: %s\n",
@@ -853,7 +864,7 @@ cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
         for (GList *iter = data_set->nodes; iter; iter = iter->next) {
             pe_node_t *node = (pe_node_t *) iter->data;
 
-            rc = clear_rsc_failures(controld_api, node->details->uname, NULL,
+            rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
                                     operation, interval_spec, data_set);
             if (rc != pcmk_rc_ok) {
                 printf("Cleaned all resource failures on all nodes, but unable to clean history: %s\n",
@@ -868,17 +879,17 @@ cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
 }
 
 void
-cli_resource_check(cib_t * cib_conn, pe_resource_t *rsc)
+cli_resource_check(pcmk__output_t *out, cib_t * cib_conn, pe_resource_t *rsc)
 {
     bool printed = false;
     char *role_s = NULL;
     char *managed = NULL;
     pe_resource_t *parent = uber_parent(rsc);
 
-    find_resource_attr(cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
+    find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
                        NULL, NULL, NULL, XML_RSC_ATTR_MANAGED, &managed);
 
-    find_resource_attr(cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
+    find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
                        NULL, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, &role_s);
 
     if(role_s) {
@@ -920,11 +931,12 @@ cli_resource_check(cib_t * cib_conn, pe_resource_t *rsc)
 
 // \return Standard Pacemaker return code
 int
-cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
-                  const char *rsc_id, pe_working_set_t *data_set)
+cli_resource_fail(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
+                  const char *host_uname, const char *rsc_id,
+                  pe_working_set_t *data_set)
 {
     crm_notice("Failing %s on %s", rsc_id, host_uname);
-    return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, data_set);
+    return send_lrm_rsc_op(out, controld_api, true, host_uname, rsc_id, data_set);
 }
 
 static GHashTable *
@@ -1055,7 +1067,7 @@ static void dump_list(GList *items, const char *tag)
     }
 }
 
-static void display_list(GList *items, const char *tag) 
+static void display_list(pcmk__output_t *out, GList *items, const char *tag)
 {
     GList *item = NULL;
 
@@ -1103,7 +1115,8 @@ update_working_set_xml(pe_working_set_t *data_set, xmlNode **xml)
  *       data_set->input and data_set->now.
  */
 static int
-update_working_set_from_cib(pe_working_set_t * data_set, cib_t *cib)
+update_working_set_from_cib(pcmk__output_t *out, pe_working_set_t * data_set,
+                            cib_t *cib)
 {
     xmlNode *cib_xml_copy = NULL;
     int rc = pcmk_rc_ok;
@@ -1127,7 +1140,8 @@ update_working_set_from_cib(pe_working_set_t * data_set, cib_t *cib)
 
 // \return Standard Pacemaker return code
 static int
-update_dataset(cib_t *cib, pe_working_set_t * data_set, bool simulate)
+update_dataset(pcmk__output_t *out, cib_t *cib, pe_working_set_t * data_set,
+               bool simulate)
 {
     char *pid = NULL;
     char *shadow_file = NULL;
@@ -1135,7 +1149,7 @@ update_dataset(cib_t *cib, pe_working_set_t * data_set, bool simulate)
     int rc = pcmk_rc_ok;
 
     pe_reset_working_set(data_set);
-    rc = update_working_set_from_cib(data_set, cib);
+    rc = update_working_set_from_cib(out, data_set, cib);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
@@ -1168,7 +1182,7 @@ update_dataset(cib_t *cib, pe_working_set_t * data_set, bool simulate)
 
         pcmk__schedule_actions(data_set, data_set->input, NULL);
         run_simulation(data_set, shadow_cib, NULL, TRUE);
-        rc = update_dataset(shadow_cib, data_set, FALSE);
+        rc = update_dataset(out, shadow_cib, data_set, FALSE);
 
     } else {
         cluster_status(data_set);
@@ -1260,9 +1274,9 @@ max_delay_in(pe_working_set_t * data_set, GList *resources)
  * \return Standard Pacemaker return code (exits on certain failures)
  */
 int
-cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_lifetime,
-                     int timeout_ms, cib_t *cib, int cib_options,
-                     gboolean promoted_role_only, gboolean force)
+cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
+                     const char *move_lifetime, int timeout_ms, cib_t *cib,
+                     int cib_options, gboolean promoted_role_only, gboolean force)
 {
     int rc = pcmk_rc_ok;
     int lpc = 0;
@@ -1322,7 +1336,7 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_life
         goto done;
     }
     pe__set_working_set_flags(data_set, pe_flag_no_counts|pe_flag_no_compat);
-    rc = update_dataset(cib, data_set, FALSE);
+    rc = update_dataset(out, cib, data_set, FALSE);
     if(rc != pcmk_rc_ok) {
         fprintf(stdout, "Could not get new resource list: %s (%d)\n", pcmk_strerror(rc), rc);
         goto done;
@@ -1336,7 +1350,7 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_life
     if (stop_via_ban) {
         /* Stop the clone or bundle instance by banning it from the host */
         BE_QUIET = TRUE;
-        rc = cli_resource_ban(rsc_id, host, move_lifetime, NULL, cib,
+        rc = cli_resource_ban(out, rsc_id, host, move_lifetime, NULL, cib,
                               cib_options, promoted_role_only);
 
     } else {
@@ -1346,11 +1360,11 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_life
          */
         char *lookup_id = clone_strip(rsc->id);
 
-        find_resource_attr(cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
+        find_resource_attr(out, cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
                            NULL, XML_RSC_ATTR_TARGET_ROLE, &orig_target_role);
         free(lookup_id);
-        rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
-                                           XML_RSC_ATTR_TARGET_ROLE,
+        rc = cli_resource_update_attribute(out, rsc, rsc_id, NULL, XML_TAG_META_SETS,
+                                           NULL, XML_RSC_ATTR_TARGET_ROLE,
                                            RSC_STOPPED, FALSE, cib, cib_options,
                                            data_set, force);
     }
@@ -1365,7 +1379,7 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_life
         goto done;
     }
 
-    rc = update_dataset(cib, data_set, TRUE);
+    rc = update_dataset(out, cib, data_set, TRUE);
     if(rc != pcmk_rc_ok) {
         fprintf(stderr, "Could not determine which resources would be stopped\n");
         goto failure;
@@ -1376,7 +1390,7 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_life
 
     list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
     fprintf(stdout, "Waiting for %d resources to stop:\n", g_list_length(list_delta));
-    display_list(list_delta, " * ");
+    display_list(out, list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while (list_delta != NULL) {
@@ -1392,7 +1406,7 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_life
                 timeout -= sleep_interval;
                 crm_trace("%ds remaining", timeout);
             }
-            rc = update_dataset(cib, data_set, FALSE);
+            rc = update_dataset(out, cib, data_set, FALSE);
             if(rc != pcmk_rc_ok) {
                 fprintf(stderr, "Could not determine which resources were stopped\n");
                 goto failure;
@@ -1412,7 +1426,7 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_life
         if(before == g_list_length(list_delta)) {
             /* aborted during stop phase, print the contents of list_delta */
             fprintf(stderr, "Could not complete shutdown of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
-            display_list(list_delta, " * ");
+            display_list(out, list_delta, " * ");
             rc = ETIME;
             goto failure;
         }
@@ -1423,15 +1437,15 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_life
         rc = cli_resource_clear(rsc_id, host, NULL, cib, cib_options, TRUE, force);
 
     } else if (orig_target_role) {
-        rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
+        rc = cli_resource_update_attribute(out, rsc, rsc_id, NULL, XML_TAG_META_SETS,
                                            NULL, XML_RSC_ATTR_TARGET_ROLE,
                                            orig_target_role, FALSE, cib,
                                            cib_options, data_set, force);
         free(orig_target_role);
         orig_target_role = NULL;
     } else {
-        rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
-                                           XML_RSC_ATTR_TARGET_ROLE, cib,
+        rc = cli_resource_delete_attribute(out, rsc, rsc_id, NULL, XML_TAG_META_SETS,
+                                           NULL, XML_RSC_ATTR_TARGET_ROLE, cib,
                                            cib_options, data_set, force);
     }
 
@@ -1446,7 +1460,7 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_life
     target_active = restart_target_active;
     list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
     fprintf(stdout, "Waiting for %d resources to start again:\n", g_list_length(list_delta));
-    display_list(list_delta, " * ");
+    display_list(out, list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while (waiting_for_starts(list_delta, rsc, host)) {
@@ -1464,7 +1478,7 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_life
                 crm_trace("%ds remaining", timeout);
             }
 
-            rc = update_dataset(cib, data_set, FALSE);
+            rc = update_dataset(out, cib, data_set, FALSE);
             if(rc != pcmk_rc_ok) {
                 fprintf(stderr, "Could not determine which resources were started\n");
                 goto failure;
@@ -1487,7 +1501,7 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_life
         if(before == g_list_length(list_delta)) {
             /* aborted during start phase, print the contents of list_delta */
             fprintf(stdout, "Could not complete restart of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
-            display_list(list_delta, " * ");
+            display_list(out, list_delta, " * ");
             rc = ETIME;
             goto failure;
         }
@@ -1501,12 +1515,12 @@ cli_resource_restart(pe_resource_t *rsc, const char *host, const char *move_life
     if (stop_via_ban) {
         cli_resource_clear(rsc_id, host, NULL, cib, cib_options, TRUE, force);
     } else if (orig_target_role) {
-        cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
+        cli_resource_update_attribute(out, rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
                                       XML_RSC_ATTR_TARGET_ROLE, orig_target_role,
                                       FALSE, cib, cib_options, data_set, force);
         free(orig_target_role);
     } else {
-        cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
+        cli_resource_delete_attribute(out, rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
                                       XML_RSC_ATTR_TARGET_ROLE, cib, cib_options,
                                       data_set, force);
     }
@@ -1571,7 +1585,7 @@ actions_are_pending(GListPtr actions)
  * \return void
  */
 static void
-print_pending_actions(GListPtr actions)
+print_pending_actions(pcmk__output_t *out, GListPtr actions)
 {
     GListPtr action;
 
@@ -1610,7 +1624,7 @@ print_pending_actions(GListPtr actions)
  * \return Standard Pacemaker return code
  */
 int
-wait_till_stable(int timeout_ms, cib_t * cib)
+wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib)
 {
     pe_working_set_t *data_set = NULL;
     int rc = pcmk_rc_ok;
@@ -1632,7 +1646,7 @@ wait_till_stable(int timeout_ms, cib_t * cib)
         if (time_diff > 0) {
             crm_info("Waiting up to %ld seconds for cluster actions to complete", time_diff);
         } else {
-            print_pending_actions(data_set->actions);
+            print_pending_actions(out, data_set->actions);
             pe_free_working_set(data_set);
             return ETIME;
         }
@@ -1642,7 +1656,7 @@ wait_till_stable(int timeout_ms, cib_t * cib)
 
         /* Get latest transition graph */
         pe_reset_working_set(data_set);
-        rc = update_working_set_from_cib(data_set, cib);
+        rc = update_working_set_from_cib(out, data_set, cib);
         if (rc != pcmk_rc_ok) {
             pe_free_working_set(data_set);
             return rc;
@@ -1675,11 +1689,11 @@ wait_till_stable(int timeout_ms, cib_t * cib)
 }
 
 crm_exit_t
-cli_resource_execute_from_params(const char *rsc_name, const char *rsc_class,
-                                 const char *rsc_prov, const char *rsc_type,
-                                 const char *action, GHashTable *params,
-                                 GHashTable *override_hash, int timeout_ms,
-                                 int resource_verbose, gboolean force)
+cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
+                                 const char *rsc_class, const char *rsc_prov,
+                                 const char *rsc_type, const char *action,
+                                 GHashTable *params, GHashTable *override_hash,
+                                 int timeout_ms, int resource_verbose, gboolean force)
 {
     GHashTable *params_copy = NULL;
     crm_exit_t exit_code = CRM_EX_OK;
@@ -1815,9 +1829,10 @@ done:
 }
 
 crm_exit_t
-cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
-                     const char *rsc_action, GHashTable *override_hash,
-                     int timeout_ms, cib_t * cib, pe_working_set_t *data_set,
+cli_resource_execute(pcmk__output_t *out, pe_resource_t *rsc,
+                     const char *requested_name, const char *rsc_action,
+                     GHashTable *override_hash, int timeout_ms,
+                     cib_t * cib, pe_working_set_t *data_set,
                      int resource_verbose, gboolean force)
 {
     crm_exit_t exit_code = CRM_EX_OK;
@@ -1842,7 +1857,7 @@ cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
         action = rsc_action+6;
 
         if(pe_rsc_is_clone(rsc)) {
-            int rc = cli_resource_search(rsc, requested_name, data_set);
+            int rc = cli_resource_search(out, rsc, requested_name, data_set);
             if(rc > 0 && force == FALSE) {
                 CMD_ERR("It is not safe to %s %s here: the cluster claims it is already active",
                         action, rsc->id);
@@ -1879,7 +1894,7 @@ cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
 
     rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;
 
-    exit_code = cli_resource_execute_from_params(rid, rclass, rprov, rtype, action,
+    exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, action,
                                                  params, override_hash, timeout_ms,
                                                  resource_verbose, force);
     return exit_code;
@@ -1887,10 +1902,10 @@ cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
 
 // \return Standard Pacemaker return code
 int
-cli_resource_move(pe_resource_t *rsc, const char *rsc_id, const char *host_name,
-                  const char *move_lifetime, cib_t *cib, int cib_options,
-                  pe_working_set_t *data_set, gboolean promoted_role_only,
-                  gboolean force)
+cli_resource_move(pcmk__output_t *out, pe_resource_t *rsc, const char *rsc_id,
+                  const char *host_name, const char *move_lifetime, cib_t *cib,
+                  int cib_options, pe_working_set_t *data_set,
+                  gboolean promoted_role_only, gboolean force)
 {
     int rc = pcmk_rc_ok;
     unsigned int count = 0;
@@ -1966,7 +1981,7 @@ cli_resource_move(pe_resource_t *rsc, const char *rsc_id, const char *host_name,
                        cib_options, TRUE, force);
 
     /* Record an explicit preference for 'dest' */
-    rc = cli_resource_prefer(rsc_id, dest->details->uname, move_lifetime,
+    rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
                              cib, cib_options, promoted_role_only);
 
     crm_trace("%s%s now prefers node %s%s",
@@ -1978,7 +1993,7 @@ cli_resource_move(pe_resource_t *rsc, const char *rsc_id, const char *host_name,
     if(force && (cur_is_dest == FALSE)) {
         /* Ban the original location if possible */
         if(current) {
-            (void)cli_resource_ban(rsc_id, current->details->uname, move_lifetime,
+            (void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
                                    NULL, cib, cib_options, promoted_role_only);
 
         } else if(count > 1) {
@@ -1999,7 +2014,8 @@ cli_resource_move(pe_resource_t *rsc, const char *rsc_id, const char *host_name,
 }
 
 static void
-cli_resource_why_without_rsc_and_host(cib_t *cib_conn,GListPtr resources)
+cli_resource_why_without_rsc_and_host(pcmk__output_t *out, cib_t *cib_conn,
+                                      GListPtr resources)
 {
     GListPtr lpc = NULL;
     GListPtr hosts = NULL;
@@ -2014,7 +2030,7 @@ cli_resource_why_without_rsc_and_host(cib_t *cib_conn,GListPtr resources)
             printf("Resource %s is running\n", rsc->id);
         }
 
-        cli_resource_check(cib_conn, rsc);
+        cli_resource_check(out, cib_conn, rsc);
         g_list_free(hosts);
         hosts = NULL;
      }
@@ -2022,19 +2038,21 @@ cli_resource_why_without_rsc_and_host(cib_t *cib_conn,GListPtr resources)
 }
 
 static void
-cli_resource_why_with_rsc_and_host(cib_t *cib_conn, GListPtr resources,
-                                   pe_resource_t *rsc, const char *host_uname)
+cli_resource_why_with_rsc_and_host(pcmk__output_t *out, cib_t *cib_conn,
+                                   GListPtr resources, pe_resource_t *rsc,
+                                   const char *host_uname)
 {
     if (resource_is_running_on(rsc, host_uname)) {
         printf("Resource %s is running on host %s\n",rsc->id,host_uname);
     } else {
         printf("Resource %s is not running on host %s\n", rsc->id, host_uname);
     }
-    cli_resource_check(cib_conn, rsc);
+    cli_resource_check(out, cib_conn, rsc);
 }
 
 static void
-cli_resource_why_without_rsc_with_host(cib_t *cib_conn,GListPtr resources,pe_node_t *node)
+cli_resource_why_without_rsc_with_host(pcmk__output_t *out, cib_t *cib_conn,
+                                       GListPtr resources, pe_node_t *node)
 {
     const char* host_uname =  node->details->uname;
     GListPtr allResources = node->details->allocated_rsc;
@@ -2045,14 +2063,14 @@ cli_resource_why_without_rsc_with_host(cib_t *cib_conn,GListPtr resources,pe_nod
     for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
         pe_resource_t *rsc = (pe_resource_t *) lpc->data;
         printf("Resource %s is running on host %s\n",rsc->id,host_uname);
-        cli_resource_check(cib_conn,rsc);
+        cli_resource_check(out, cib_conn, rsc);
     }
 
     for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
         pe_resource_t *rsc = (pe_resource_t *) lpc->data;
         printf("Resource %s is assigned to host %s but not running\n",
                rsc->id, host_uname);
-        cli_resource_check(cib_conn,rsc);
+        cli_resource_check(out, cib_conn, rsc);
      }
 
      g_list_free(allResources);
@@ -2061,33 +2079,33 @@ cli_resource_why_without_rsc_with_host(cib_t *cib_conn,GListPtr resources,pe_nod
 }
 
 static void
-cli_resource_why_with_rsc_without_host(cib_t *cib_conn, GListPtr resources,
-                                       pe_resource_t *rsc)
+cli_resource_why_with_rsc_without_host(pcmk__output_t *out, cib_t *cib_conn,
+                                       GListPtr resources, pe_resource_t *rsc)
 {
     GListPtr hosts = NULL;
 
     rsc->fns->location(rsc, &hosts, TRUE);
     printf("Resource %s is %srunning\n", rsc->id, (hosts? "" : "not "));
-    cli_resource_check(cib_conn, rsc);
+    cli_resource_check(out, cib_conn, rsc);
     g_list_free(hosts);
 }
 
-void cli_resource_why(cib_t *cib_conn, GListPtr resources, pe_resource_t *rsc,
-                      pe_node_t *node)
+void cli_resource_why(pcmk__output_t *out, cib_t *cib_conn, GListPtr resources,
+                      pe_resource_t *rsc, pe_node_t *node)
 {
     const char *host_uname = (node == NULL)? NULL : node->details->uname;
 
     if ((rsc == NULL) && (host_uname == NULL)) {
-        cli_resource_why_without_rsc_and_host(cib_conn, resources);
+        cli_resource_why_without_rsc_and_host(out, cib_conn, resources);
 
     } else if ((rsc != NULL) && (host_uname != NULL)) {
-        cli_resource_why_with_rsc_and_host(cib_conn, resources, rsc,
+        cli_resource_why_with_rsc_and_host(out, cib_conn, resources, rsc,
                                            host_uname);
 
     } else if ((rsc == NULL) && (host_uname != NULL)) {
-        cli_resource_why_without_rsc_with_host(cib_conn, resources, node);
+        cli_resource_why_without_rsc_with_host(out, cib_conn, resources, node);
 
     } else if ((rsc != NULL) && (host_uname == NULL)) {
-        cli_resource_why_with_rsc_without_host(cib_conn, resources, rsc);
+        cli_resource_why_with_rsc_without_host(out, cib_conn, resources, rsc);
     }
 }
-- 
1.8.3.1


From 1194f91a9c21877a0aac8d7fe579307a1b024971 Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Wed, 26 Aug 2020 16:37:34 -0400
Subject: [PATCH 03/19] Refactor: tools: Use is_quiet in crm_resource.

This gets rid of the BE_QUIET global.
---
 tools/crm_resource.c         | 15 +++++++--------
 tools/crm_resource.h         |  2 --
 tools/crm_resource_ban.c     |  2 +-
 tools/crm_resource_runtime.c | 18 +++++++++---------
 4 files changed, 17 insertions(+), 20 deletions(-)

diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index 7a661a4..e663f55 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -152,7 +152,6 @@ gboolean restart_cb(const gchar *option_name, const gchar *optarg,
 gboolean wait_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean why_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 
-bool BE_QUIET = FALSE;
 static crm_exit_t exit_code = CRM_EX_OK;
 static pcmk__output_t *out = NULL;
 
@@ -639,7 +638,7 @@ attr_set_type_cb(const gchar *option_name, const gchar *optarg, gpointer data, G
 gboolean
 class_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (!(pcmk_get_ra_caps(optarg) & pcmk_ra_cap_params)) {
-        if (BE_QUIET == FALSE) {
+        if (!out->is_quiet(out)) {
             g_set_error(error, G_OPTION_ERROR, CRM_EX_INVALID_PARAM,
                         "Standard %s does not support parameters\n", optarg);
         }
@@ -991,7 +990,7 @@ cleanup(pcmk__output_t *out, pe_resource_t *rsc)
     rc = cli_resource_delete(out, controld_api, options.host_uname, rsc, options.operation,
                              options.interval_spec, TRUE, data_set, options.force);
 
-    if ((rc == pcmk_rc_ok) && !BE_QUIET) {
+    if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
         // Show any reasons why resource might stay stopped
         cli_resource_check(out, cib_conn, rsc);
     }
@@ -1011,7 +1010,7 @@ clear_constraints(pcmk__output_t *out, xmlNodePtr *cib_xml_copy)
     pe_node_t *dest = NULL;
     int rc = pcmk_rc_ok;
 
-    if (BE_QUIET == FALSE) {
+    if (!out->is_quiet(out)) {
         before = build_constraint_list(data_set->input);
     }
 
@@ -1024,7 +1023,7 @@ clear_constraints(pcmk__output_t *out, xmlNodePtr *cib_xml_copy)
         dest = pe_find_node(data_set->nodes, options.host_uname);
         if (dest == NULL) {
             rc = pcmk_rc_node_unknown;
-            if (BE_QUIET == FALSE) {
+            if (!out->is_quiet(out)) {
                 g_list_free(before);
             }
             return rc;
@@ -1037,7 +1036,7 @@ clear_constraints(pcmk__output_t *out, xmlNodePtr *cib_xml_copy)
                                 cib_conn, options.cib_options, TRUE, options.force);
     }
 
-    if (BE_QUIET == FALSE) {
+    if (!out->is_quiet(out)) {
         rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local | cib_sync_call);
         rc = pcmk_legacy2rc(rc);
 
@@ -1321,7 +1320,7 @@ refresh_resource(pcmk__output_t *out, pe_resource_t *rsc)
     rc = cli_resource_delete(out, controld_api, options.host_uname, rsc, NULL,
                              0, FALSE, data_set, options.force);
 
-    if ((rc == pcmk_rc_ok) && !BE_QUIET) {
+    if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
         // Show any reasons why resource might stay stopped
         cli_resource_check(out, cib_conn, rsc);
     }
@@ -1550,7 +1549,7 @@ main(int argc, char **argv)
     }
 
     options.resource_verbose = args->verbosity;
-    BE_QUIET = args->quiet;
+    out->quiet = args->quiet;
 
     crm_log_args(argc, argv);
 
diff --git a/tools/crm_resource.h b/tools/crm_resource.h
index bf99f24..0100488 100644
--- a/tools/crm_resource.h
+++ b/tools/crm_resource.h
@@ -22,8 +22,6 @@
 #include <crm/pengine/internal.h>
 #include <pacemaker-internal.h>
 
-extern bool BE_QUIET;
-
 /* ban */
 int cli_resource_prefer(pcmk__output_t *out, const char *rsc_id, const char *host,
                         const char *move_lifetime, cib_t * cib_conn, int cib_options,
diff --git a/tools/crm_resource_ban.c b/tools/crm_resource_ban.c
index 95e5a17..abed209 100644
--- a/tools/crm_resource_ban.c
+++ b/tools/crm_resource_ban.c
@@ -88,7 +88,7 @@ cli_resource_ban(pcmk__output_t *out, const char *rsc_id, const char *host,
     location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
     crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host);
 
-    if (BE_QUIET == FALSE) {
+    if (!out->is_quiet(out)) {
         CMD_ERR("WARNING: Creating rsc_location constraint '%s'"
                 " with a score of -INFINITY for resource %s"
                 " on %s.", ID(location), rsc_id, host);
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 42d33bd..3e1f985 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -22,7 +22,7 @@ do_find_resource(pcmk__output_t *out, const char *rsc, pe_resource_t * the_rsc,
     for (lpc = the_rsc->running_on; lpc != NULL; lpc = lpc->next) {
         pe_node_t *node = (pe_node_t *) lpc->data;
 
-        if (BE_QUIET) {
+        if (out->is_quiet(out)) {
             fprintf(stdout, "%s\n", node->details->uname);
         } else {
             const char *state = "";
@@ -36,7 +36,7 @@ do_find_resource(pcmk__output_t *out, const char *rsc, pe_resource_t * the_rsc,
         found++;
     }
 
-    if (BE_QUIET == FALSE && found == 0) {
+    if (!out->is_quiet(out) && found == 0) {
         fprintf(stderr, "resource %s is NOT running\n", rsc);
     }
 
@@ -220,7 +220,7 @@ find_matching_attr_resources(pcmk__output_t *out, pe_resource_t * rsc,
 
         if(rc != pcmk_rc_ok) {
             rsc = rsc->parent;
-            if (BE_QUIET == FALSE) {
+            if (!out->is_quiet(out)) {
                 printf("Performing %s of '%s' on '%s', the parent of '%s'\n", cmd, attr_name, rsc->id, rsc_id);
             }
         }
@@ -235,7 +235,7 @@ find_matching_attr_resources(pcmk__output_t *out, pe_resource_t * rsc,
 
             if(rc == pcmk_rc_ok) {
                 rsc = child;
-                if (BE_QUIET == FALSE) {
+                if (!out->is_quiet(out)) {
                     printf("A value for '%s' already exists in child '%s', performing %s on that instead of '%s'\n", attr_name, lookup_id, cmd, rsc_id);
                 }
             }
@@ -282,7 +282,7 @@ cli_resource_update_attribute(pcmk__output_t *out, pe_resource_t *rsc,
             rc = find_resource_attr(out, cib, XML_ATTR_ID, uber_parent(rsc)->id,
                                     XML_TAG_META_SETS, attr_set, attr_id,
                                     attr_name, &local_attr_id);
-            if (rc == pcmk_rc_ok && BE_QUIET == FALSE) {
+            if (rc == pcmk_rc_ok && !out->is_quiet(out)) {
                 printf("WARNING: There is already a meta attribute for '%s' called '%s' (id=%s)\n",
                        uber_parent(rsc)->id, attr_name, local_attr_id);
                 printf("         Delete '%s' first or use the force option to override\n",
@@ -359,7 +359,7 @@ cli_resource_update_attribute(pcmk__output_t *out, pe_resource_t *rsc,
         rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
         rc = pcmk_legacy2rc(rc);
 
-        if (rc == pcmk_rc_ok && BE_QUIET == FALSE) {
+        if (rc == pcmk_rc_ok && !out->is_quiet(out)) {
             printf("Set '%s' option: id=%s%s%s%s%s value=%s\n", lookup_id, local_attr_id,
                    attr_set ? " set=" : "", attr_set ? attr_set : "",
                    attr_name ? " name=" : "", attr_name ? attr_name : "", attr_value);
@@ -466,7 +466,7 @@ cli_resource_delete_attribute(pcmk__output_t *out, pe_resource_t *rsc,
         rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);
         rc = pcmk_legacy2rc(rc);
 
-        if (rc == pcmk_rc_ok && BE_QUIET == FALSE) {
+        if (rc == pcmk_rc_ok && !out->is_quiet(out)) {
             printf("Deleted '%s' option: id=%s%s%s%s%s\n", lookup_id, local_attr_id,
                    attr_set ? " set=" : "", attr_set ? attr_set : "",
                    attr_name ? " name=" : "", attr_name ? attr_name : "");
@@ -1349,7 +1349,7 @@ cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
 
     if (stop_via_ban) {
         /* Stop the clone or bundle instance by banning it from the host */
-        BE_QUIET = TRUE;
+        out->quiet = true;
         rc = cli_resource_ban(out, rsc_id, host, move_lifetime, NULL, cib,
                               cib_options, promoted_role_only);
 
@@ -1631,7 +1631,7 @@ wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib)
     int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
     time_t expire_time = time(NULL) + timeout_s;
     time_t time_diff;
-    bool printed_version_warning = BE_QUIET; // i.e. don't print if quiet
+    bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet
 
     data_set = pe_new_working_set();
     if (data_set == NULL) {
-- 
1.8.3.1


From 155a0a8e078061cebbe3354404b58882196b0350 Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Mon, 21 Sep 2020 15:59:40 -0400
Subject: [PATCH 04/19] Feature: tools: Add an output message for a list of
 resource names.

This is kind of an unusual output message in that it just dumps a list
of resource names with no other information.  It appears to only list
primitive resources, too.  This doesn't seem especially useful anywhere
else, so I am just adding it to crm_resource.

Note that this is one of the basic XML lists, wrapped with <list> and
<item> tags.  There's no additional useful information for the items of
this list.
---
 tools/crm_resource.c       | 31 ++++++-----------------
 tools/crm_resource.h       |  3 +++
 tools/crm_resource_print.c | 63 +++++++++++++++++++++++++++++++++-------------
 3 files changed, 56 insertions(+), 41 deletions(-)

diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index e663f55..d9e8e70 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1176,28 +1176,6 @@ list_providers(pcmk__output_t *out, const char *agent_spec, crm_exit_t *exit_cod
     return rc;
 }
 
-static int
-list_raw(pcmk__output_t *out)
-{
-    int rc = pcmk_rc_ok;
-    int found = 0;
-    GListPtr lpc = NULL;
-
-    for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
-        pe_resource_t *rsc = (pe_resource_t *) lpc->data;
-
-        found++;
-        cli_resource_print_raw(out, rsc);
-    }
-
-    if (found == 0) {
-        printf("NO resources configured\n");
-        rc = ENXIO;
-    }
-
-    return rc;
-}
-
 static void
 list_stacks_and_constraints(pcmk__output_t *out, pe_resource_t *rsc, bool recursive)
 {
@@ -1634,6 +1612,8 @@ main(int argc, char **argv)
         }
     }
 
+    crm_resource_register_messages(out);
+
     if (args->version) {
         out->version(out, false);
         goto done;
@@ -1741,7 +1721,12 @@ main(int argc, char **argv)
             break;
 
         case cmd_list_instances:
-            rc = list_raw(out);
+            rc = out->message(out, "resource-names-list", data_set->resources);
+
+            if (rc != pcmk_rc_ok) {
+                rc = ENXIO;
+            }
+
             break;
 
         case cmd_list_standards:
diff --git a/tools/crm_resource.h b/tools/crm_resource.h
index 0100488..28a3760 100644
--- a/tools/crm_resource.h
+++ b/tools/crm_resource.h
@@ -14,6 +14,7 @@
 #include <crm/services.h>
 #include <crm/common/xml.h>
 #include <crm/common/mainloop.h>
+#include <crm/common/output_internal.h>
 
 #include <crm/cib.h>
 #include <crm/common/attrd_internal.h>
@@ -105,3 +106,5 @@ int update_working_set_xml(pe_working_set_t *data_set, xmlNode **xml);
 int wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib);
 void cli_resource_why(pcmk__output_t *out, cib_t *cib_conn, GListPtr resources,
                       pe_resource_t *rsc, pe_node_t *node);
+
+void crm_resource_register_messages(pcmk__output_t *out);
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
index de1c608..e62122f 100644
--- a/tools/crm_resource_print.c
+++ b/tools/crm_resource_print.c
@@ -9,6 +9,7 @@
 
 #include <crm_resource.h>
 #include <crm/common/xml_internal.h>
+#include <crm/common/output_internal.h>
 
 #define cons_string(x) x?x:"NA"
 void
@@ -82,24 +83,6 @@ cli_resource_print_cts(pcmk__output_t *out, pe_resource_t * rsc)
     }
 }
 
-
-void
-cli_resource_print_raw(pcmk__output_t *out, pe_resource_t * rsc)
-{
-    GListPtr lpc = NULL;
-    GListPtr children = rsc->children;
-
-    if (children == NULL) {
-        printf("%s\n", rsc->id);
-    }
-
-    for (lpc = children; lpc != NULL; lpc = lpc->next) {
-        pe_resource_t *child = (pe_resource_t *) lpc->data;
-
-        cli_resource_print_raw(out, child);
-    }
-}
-
 // \return Standard Pacemaker return code
 int
 cli_resource_print_list(pcmk__output_t *out, pe_working_set_t * data_set, bool raw)
@@ -338,3 +321,47 @@ cli_resource_print_property(pcmk__output_t *out, pe_resource_t *rsc,
     }
     return ENXIO;
 }
+
+static void
+add_resource_name(pcmk__output_t *out, pe_resource_t *rsc) {
+    if (rsc->children == NULL) {
+        out->list_item(out, "resource", "%s", rsc->id);
+    } else {
+        for (GListPtr lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
+            pe_resource_t *child = (pe_resource_t *) lpc->data;
+            add_resource_name(out, child);
+        }
+    }
+}
+
+PCMK__OUTPUT_ARGS("resource-names-list", "GListPtr")
+static int
+resource_names(pcmk__output_t *out, va_list args) {
+    GListPtr resources = va_arg(args, GListPtr);
+
+    if (resources == NULL) {
+        out->err(out, "NO resources configured\n");
+        return pcmk_rc_no_output;
+    }
+
+    out->begin_list(out, NULL, NULL, "Resource Names");
+
+    for (GListPtr lpc = resources; lpc != NULL; lpc = lpc->next) {
+        pe_resource_t *rsc = (pe_resource_t *) lpc->data;
+        add_resource_name(out, rsc);
+    }
+
+    out->end_list(out);
+    return pcmk_rc_ok;
+}
+
+static pcmk__message_entry_t fmt_functions[] = {
+    { "resource-names-list", "default", resource_names },
+
+    { NULL, NULL, NULL }
+};
+
+void
+crm_resource_register_messages(pcmk__output_t *out) {
+    pcmk__register_messages(out, fmt_functions);
+}
-- 
1.8.3.1


From 1d706932132e1b77a991c7bd5b869c593ef355b7 Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Mon, 21 Sep 2020 16:08:51 -0400
Subject: [PATCH 05/19] Feature: tools: Use the existing resource-list message
 in crm_resource.

This replaces cli_resource_print_list with existing formatted output
code, getting rid of one more place where the older style output
functions are still being used.

Note that this does change the format for text output.  The older output
used indentation for displaying the members of a clone and probably
other things.  There's not really a good way to do this with the
existing text output code, short of adding more command line options for
controlling what parts of the list formatting are used.

That seems like a bit much for this one use, so instead just enable the
fancier list formatting if we are listing resources.
---
 cts/cli/regression.tools.exp |  3 ++-
 tools/crm_resource.c         | 20 +++++++++++++++++---
 tools/crm_resource.h         |  1 -
 tools/crm_resource_print.c   | 28 ----------------------------
 4 files changed, 19 insertions(+), 33 deletions(-)

diff --git a/cts/cli/regression.tools.exp b/cts/cli/regression.tools.exp
index 35e7a8c..10ec53b 100644
--- a/cts/cli/regression.tools.exp
+++ b/cts/cli/regression.tools.exp
@@ -926,7 +926,8 @@ Set 'dummy' option: id=dummy-instance_attributes-delay set=dummy-instance_attrib
 =#=#=#= End test: Create a resource attribute - OK (0) =#=#=#=
 * Passed: crm_resource   - Create a resource attribute
 =#=#=#= Begin test: List the configured resources =#=#=#=
- dummy	(ocf::pacemaker:Dummy):	 Stopped
+Full List of Resources:
+  * dummy	(ocf::pacemaker:Dummy):	 Stopped
 =#=#=#= Current cib after: List the configured resources =#=#=#=
 <cib epoch="19" num_updates="0" admin_epoch="1">
   <configuration>
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index d9e8e70..2d71e7a 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1610,8 +1610,13 @@ main(int argc, char **argv)
         } else {
             pcmk__force_args(context, &error, "%s --xml-substitute", g_get_prgname());
         }
+    } else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches)) {
+        if (options.rsc_cmd == cmd_list_resources) {
+            pcmk__force_args(context, &error, "%s --text-fancy", g_get_prgname());
+        }
     }
 
+    pe__register_messages(out);
     crm_resource_register_messages(out);
 
     if (args->version) {
@@ -1715,10 +1720,19 @@ main(int argc, char **argv)
     }
 
     switch (options.rsc_cmd) {
-        case cmd_list_resources:
-            rc = pcmk_rc_ok;
-            cli_resource_print_list(out, data_set, FALSE);
+        case cmd_list_resources: {
+            GListPtr all = NULL;
+            all = g_list_prepend(all, strdup("*"));
+            rc = out->message(out, "resource-list", data_set,
+                              pe_print_rsconly | pe_print_pending,
+                              FALSE, TRUE, FALSE, TRUE, all, all, FALSE);
+            g_list_free_full(all, free);
+
+            if (rc == pcmk_rc_no_output) {
+                rc = ENXIO;
+            }
             break;
+        }
 
         case cmd_list_instances:
             rc = out->message(out, "resource-names-list", data_set->resources);
diff --git a/tools/crm_resource.h b/tools/crm_resource.h
index 28a3760..6b6dab2 100644
--- a/tools/crm_resource.h
+++ b/tools/crm_resource.h
@@ -46,7 +46,6 @@ void cli_resource_print_colocation(pcmk__output_t *out, pe_resource_t * rsc,
 
 int cli_resource_print(pcmk__output_t *out, pe_resource_t *rsc, pe_working_set_t *data_set,
                        bool expanded);
-int cli_resource_print_list(pcmk__output_t *out, pe_working_set_t * data_set, bool raw);
 int cli_resource_print_attribute(pcmk__output_t *out, pe_resource_t *rsc,
                                  const char *attr, const char *attr_set_type,
                                  pe_working_set_t *data_set);
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
index e62122f..f7356fb 100644
--- a/tools/crm_resource_print.c
+++ b/tools/crm_resource_print.c
@@ -85,34 +85,6 @@ cli_resource_print_cts(pcmk__output_t *out, pe_resource_t * rsc)
 
 // \return Standard Pacemaker return code
 int
-cli_resource_print_list(pcmk__output_t *out, pe_working_set_t * data_set, bool raw)
-{
-    int found = 0;
-
-    GListPtr lpc = NULL;
-    int opts = pe_print_printf | pe_print_rsconly | pe_print_pending;
-
-    for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
-        pe_resource_t *rsc = (pe_resource_t *) lpc->data;
-
-        if (pcmk_is_set(rsc->flags, pe_rsc_orphan)
-            && rsc->fns->active(rsc, TRUE) == FALSE) {
-            continue;
-        }
-        rsc->fns->print(rsc, NULL, opts, stdout);
-        found++;
-    }
-
-    if (found == 0) {
-        printf("NO resources configured\n");
-        return ENXIO;
-    }
-
-    return pcmk_rc_ok;
-}
-
-// \return Standard Pacemaker return code
-int
 cli_resource_print_operations(pcmk__output_t *out, const char *rsc_id,
                               const char *host_uname, bool active,
                               pe_working_set_t * data_set)
-- 
1.8.3.1


From 86603abc8785e853b85251f431fe86ca28bb35df Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Mon, 21 Sep 2020 16:11:21 -0400
Subject: [PATCH 06/19] Feature: liblrmd: Add output messages for agents,
 providers, and standards.

And use these messages in crm_resource.  For XML output, standards use
the basic list formatting with <list> and <item> tags.  All other
messages use their own custom lists, because those need to include
additional information.
---
 include/crm/lrmd_internal.h |   3 +
 lib/lrmd/Makefile.am        |   4 +-
 lib/lrmd/lrmd_output.c      | 145 ++++++++++++++++++++++++++++++++++++++++++++
 tools/crm_resource.c        |  78 +++++++++++++++---------
 4 files changed, 198 insertions(+), 32 deletions(-)
 create mode 100644 lib/lrmd/lrmd_output.c

diff --git a/include/crm/lrmd_internal.h b/include/crm/lrmd_internal.h
index 498a9ba..720e1a3 100644
--- a/include/crm/lrmd_internal.h
+++ b/include/crm/lrmd_internal.h
@@ -15,6 +15,7 @@
 #include <libxml/tree.h>                // xmlNode
 #include <crm/common/ipc.h>             // crm_ipc_t
 #include <crm/common/mainloop.h>        // mainloop_io_t, ipc_client_callbacks
+#include <crm/common/output_internal.h> // pcmk__output_t
 #include <crm/common/remote_internal.h> // pcmk__remote_t
 #include <crm/lrmd.h>                   // lrmd_t, lrmd_event_data_t
 
@@ -66,4 +67,6 @@ void remote_proxy_relay_event(remote_proxy_t *proxy, xmlNode *msg);
 void remote_proxy_relay_response(remote_proxy_t *proxy, xmlNode *msg,
                                  int msg_id);
 
+void lrmd__register_messages(pcmk__output_t *out);
+
 #endif
diff --git a/lib/lrmd/Makefile.am b/lib/lrmd/Makefile.am
index 41ba7fd..09e40b1 100644
--- a/lib/lrmd/Makefile.am
+++ b/lib/lrmd/Makefile.am
@@ -1,5 +1,5 @@
 #
-# Copyright 2012-2018 the Pacemaker project contributors
+# Copyright 2012-2020 the Pacemaker project contributors
 #
 # The version control history for this file may have further details.
 #
@@ -18,4 +18,4 @@ liblrmd_la_LDFLAGS	+= $(LDFLAGS_HARDENED_LIB)
 liblrmd_la_LIBADD	= $(top_builddir)/lib/common/libcrmcommon.la	\
 			$(top_builddir)/lib/services/libcrmservice.la	\
 			$(top_builddir)/lib/fencing/libstonithd.la
-liblrmd_la_SOURCES	= lrmd_client.c proxy_common.c lrmd_alerts.c
+liblrmd_la_SOURCES	= lrmd_client.c proxy_common.c lrmd_alerts.c lrmd_output.c
diff --git a/lib/lrmd/lrmd_output.c b/lib/lrmd/lrmd_output.c
new file mode 100644
index 0000000..7dc0709
--- /dev/null
+++ b/lib/lrmd/lrmd_output.c
@@ -0,0 +1,145 @@
+/*
+ * Copyright 2020 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include <stdarg.h>
+
+#include <crm/lrmd_internal.h>
+#include <crm/common/output_internal.h>
+
+static int
+default_list(pcmk__output_t *out, lrmd_list_t *list, const char *title) {
+    lrmd_list_t *iter = NULL;
+
+    out->begin_list(out, NULL, NULL, "%s", title);
+
+    for (iter = list; iter != NULL; iter = iter->next) {
+        out->list_item(out, NULL, "%s", iter->val);
+    }
+
+    out->end_list(out);
+    lrmd_list_freeall(list);
+    return pcmk_rc_ok;
+}
+
+static int
+xml_list(pcmk__output_t *out, lrmd_list_t *list, const char *ele) {
+    lrmd_list_t *iter = NULL;
+
+    for (iter = list; iter != NULL; iter = iter->next) {
+        pcmk__output_create_xml_text_node(out, ele, iter->val);
+    }
+
+    lrmd_list_freeall(list);
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("alternatives-list", "lrmd_list_t *", "const char *")
+static int
+lrmd__alternatives_list_xml(pcmk__output_t *out, va_list args) {
+    lrmd_list_t *list = va_arg(args, lrmd_list_t *);
+    const char *agent_spec = va_arg(args, const char *);
+
+    xmlNodePtr node = pcmk__output_xml_create_parent(out, "providers");
+
+    xmlSetProp(node, (pcmkXmlStr) "for", (pcmkXmlStr) agent_spec);
+    return xml_list(out, list, "provider");
+}
+
+PCMK__OUTPUT_ARGS("alternatives-list", "lrmd_list_t *", "const char *")
+static int
+lrmd__alternatives_list(pcmk__output_t *out, va_list args) {
+    lrmd_list_t *list = va_arg(args, lrmd_list_t *);
+    const char *agent_spec G_GNUC_UNUSED = va_arg(args, const char *);
+
+    return default_list(out, list, "Providers");
+}
+
+PCMK__OUTPUT_ARGS("agents-list", "lrmd_list_t *", "const char *", "char *")
+static int
+lrmd__agents_list_xml(pcmk__output_t *out, va_list args) {
+    lrmd_list_t *list = va_arg(args, lrmd_list_t *);
+    const char *agent_spec = va_arg(args, const char *);
+    char *provider = va_arg(args, char *);
+
+    xmlNodePtr node = pcmk__output_xml_create_parent(out, "agents");
+    xmlSetProp(node, (pcmkXmlStr) "standard", (pcmkXmlStr) agent_spec);
+
+    if (!pcmk__str_empty(provider)) {
+        xmlSetProp(node, (pcmkXmlStr) "provider", (pcmkXmlStr) provider);
+    }
+
+    return xml_list(out, list, "agent");
+}
+
+PCMK__OUTPUT_ARGS("agents-list", "lrmd_list_t *", "const char *", "char *")
+static int
+lrmd__agents_list(pcmk__output_t *out, va_list args) {
+    lrmd_list_t *list = va_arg(args, lrmd_list_t *);
+    const char *agent_spec = va_arg(args, const char *);
+    char *provider = va_arg(args, char *);
+
+    int rc;
+    char *title = crm_strdup_printf("%s agents", pcmk__str_empty(provider) ? agent_spec : provider);
+
+    rc = default_list(out, list, title);
+    free(title);
+    return rc;
+}
+
+PCMK__OUTPUT_ARGS("providers-list", "lrmd_list_t *", "const char *")
+static int
+lrmd__providers_list_xml(pcmk__output_t *out, va_list args) {
+    lrmd_list_t *list = va_arg(args, lrmd_list_t *);
+    const char *agent_spec = va_arg(args, const char *);
+
+    xmlNodePtr node = pcmk__output_xml_create_parent(out, "providers");
+
+    xmlSetProp(node, (pcmkXmlStr) "standard", (pcmkXmlStr) "ocf");
+
+    if (agent_spec != NULL) {
+        xmlSetProp(node, (pcmkXmlStr) "agent", (pcmkXmlStr) agent_spec);
+    }
+
+    return xml_list(out, list, "provider");
+}
+
+PCMK__OUTPUT_ARGS("providers-list", "lrmd_list_t *", "const char *")
+static int
+lrmd__providers_list(pcmk__output_t *out, va_list args) {
+    lrmd_list_t *list = va_arg(args, lrmd_list_t *);
+    const char *agent_spec G_GNUC_UNUSED = va_arg(args, const char *);
+
+    return default_list(out, list, "Providers");
+}
+
+PCMK__OUTPUT_ARGS("standards-list", "lrmd_list_t *")
+static int
+lrmd__standards_list(pcmk__output_t *out, va_list args) {
+    lrmd_list_t *list = va_arg(args, lrmd_list_t *);
+
+    return default_list(out, list, "Standards");
+}
+
+static pcmk__message_entry_t fmt_functions[] = {
+    { "alternatives-list", "default", lrmd__alternatives_list },
+    { "alternatives-list", "xml", lrmd__alternatives_list_xml },
+    { "agents-list", "default", lrmd__agents_list },
+    { "agents-list", "xml", lrmd__agents_list_xml },
+    { "providers-list", "default", lrmd__providers_list },
+    { "providers-list", "xml", lrmd__providers_list_xml },
+    { "standards-list", "default", lrmd__standards_list },
+
+    { NULL, NULL, NULL }
+};
+
+void
+lrmd__register_messages(pcmk__output_t *out) {
+    pcmk__register_messages(out, fmt_functions);
+}
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index 2d71e7a..df9c623 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -8,6 +8,7 @@
  */
 
 #include <crm_resource.h>
+#include <crm/lrmd_internal.h>
 #include <crm/common/cmdline_internal.h>
 #include <crm/common/lists_internal.h>
 #include <pacemaker-internal.h>
@@ -1092,25 +1093,24 @@ static int
 list_agents(pcmk__output_t *out, const char *agent_spec, crm_exit_t *exit_code)
 {
     int rc = pcmk_rc_ok;
-    lrmd_list_t *list = NULL;
-    lrmd_list_t *iter = NULL;
     char *provider = strchr(agent_spec, ':');
     lrmd_t *lrmd_conn = lrmd_api_new();
+    lrmd_list_t *list = NULL;
 
     if (provider) {
         *provider++ = 0;
     }
+
     rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, agent_spec, provider);
 
     if (rc > 0) {
-        for (iter = list; iter != NULL; iter = iter->next) {
-            printf("%s\n", iter->val);
-        }
-        lrmd_list_freeall(list);
-        rc = pcmk_rc_ok;
+        rc = out->message(out, "agents-list", list, agent_spec, provider);
     } else {
-        *exit_code = CRM_EX_NOSUCH;
         rc = pcmk_rc_error;
+    }
+
+    if (rc != pcmk_rc_ok) {
+        *exit_code = CRM_EX_NOSUCH;
         if (provider == NULL) {
             g_set_error(&error, PCMK__EXITC_ERROR, *exit_code,
                         "No agents found for standard '%s'", agent_spec);
@@ -1130,19 +1130,41 @@ list_providers(pcmk__output_t *out, const char *agent_spec, crm_exit_t *exit_cod
 {
     int rc;
     const char *text = NULL;
-    lrmd_list_t *list = NULL;
-    lrmd_list_t *iter = NULL;
     lrmd_t *lrmd_conn = lrmd_api_new();
+    lrmd_list_t *list = NULL;
 
     switch (options.rsc_cmd) {
+        case cmd_list_alternatives:
+            rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, agent_spec, &list);
+
+            if (rc > 0) {
+                rc = out->message(out, "alternatives-list", list, agent_spec);
+            } else {
+                rc = pcmk_rc_error;
+            }
+
+            text = "OCF providers";
+            break;
         case cmd_list_standards:
             rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);
+
+            if (rc > 0) {
+                rc = out->message(out, "standards-list", list);
+            } else {
+                rc = pcmk_rc_error;
+            }
+
             text = "standards";
             break;
         case cmd_list_providers:
-        case cmd_list_alternatives:
-            rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, agent_spec,
-                                                     &list);
+            rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, agent_spec, &list);
+
+            if (rc > 0) {
+                rc = out->message(out, "providers-list", list, agent_spec);
+            } else {
+                rc = pcmk_rc_error;
+            }
+
             text = "OCF providers";
             break;
         default:
@@ -1152,24 +1174,19 @@ list_providers(pcmk__output_t *out, const char *agent_spec, crm_exit_t *exit_cod
             return pcmk_rc_error;
     }
 
-    if (rc > 0) {
-        for (iter = list; iter != NULL; iter = iter->next) {
-            printf("%s\n", iter->val);
-        }
-        lrmd_list_freeall(list);
-        rc = pcmk_rc_ok;
-
-    } else if (agent_spec != NULL) {
-        *exit_code = CRM_EX_NOSUCH;
-        rc = pcmk_rc_error;
-        g_set_error(&error, PCMK__EXITC_ERROR, *exit_code,
-                    "No %s found for %s", text, agent_spec);
+    if (rc != pcmk_rc_ok) {
+        if (agent_spec != NULL) {
+            *exit_code = CRM_EX_NOSUCH;
+            rc = pcmk_rc_error;
+            g_set_error(&error, PCMK__EXITC_ERROR, *exit_code,
+                        "No %s found for %s", text, agent_spec);
 
-    } else {
-        *exit_code = CRM_EX_NOSUCH;
-        rc = pcmk_rc_error;
-        g_set_error(&error, PCMK__EXITC_ERROR, *exit_code,
-                    "No %s found", text);
+        } else {
+            *exit_code = CRM_EX_NOSUCH;
+            rc = pcmk_rc_error;
+            g_set_error(&error, PCMK__EXITC_ERROR, *exit_code,
+                        "No %s found", text);
+        }
     }
 
     lrmd_api_delete(lrmd_conn);
@@ -1618,6 +1635,7 @@ main(int argc, char **argv)
 
     pe__register_messages(out);
     crm_resource_register_messages(out);
+    lrmd__register_messages(out);
 
     if (args->version) {
         out->version(out, false);
-- 
1.8.3.1


From 64336b4ff485d6ddcadf6a5b6235084fd7d85101 Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Mon, 21 Sep 2020 16:44:04 -0400
Subject: [PATCH 07/19] Feature: tools: Use formatted output for props, attrs,
 and metadata.

For the most part, these can just be built out of the basic formatted
output tools.  The text versions exist separately to preserve the old
output style of crm_resource.
---
 tools/crm_resource.c       | 40 ++++++++++++++++---
 tools/crm_resource.h       |  5 ---
 tools/crm_resource_print.c | 95 +++++++++++++++++++++++++++-------------------
 3 files changed, 90 insertions(+), 50 deletions(-)

diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index df9c623..dcb769f 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1378,7 +1378,7 @@ show_metadata(pcmk__output_t *out, const char *agent_spec, crm_exit_t *exit_code
         rc = pcmk_legacy2rc(rc);
 
         if (metadata) {
-            printf("%s\n", metadata);
+            out->output_xml(out, "metadata", metadata);
         } else {
             *exit_code = crm_errno2exit(rc);
             g_set_error(&error, PCMK__EXITC_ERROR, *exit_code,
@@ -1904,17 +1904,47 @@ main(int argc, char **argv)
             break;
 
         case cmd_get_property:
-            rc = cli_resource_print_property(out, rsc, options.prop_name, data_set);
+            rc = out->message(out, "property", rsc, options.prop_name);
+            if (rc == pcmk_rc_no_output) {
+                rc = ENXIO;
+            }
+
             break;
 
         case cmd_set_property:
             rc = set_property();
             break;
 
-        case cmd_get_param:
-            rc = cli_resource_print_attribute(out, rsc, options.prop_name,
-                                              options.attr_set_type, data_set);
+        case cmd_get_param: {
+            unsigned int count = 0;
+            GHashTable *params = NULL;
+            pe_node_t *current = pe__find_active_on(rsc, &count, NULL);
+
+            if (count > 1) {
+                out->err(out, "%s is active on more than one node,"
+                         " returning the default value for %s", rsc->id, crm_str(options.prop_name));
+                current = NULL;
+            }
+
+            params = crm_str_table_new();
+
+            if (pcmk__str_eq(options.attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
+                get_rsc_attributes(params, rsc, current, data_set);
+
+            } else if (pcmk__str_eq(options.attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
+                /* No need to redirect to the parent */
+                get_meta_attributes(params, rsc, current, data_set);
+
+            } else {
+                pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_UTILIZATION, NULL, params,
+                                           NULL, FALSE, data_set);
+            }
+
+            crm_debug("Looking up %s in %s", options.prop_name, rsc->id);
+            rc = out->message(out, "attribute", rsc, options.prop_name, params);
+            g_hash_table_destroy(params);
             break;
+        }
 
         case cmd_set_param:
             if (pcmk__str_empty(options.prop_value)) {
diff --git a/tools/crm_resource.h b/tools/crm_resource.h
index 6b6dab2..4fc7c71 100644
--- a/tools/crm_resource.h
+++ b/tools/crm_resource.h
@@ -46,11 +46,6 @@ void cli_resource_print_colocation(pcmk__output_t *out, pe_resource_t * rsc,
 
 int cli_resource_print(pcmk__output_t *out, pe_resource_t *rsc, pe_working_set_t *data_set,
                        bool expanded);
-int cli_resource_print_attribute(pcmk__output_t *out, pe_resource_t *rsc,
-                                 const char *attr, const char *attr_set_type,
-                                 pe_working_set_t *data_set);
-int cli_resource_print_property(pcmk__output_t *out, pe_resource_t *rsc, const char *attr,
-                                pe_working_set_t *data_set);
 int cli_resource_print_operations(pcmk__output_t *out, const char *rsc_id,
                                   const char *host_uname, bool active,
                                   pe_working_set_t * data_set);
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
index f7356fb..093eb75 100644
--- a/tools/crm_resource_print.c
+++ b/tools/crm_resource_print.c
@@ -235,63 +235,74 @@ cli_resource_print(pcmk__output_t *out, pe_resource_t *rsc,
     return pcmk_rc_ok;
 }
 
-// \return Standard Pacemaker return code
-int
-cli_resource_print_attribute(pcmk__output_t *out, pe_resource_t *rsc, const char *attr,
-                             const char *attr_set_type, pe_working_set_t * data_set)
-{
-    int rc = ENXIO;
-    unsigned int count = 0;
-    GHashTable *params = NULL;
-    const char *value = NULL;
-    pe_node_t *current = pe__find_active_on(rsc, &count, NULL);
-
-    if (count > 1) {
-        CMD_ERR("%s is active on more than one node,"
-                " returning the default value for %s", rsc->id, crm_str(attr));
-        current = NULL;
+PCMK__OUTPUT_ARGS("attribute", "pe_resource_t *", "char *", "GHashTable *")
+static int
+attribute_default(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    char *attr = va_arg(args, char *);
+    GHashTable *params = va_arg(args, GHashTable *);
+
+    const char *value = g_hash_table_lookup(params, attr);
+
+    if (value != NULL) {
+        out->begin_list(out, NULL, NULL, "Attributes");
+        out->list_item(out, attr, "%s", value);
+        out->end_list(out);
+    } else {
+        out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
     }
 
-    params = crm_str_table_new();
+    return pcmk_rc_ok;
+}
 
-    if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
-        get_rsc_attributes(params, rsc, current, data_set);
+PCMK__OUTPUT_ARGS("attribute", "pe_resource_t *", "char *", "GHashTable *")
+static int
+attribute_text(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    char *attr = va_arg(args, char *);
+    GHashTable *params = va_arg(args, GHashTable *);
 
-    } else if (pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
-        /* No need to redirect to the parent */
-        get_meta_attributes(params, rsc, current, data_set);
+    const char *value = g_hash_table_lookup(params, attr);
 
+    if (value != NULL) {
+        out->info(out, "%s", value);
     } else {
-        pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_UTILIZATION, NULL, params,
-                                   NULL, FALSE, data_set);
+        out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
     }
 
-    crm_debug("Looking up %s in %s", attr, rsc->id);
-    value = g_hash_table_lookup(params, attr);
-    if (value != NULL) {
-        fprintf(stdout, "%s\n", value);
-        rc = pcmk_rc_ok;
+    return pcmk_rc_ok;
+}
 
-    } else {
-        CMD_ERR("Attribute '%s' not found for '%s'", attr, rsc->id);
+PCMK__OUTPUT_ARGS("property", "pe_resource_t *", "char *")
+static int
+property_default(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    char *attr = va_arg(args, char *);
+
+    const char *value = crm_element_value(rsc->xml, attr);
+
+    if (value != NULL) {
+        out->begin_list(out, NULL, NULL, "Properties");
+        out->list_item(out, attr, "%s", value);
+        out->end_list(out);
     }
 
-    g_hash_table_destroy(params);
-    return rc;
+    return pcmk_rc_ok;
 }
 
-// \return Standard Pacemaker return code
-int
-cli_resource_print_property(pcmk__output_t *out, pe_resource_t *rsc,
-                            const char *attr, pe_working_set_t * data_set)
-{
+PCMK__OUTPUT_ARGS("property", "pe_resource_t *", "char *")
+static int
+property_text(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    char *attr = va_arg(args, char *);
+
     const char *value = crm_element_value(rsc->xml, attr);
 
     if (value != NULL) {
-        fprintf(stdout, "%s\n", value);
-        return pcmk_rc_ok;
+        out->info(out, "%s", value);
     }
-    return ENXIO;
+
+    return pcmk_rc_ok;
 }
 
 static void
@@ -328,6 +339,10 @@ resource_names(pcmk__output_t *out, va_list args) {
 }
 
 static pcmk__message_entry_t fmt_functions[] = {
+    { "attribute", "default", attribute_default },
+    { "attribute", "text", attribute_text },
+    { "property", "default", property_default },
+    { "property", "text", property_text },
     { "resource-names-list", "default", resource_names },
 
     { NULL, NULL, NULL }
-- 
1.8.3.1


From 575c2231baf11f24d953508000780a88ce7ad57c Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Mon, 26 Oct 2020 10:09:16 -0400
Subject: [PATCH 08/19] Feature: scheduler: Add a message for resource config
 printing.

This prints out the XML or raw XML config for a given resource.
---
 cts/cli/regression.tools.exp   |  2 +-
 include/crm/pengine/internal.h |  1 +
 lib/pengine/pe_output.c        | 21 +++++++++++++++++++++
 tools/crm_resource_print.c     |  7 ++-----
 4 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/cts/cli/regression.tools.exp b/cts/cli/regression.tools.exp
index 10ec53b..738e800 100644
--- a/cts/cli/regression.tools.exp
+++ b/cts/cli/regression.tools.exp
@@ -972,7 +972,7 @@ dummy
 * Passed: crm_resource   - List IDs of instantiated resources
 =#=#=#= Begin test: Show XML configuration of resource =#=#=#=
  dummy	(ocf::pacemaker:Dummy):	 Stopped
-xml:
+Resource XML:
 <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
   <meta_attributes id="dummy-meta_attributes"/>
   <instance_attributes id="dummy-instance_attributes">
diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
index d658e86..00b6b4c 100644
--- a/include/crm/pengine/internal.h
+++ b/include/crm/pengine/internal.h
@@ -301,6 +301,7 @@ int pe__node_list_text(pcmk__output_t *out, va_list args);
 int pe__node_list_xml(pcmk__output_t *out, va_list args);
 int pe__op_history_text(pcmk__output_t *out, va_list args);
 int pe__op_history_xml(pcmk__output_t *out, va_list args);
+int pe__resource_config(pcmk__output_t *out, va_list args);
 int pe__resource_history_text(pcmk__output_t *out, va_list args);
 int pe__resource_history_xml(pcmk__output_t *out, va_list args);
 int pe__resource_xml(pcmk__output_t *out, va_list args);
diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
index 9d43e5f..dd3a880 100644
--- a/lib/pengine/pe_output.c
+++ b/lib/pengine/pe_output.c
@@ -1583,6 +1583,26 @@ pe__op_history_xml(pcmk__output_t *out, va_list args) {
     return pcmk_rc_ok;
 }
 
+PCMK__OUTPUT_ARGS("resource-config", "pe_resource_t *", "gboolean")
+int pe__resource_config(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    gboolean raw = va_arg(args, gboolean);
+
+    char *rsc_xml = NULL;
+
+    if (raw) {
+        rsc_xml = dump_xml_formatted(rsc->orig_xml ? rsc->orig_xml : rsc->xml);
+    } else {
+        rsc_xml = dump_xml_formatted(rsc->xml);
+    }
+
+    out->info(out, "Resource XML:");
+    out->output_xml(out, "xml", rsc_xml);
+
+    free(rsc_xml);
+    return pcmk_rc_ok;
+}
+
 PCMK__OUTPUT_ARGS("resource-history", "pe_resource_t *", "const char *", "gboolean", "int", "time_t", "gboolean")
 int
 pe__resource_history_text(pcmk__output_t *out, va_list args) {
@@ -1872,6 +1892,7 @@ static pcmk__message_entry_t fmt_functions[] = {
     { "primitive", "html",  pe__resource_html },
     { "primitive", "text",  pe__resource_text },
     { "primitive", "log",  pe__resource_text },
+    { "resource-config", "default", pe__resource_config },
     { "resource-history", "default", pe__resource_history_text },
     { "resource-history", "xml", pe__resource_history_xml },
     { "resource-list", "default", pe__resource_list },
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
index 093eb75..99217aa 100644
--- a/tools/crm_resource_print.c
+++ b/tools/crm_resource_print.c
@@ -223,15 +223,12 @@ int
 cli_resource_print(pcmk__output_t *out, pe_resource_t *rsc,
                    pe_working_set_t *data_set, bool expanded)
 {
-    char *rsc_xml = NULL;
     int opts = pe_print_printf | pe_print_pending;
 
     rsc->fns->print(rsc, NULL, opts, stdout);
 
-    rsc_xml = dump_xml_formatted((!expanded && rsc->orig_xml)?
-                                 rsc->orig_xml : rsc->xml);
-    fprintf(stdout, "%sxml:\n%s\n", expanded ? "" : "raw ", rsc_xml);
-    free(rsc_xml);
+    out->message(out, "resource-config", rsc, !expanded);
+
     return pcmk_rc_ok;
 }
 
-- 
1.8.3.1


From 711d9412de519ac897ebe51f79e6bf47275eb5ab Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Thu, 27 Aug 2020 10:11:51 -0400
Subject: [PATCH 09/19] Feature: tools: Use formatted output for resource
 output in crm_resource.

This switches crm_resource away from using the old style resource
printing functions and onto using formatted output.
---
 cts/cli/regression.tools.exp | 3 +--
 lib/common/output_xml.c      | 1 +
 tools/crm_resource_print.c   | 9 +++++++--
 tools/crm_resource_runtime.c | 2 ++
 4 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/cts/cli/regression.tools.exp b/cts/cli/regression.tools.exp
index 738e800..4abbb75 100644
--- a/cts/cli/regression.tools.exp
+++ b/cts/cli/regression.tools.exp
@@ -971,7 +971,7 @@ dummy
 =#=#=#= End test: List IDs of instantiated resources - OK (0) =#=#=#=
 * Passed: crm_resource   - List IDs of instantiated resources
 =#=#=#= Begin test: Show XML configuration of resource =#=#=#=
- dummy	(ocf::pacemaker:Dummy):	 Stopped
+dummy	(ocf::pacemaker:Dummy):	 Stopped
 Resource XML:
 <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
   <meta_attributes id="dummy-meta_attributes"/>
@@ -979,7 +979,6 @@ Resource XML:
     <nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
   </instance_attributes>
 </primitive>
-
 =#=#=#= End test: Show XML configuration of resource - OK (0) =#=#=#=
 * Passed: crm_resource   - Show XML configuration of resource
 =#=#=#= Begin test: Require a destination when migrating a resource that is stopped =#=#=#=
diff --git a/lib/common/output_xml.c b/lib/common/output_xml.c
index 6a6ed6e..bba21e7 100644
--- a/lib/common/output_xml.c
+++ b/lib/common/output_xml.c
@@ -60,6 +60,7 @@ static subst_t substitutions[] = {
     { "Operations",                     "node_history" },
     { "Negative Location Constraints",  "bans" },
     { "Node Attributes",                "node_attributes" },
+    { "Resource Config",                "resource_config" },
 
     { NULL, NULL }
 };
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
index 99217aa..447c57d 100644
--- a/tools/crm_resource_print.c
+++ b/tools/crm_resource_print.c
@@ -223,12 +223,17 @@ int
 cli_resource_print(pcmk__output_t *out, pe_resource_t *rsc,
                    pe_working_set_t *data_set, bool expanded)
 {
-    int opts = pe_print_printf | pe_print_pending;
+    unsigned int opts = pe_print_pending;
+    GListPtr all = NULL;
 
-    rsc->fns->print(rsc, NULL, opts, stdout);
+    all = g_list_prepend(all, strdup("*"));
 
+    out->begin_list(out, NULL, NULL, "Resource Config");
+    out->message(out, crm_map_element_name(rsc->xml), opts, rsc, all, all);
     out->message(out, "resource-config", rsc, !expanded);
+    out->end_list(out);
 
+    g_list_free_full(all, free);
     return pcmk_rc_ok;
 }
 
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 3e1f985..7aae8cc 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -142,6 +142,8 @@ find_resource_attr(pcmk__output_t *out, cib_t * the_cib, const char *attr,
                    crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
         }
 
+        out->spacer(out);
+
     } else if(value) {
         const char *tmp = crm_element_value(xml_search, attr);
 
-- 
1.8.3.1


From b89fa86c642d33ddfd9db6015bb3e74f6645623c Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Thu, 27 Aug 2020 10:29:00 -0400
Subject: [PATCH 10/19] Feature: tools: Use formatted output for finding
 resources.

---
 tools/crm_resource_runtime.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 7aae8cc..dca9553 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -23,21 +23,21 @@ do_find_resource(pcmk__output_t *out, const char *rsc, pe_resource_t * the_rsc,
         pe_node_t *node = (pe_node_t *) lpc->data;
 
         if (out->is_quiet(out)) {
-            fprintf(stdout, "%s\n", node->details->uname);
+            out->info(out, "%s", node->details->uname);
         } else {
             const char *state = "";
 
             if (!pe_rsc_is_clone(the_rsc) && the_rsc->fns->state(the_rsc, TRUE) == RSC_ROLE_MASTER) {
                 state = "Master";
             }
-            fprintf(stdout, "resource %s is running on: %s %s\n", rsc, node->details->uname, state);
+            out->info(out, "resource %s is running on: %s %s", rsc, node->details->uname, state);
         }
 
         found++;
     }
 
     if (!out->is_quiet(out) && found == 0) {
-        fprintf(stderr, "resource %s is NOT running\n", rsc);
+        out->err(out, "resource %s is NOT running", rsc);
     }
 
     return found;
-- 
1.8.3.1


From ed030eead6b704400fe6362c5b4a320798675995 Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Tue, 20 Oct 2020 12:59:25 -0400
Subject: [PATCH 11/19] Feature: tools: Use subprocess_output for crm_resource
 execution results.

---
 tools/crm_resource_runtime.c | 32 ++------------------------------
 1 file changed, 2 insertions(+), 30 deletions(-)

diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index dca9553..22b469e 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -1772,9 +1772,6 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
     }
 
     if (services_action_sync(op)) {
-        int more, lpc, last;
-        char *local_copy = NULL;
-
         exit_code = op->rc;
 
         if (op->status == PCMK_LRM_OP_DONE) {
@@ -1791,33 +1788,8 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
         if (resource_verbose == 0 && pcmk__str_eq(action, "validate-all", pcmk__str_casei))
             goto done;
 
-        if (op->stdout_data) {
-            local_copy = strdup(op->stdout_data);
-            more = strlen(local_copy);
-            last = 0;
-
-            for (lpc = 0; lpc < more; lpc++) {
-                if (local_copy[lpc] == '\n' || local_copy[lpc] == 0) {
-                    local_copy[lpc] = 0;
-                    printf(" >  stdout: %s\n", local_copy + last);
-                    last = lpc + 1;
-                }
-            }
-            free(local_copy);
-        }
-        if (op->stderr_data) {
-            local_copy = strdup(op->stderr_data);
-            more = strlen(local_copy);
-            last = 0;
-
-            for (lpc = 0; lpc < more; lpc++) {
-                if (local_copy[lpc] == '\n' || local_copy[lpc] == 0) {
-                    local_copy[lpc] = 0;
-                    printf(" >  stderr: %s\n", local_copy + last);
-                    last = lpc + 1;
-                }
-            }
-            free(local_copy);
+        if (op->stdout_data || op->stderr_data) {
+            out->subprocess_output(out, op->rc, op->stdout_data, op->stderr_data);
         }
     } else {
         exit_code = op->rc == 0 ? CRM_EX_ERROR : op->rc;
-- 
1.8.3.1


From fb9792d7b99347b82b51a281ba10b0349ee45d5d Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Thu, 27 Aug 2020 11:29:11 -0400
Subject: [PATCH 12/19] Feature: tools: Use formatted output for crm_resource
 info messages.

Basically, anything that's being printed out for informational purposes
should use out->info.  This won't show up in the XML format, but that's
never existed for crm_resource in the first place.

Also, errors (most things printed to stderr, uses of CMD_ERR, etc.)
should use out->err.
---
 cts/cli/regression.tools.exp |   2 +-
 tools/crm_resource.c         |  24 ++---
 tools/crm_resource_ban.c     |  32 +++----
 tools/crm_resource_runtime.c | 223 +++++++++++++++++++++----------------------
 4 files changed, 137 insertions(+), 144 deletions(-)

diff --git a/cts/cli/regression.tools.exp b/cts/cli/regression.tools.exp
index 4abbb75..935dce8 100644
--- a/cts/cli/regression.tools.exp
+++ b/cts/cli/regression.tools.exp
@@ -3223,10 +3223,10 @@ Migration will take effect until:
 =#=#=#= End test: Try to move a resource previously moved with a lifetime - OK (0) =#=#=#=
 * Passed: crm_resource   - Try to move a resource previously moved with a lifetime
 =#=#=#= Begin test: Ban dummy from node1 for a short time =#=#=#=
+Migration will take effect until:
 WARNING: Creating rsc_location constraint 'cli-ban-dummy-on-node1' with a score of -INFINITY for resource dummy on node1.
 	This will prevent dummy from running on node1 until the constraint is removed using the clear option or by editing the CIB with an appropriate tool
 	This will be the case even if node1 is the last node in the cluster
-Migration will take effect until:
 =#=#=#= Current cib after: Ban dummy from node1 for a short time =#=#=#=
 <cib epoch="53" num_updates="0" admin_epoch="1">
   <configuration>
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index dcb769f..0532095 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -238,7 +238,7 @@ resource_ipc_timeout(gpointer data)
     }
 
     g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_TIMEOUT,
-                "\nAborting because no messages received in %d seconds", MESSAGE_TIMEOUT_S);
+                "Aborting because no messages received in %d seconds", MESSAGE_TIMEOUT_S);
 
     quit_main_loop(CRM_EX_TIMEOUT);
     return FALSE;
@@ -258,18 +258,19 @@ controller_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
 
         case pcmk_ipc_event_reply:
             if (status != CRM_EX_OK) {
-                fprintf(stderr, "\nError: bad reply from controller: %s\n",
-                        crm_exit_str(status));
+                out->err(out, "Error: bad reply from controller: %s",
+                         crm_exit_str(status));
                 pcmk_disconnect_ipc(api);
                 quit_main_loop(status);
             } else {
-                fprintf(stderr, ".");
                 if ((pcmk_controld_api_replies_expected(api) == 0)
                     && mainloop && g_main_loop_is_running(mainloop)) {
-                    fprintf(stderr, " OK\n");
+                    out->info(out, "... got reply (done)");
                     crm_debug("Got all the replies we expected");
                     pcmk_disconnect_ipc(api);
                     quit_main_loop(CRM_EX_OK);
+                } else {
+                    out->info(out, "... got reply");
                 }
             }
             break;
@@ -285,8 +286,8 @@ start_mainloop(pcmk_ipc_api_t *capi)
     unsigned int count = pcmk_controld_api_replies_expected(capi);
 
     if (count > 0) {
-        fprintf(stderr, "Waiting for %d %s from the controller",
-                count, pcmk__plural_alt(count, "reply", "replies"));
+        out->info(out, "Waiting for %d %s from the controller",
+                  count, pcmk__plural_alt(count, "reply", "replies"));
         exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects
         mainloop = g_main_loop_new(NULL, FALSE);
         g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL);
@@ -1055,7 +1056,7 @@ clear_constraints(pcmk__output_t *out, xmlNodePtr *cib_xml_copy)
         remaining = pcmk__subtract_lists(before, after, (GCompareFunc) strcmp);
 
         for (ele = remaining; ele != NULL; ele = ele->next) {
-            printf("Removing constraint: %s\n", (char *) ele->data);
+            out->info(out, "Removing constraint: %s", (char *) ele->data);
         }
 
         g_list_free(before);
@@ -1281,8 +1282,8 @@ refresh(pcmk__output_t *out)
     }
 
     if (controld_api == NULL) {
-        printf("Dry run: skipping clean-up of %s due to CIB_file\n",
-               options.host_uname? options.host_uname : "all nodes");
+        out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
+                  options.host_uname? options.host_uname : "all nodes");
         rc = pcmk_rc_ok;
         return rc;
     }
@@ -1724,7 +1725,8 @@ main(int argc, char **argv)
     if (options.require_crmd) {
         rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
         if (rc != pcmk_rc_ok) {
-            CMD_ERR("Error connecting to the controller: %s", pcmk_rc_str(rc));
+            g_set_error(&error, PCMK__RC_ERROR, rc,
+                        "Error connecting to the controller: %s", pcmk_rc_str(rc));
             goto done;
         }
         pcmk_register_ipc_callback(controld_api, controller_event_callback,
diff --git a/tools/crm_resource_ban.c b/tools/crm_resource_ban.c
index abed209..4e3ab8b 100644
--- a/tools/crm_resource_ban.c
+++ b/tools/crm_resource_ban.c
@@ -25,18 +25,18 @@ parse_cli_lifetime(pcmk__output_t *out, const char *move_lifetime)
 
     duration = crm_time_parse_duration(move_lifetime);
     if (duration == NULL) {
-        CMD_ERR("Invalid duration specified: %s", move_lifetime);
-        CMD_ERR("Please refer to"
-                " https://en.wikipedia.org/wiki/ISO_8601#Durations"
-                " for examples of valid durations");
+        out->err(out, "Invalid duration specified: %s\n"
+                      "Please refer to https://en.wikipedia.org/wiki/ISO_8601#Durations "
+                      "for examples of valid durations", move_lifetime);
         return NULL;
     }
 
     now = crm_time_new(NULL);
     later = crm_time_add(now, duration);
     if (later == NULL) {
-        CMD_ERR("Unable to add %s to current time", move_lifetime);
-        CMD_ERR("Please report to " PACKAGE_BUGREPORT " as possible bug");
+        out->err(out, "Unable to add %s to current time\n"
+                      "Please report to " PACKAGE_BUGREPORT " as possible bug",
+                      move_lifetime);
         crm_time_free(now);
         crm_time_free(duration);
         return NULL;
@@ -48,7 +48,7 @@ parse_cli_lifetime(pcmk__output_t *out, const char *move_lifetime)
                  crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone);
     crm_time_log(LOG_INFO, "duration", duration, crm_time_log_date | crm_time_log_timeofday);
     later_s = crm_time_as_string(later, crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone);
-    printf("Migration will take effect until: %s\n", later_s);
+    out->info(out, "Migration will take effect until: %s", later_s);
 
     crm_time_free(duration);
     crm_time_free(later);
@@ -89,15 +89,15 @@ cli_resource_ban(pcmk__output_t *out, const char *rsc_id, const char *host,
     crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host);
 
     if (!out->is_quiet(out)) {
-        CMD_ERR("WARNING: Creating rsc_location constraint '%s'"
-                " with a score of -INFINITY for resource %s"
-                " on %s.", ID(location), rsc_id, host);
-        CMD_ERR("\tThis will prevent %s from %s on %s until the constraint "
-                "is removed using the clear option or by editing the CIB "
-                "with an appropriate tool",
-                rsc_id, (promoted_role_only? "being promoted" : "running"), host);
-        CMD_ERR("\tThis will be the case even if %s is"
-                " the last node in the cluster", host);
+        out->info(out, "WARNING: Creating rsc_location constraint '%s' with a "
+                       "score of -INFINITY for resource %s on %s.\n\tThis will "
+                       "prevent %s from %s on %s until the constraint is removed "
+                       "using the clear option or by editing the CIB with an "
+                       "appropriate tool\n\tThis will be the case even if %s "
+                       "is the last node in the cluster",
+                       ID(location), rsc_id, host, rsc_id,
+                       (promoted_role_only? "being promoted" : "running"),
+                       host, host);
     }
 
     crm_xml_add(location, XML_LOC_ATTR_SOURCE, rsc_id);
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 22b469e..bd377a3 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -134,12 +134,12 @@ find_resource_attr(pcmk__output_t *out, cib_t * the_cib, const char *attr,
         xmlNode *child = NULL;
 
         rc = EINVAL;
-        printf("Multiple attributes match name=%s\n", attr_name);
+        out->info(out, "Multiple attributes match name=%s", attr_name);
 
         for (child = pcmk__xml_first_child(xml_search); child != NULL;
              child = pcmk__xml_next(child)) {
-            printf("  Value: %s \t(id=%s)\n",
-                   crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
+            out->info(out, "  Value: %s \t(id=%s)",
+                      crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
         }
 
         out->spacer(out);
@@ -223,7 +223,8 @@ find_matching_attr_resources(pcmk__output_t *out, pe_resource_t * rsc,
         if(rc != pcmk_rc_ok) {
             rsc = rsc->parent;
             if (!out->is_quiet(out)) {
-                printf("Performing %s of '%s' on '%s', the parent of '%s'\n", cmd, attr_name, rsc->id, rsc_id);
+                out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
+                          cmd, attr_name, rsc->id, rsc_id);
             }
         }
         return g_list_append(result, rsc);
@@ -238,7 +239,8 @@ find_matching_attr_resources(pcmk__output_t *out, pe_resource_t * rsc,
             if(rc == pcmk_rc_ok) {
                 rsc = child;
                 if (!out->is_quiet(out)) {
-                    printf("A value for '%s' already exists in child '%s', performing %s on that instead of '%s'\n", attr_name, lookup_id, cmd, rsc_id);
+                    out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
+                              attr_name, lookup_id, cmd, rsc_id);
                 }
             }
 
@@ -272,11 +274,9 @@ cli_resource_update_attribute(pcmk__output_t *out, pe_resource_t *rsc,
     GList/*<pe_resource_t*>*/ *resources = NULL;
     const char *common_attr_id = attr_id;
 
-    if(attr_id == NULL
-       && force == FALSE
-       && find_resource_attr(
-           out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL) == EINVAL) {
-        printf("\n");
+    if (attr_id == NULL && force == FALSE) {
+        find_resource_attr (out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL,
+                            NULL, NULL, attr_name, NULL);
     }
 
     if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
@@ -285,10 +285,10 @@ cli_resource_update_attribute(pcmk__output_t *out, pe_resource_t *rsc,
                                     XML_TAG_META_SETS, attr_set, attr_id,
                                     attr_name, &local_attr_id);
             if (rc == pcmk_rc_ok && !out->is_quiet(out)) {
-                printf("WARNING: There is already a meta attribute for '%s' called '%s' (id=%s)\n",
-                       uber_parent(rsc)->id, attr_name, local_attr_id);
-                printf("         Delete '%s' first or use the force option to override\n",
-                       local_attr_id);
+                out->err(out, "WARNING: There is already a meta attribute for '%s' called '%s' (id=%s)",
+                         uber_parent(rsc)->id, attr_name, local_attr_id);
+                out->err(out, "         Delete '%s' first or use the force option to override",
+                         local_attr_id);
             }
             free(local_attr_id);
             if (rc == pcmk_rc_ok) {
@@ -362,9 +362,9 @@ cli_resource_update_attribute(pcmk__output_t *out, pe_resource_t *rsc,
         rc = pcmk_legacy2rc(rc);
 
         if (rc == pcmk_rc_ok && !out->is_quiet(out)) {
-            printf("Set '%s' option: id=%s%s%s%s%s value=%s\n", lookup_id, local_attr_id,
-                   attr_set ? " set=" : "", attr_set ? attr_set : "",
-                   attr_name ? " name=" : "", attr_name ? attr_name : "", attr_value);
+            out->info(out, "Set '%s' option: id=%s%s%s%s%s value=%s", lookup_id, local_attr_id,
+                      attr_set ? " set=" : "", attr_set ? attr_set : "",
+                      attr_name ? " name=" : "", attr_name ? attr_name : "", attr_value);
         }
 
         free_xml(xml_top);
@@ -421,11 +421,9 @@ cli_resource_delete_attribute(pcmk__output_t *out, pe_resource_t *rsc,
     int rc = pcmk_rc_ok;
     GList/*<pe_resource_t*>*/ *resources = NULL;
 
-    if(attr_id == NULL
-       && force == FALSE
-       && find_resource_attr(
-           out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL) == EINVAL) {
-        printf("\n");
+    if (attr_id == NULL && force == FALSE) {
+        find_resource_attr(out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL,
+                           NULL, NULL, attr_name, NULL);
     }
 
     if(pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
@@ -469,9 +467,9 @@ cli_resource_delete_attribute(pcmk__output_t *out, pe_resource_t *rsc,
         rc = pcmk_legacy2rc(rc);
 
         if (rc == pcmk_rc_ok && !out->is_quiet(out)) {
-            printf("Deleted '%s' option: id=%s%s%s%s%s\n", lookup_id, local_attr_id,
-                   attr_set ? " set=" : "", attr_set ? attr_set : "",
-                   attr_name ? " name=" : "", attr_name ? attr_name : "");
+            out->info(out, "Deleted '%s' option: id=%s%s%s%s%s", lookup_id, local_attr_id,
+                      attr_set ? " set=" : "", attr_set ? attr_set : "",
+                      attr_name ? " name=" : "", attr_name ? attr_name : "");
         }
 
         free(lookup_id);
@@ -497,11 +495,11 @@ send_lrm_rsc_op(pcmk__output_t *out, pcmk_ipc_api_t *controld_api, bool do_fail_
     pe_resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
 
     if (rsc == NULL) {
-        CMD_ERR("Resource %s not found", rsc_id);
+        out->err(out, "Resource %s not found", rsc_id);
         return ENXIO;
 
     } else if (rsc->variant != pe_native) {
-        CMD_ERR("We can only process primitive resources, not %s", rsc_id);
+        out->err(out, "We can only process primitive resources, not %s", rsc_id);
         return EINVAL;
     }
 
@@ -509,25 +507,25 @@ send_lrm_rsc_op(pcmk__output_t *out, pcmk_ipc_api_t *controld_api, bool do_fail_
     rsc_provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
     rsc_type = crm_element_value(rsc->xml, XML_ATTR_TYPE);
     if ((rsc_class == NULL) || (rsc_type == NULL)) {
-        CMD_ERR("Resource %s does not have a class and type", rsc_id);
+        out->err(out, "Resource %s does not have a class and type", rsc_id);
         return EINVAL;
     }
 
     if (host_uname == NULL) {
-        CMD_ERR("Please specify a node name");
+        out->err(out, "Please specify a node name");
         return EINVAL;
 
     } else {
         pe_node_t *node = pe_find_node(data_set->nodes, host_uname);
 
         if (node == NULL) {
-            CMD_ERR("Node %s not found", host_uname);
+            out->err(out, "Node %s not found", host_uname);
             return pcmk_rc_node_unknown;
         }
 
         if (!(node->details->online)) {
             if (do_fail_resource) {
-                CMD_ERR("Node %s is not online", host_uname);
+                out->err(out, "Node %s is not online", host_uname);
                 return ENOTCONN;
             } else {
                 cib_only = true;
@@ -536,8 +534,8 @@ send_lrm_rsc_op(pcmk__output_t *out, pcmk_ipc_api_t *controld_api, bool do_fail_
         if (!cib_only && pe__is_guest_or_remote_node(node)) {
             node = pe__current_node(node->details->remote_rsc);
             if (node == NULL) {
-                CMD_ERR("No cluster connection to Pacemaker Remote node %s detected",
-                        host_uname);
+                out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
+                         host_uname);
                 return ENOTCONN;
             }
             router_node = node->details->uname;
@@ -779,27 +777,27 @@ cli_resource_delete(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
     node = pe_find_node(data_set->nodes, host_uname);
 
     if (node == NULL) {
-        printf("Unable to clean up %s because node %s not found\n",
-               rsc->id, host_uname);
+        out->err(out, "Unable to clean up %s because node %s not found",
+                 rsc->id, host_uname);
         return ENODEV;
     }
 
     if (!node->details->rsc_discovery_enabled) {
-        printf("Unable to clean up %s because resource discovery disabled on %s\n",
-               rsc->id, host_uname);
+        out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
+                 rsc->id, host_uname);
         return EOPNOTSUPP;
     }
 
     if (controld_api == NULL) {
-        printf("Dry run: skipping clean-up of %s on %s due to CIB_file\n",
-               rsc->id, host_uname);
+        out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
+                 rsc->id, host_uname);
         return pcmk_rc_ok;
     }
 
     rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
     if (rc != pcmk_rc_ok) {
-        printf("Unable to clean up %s failures on %s: %s\n",
-                rsc->id, host_uname, pcmk_rc_str(rc));
+        out->err(out, "Unable to clean up %s failures on %s: %s",
+                 rsc->id, host_uname, pcmk_rc_str(rc));
         return rc;
     }
 
@@ -810,10 +808,10 @@ cli_resource_delete(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
         rc = clear_rsc_history(out, controld_api, host_uname, rsc->id, data_set);
     }
     if (rc != pcmk_rc_ok) {
-        printf("Cleaned %s failures on %s, but unable to clean history: %s\n",
-               rsc->id, host_uname, pcmk_strerror(rc));
+        out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
+                 rsc->id, host_uname, pcmk_strerror(rc));
     } else {
-        printf("Cleaned up %s on %s\n", rsc->id, host_uname);
+        out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
     }
     return rc;
 }
@@ -829,8 +827,8 @@ cli_cleanup_all(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
     const char *display_name = node_name? node_name : "all nodes";
 
     if (controld_api == NULL) {
-        printf("Dry run: skipping clean-up of %s due to CIB_file\n",
-               display_name);
+        out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
+                  display_name);
         return rc;
     }
 
@@ -838,7 +836,7 @@ cli_cleanup_all(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
         pe_node_t *node = pe_find_node(data_set->nodes, node_name);
 
         if (node == NULL) {
-            CMD_ERR("Unknown node: %s", node_name);
+            out->err(out, "Unknown node: %s", node_name);
             return ENXIO;
         }
         if (pe__is_guest_or_remote_node(node)) {
@@ -849,8 +847,8 @@ cli_cleanup_all(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
     rc = pcmk__node_attr_request_clear(NULL, node_name, NULL, operation,
                                        interval_spec, NULL, attr_options);
     if (rc != pcmk_rc_ok) {
-        printf("Unable to clean up all failures on %s: %s\n",
-                display_name, pcmk_rc_str(rc));
+        out->err(out, "Unable to clean up all failures on %s: %s",
+                 display_name, pcmk_rc_str(rc));
         return rc;
     }
 
@@ -858,8 +856,8 @@ cli_cleanup_all(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
         rc = clear_rsc_failures(out, controld_api, node_name, NULL,
                                 operation, interval_spec, data_set);
         if (rc != pcmk_rc_ok) {
-            printf("Cleaned all resource failures on %s, but unable to clean history: %s\n",
-                   node_name, pcmk_strerror(rc));
+            out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
+                     node_name, pcmk_strerror(rc));
             return rc;
         }
     } else {
@@ -869,14 +867,14 @@ cli_cleanup_all(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
             rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
                                     operation, interval_spec, data_set);
             if (rc != pcmk_rc_ok) {
-                printf("Cleaned all resource failures on all nodes, but unable to clean history: %s\n",
-                       pcmk_strerror(rc));
+                out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
+                         pcmk_strerror(rc));
                 return rc;
             }
         }
     }
 
-    printf("Cleaned up all resources on %s\n", display_name);
+    out->info(out, "Cleaned up all resources on %s", display_name);
     return rc;
 }
 
@@ -1074,7 +1072,7 @@ static void display_list(pcmk__output_t *out, GList *items, const char *tag)
     GList *item = NULL;
 
     for (item = items; item != NULL; item = item->next) {
-        fprintf(stdout, "%s%s\n", tag, (const char *)item->data);
+        out->info(out, "%s%s", tag, (const char *)item->data);
     }
 }
 
@@ -1127,12 +1125,12 @@ update_working_set_from_cib(pcmk__output_t *out, pe_working_set_t * data_set,
     rc = pcmk_legacy2rc(rc);
 
     if (rc != pcmk_rc_ok) {
-        fprintf(stderr, "Could not obtain the current CIB: %s (%d)\n", pcmk_strerror(rc), rc);
+        out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_strerror(rc), rc);
         return rc;
     }
     rc = update_working_set_xml(data_set, &cib_xml_copy);
     if (rc != pcmk_rc_ok) {
-        fprintf(stderr, "Could not upgrade the current CIB XML\n");
+        out->err(out, "Could not upgrade the current CIB XML");
         free_xml(cib_xml_copy);
         return rc;
     }
@@ -1162,7 +1160,7 @@ update_dataset(pcmk__output_t *out, cib_t *cib, pe_working_set_t * data_set,
         shadow_file = get_shadow_file(pid);
 
         if (shadow_cib == NULL) {
-            fprintf(stderr, "Could not create shadow cib: '%s'\n", pid);
+            out->err(out, "Could not create shadow cib: '%s'", pid);
             rc = ENXIO;
             goto cleanup;
         }
@@ -1170,7 +1168,7 @@ update_dataset(pcmk__output_t *out, cib_t *cib, pe_working_set_t * data_set,
         rc = write_xml_file(data_set->input, shadow_file, FALSE);
 
         if (rc < 0) {
-            fprintf(stderr, "Could not populate shadow cib: %s (%d)\n", pcmk_strerror(rc), rc);
+            out->err(out, "Could not populate shadow cib: %s (%d)", pcmk_strerror(rc), rc);
             goto cleanup;
         }
 
@@ -1178,7 +1176,7 @@ update_dataset(pcmk__output_t *out, cib_t *cib, pe_working_set_t * data_set,
         rc = pcmk_legacy2rc(rc);
 
         if (rc != pcmk_rc_ok) {
-            fprintf(stderr, "Could not connect to shadow cib: %s (%d)\n", pcmk_strerror(rc), rc);
+            out->err(out, "Could not connect to shadow cib: %s (%d)", pcmk_strerror(rc), rc);
             goto cleanup;
         }
 
@@ -1301,9 +1299,9 @@ cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
     if(resource_is_running_on(rsc, host) == FALSE) {
         const char *id = rsc->clone_name?rsc->clone_name:rsc->id;
         if(host) {
-            printf("%s is not running on %s and so cannot be restarted\n", id, host);
+            out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
         } else {
-            printf("%s is not running anywhere and so cannot be restarted\n", id);
+            out->err(out, "%s is not running anywhere and so cannot be restarted", id);
         }
         return ENXIO;
     }
@@ -1340,7 +1338,7 @@ cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
     pe__set_working_set_flags(data_set, pe_flag_no_counts|pe_flag_no_compat);
     rc = update_dataset(out, cib, data_set, FALSE);
     if(rc != pcmk_rc_ok) {
-        fprintf(stdout, "Could not get new resource list: %s (%d)\n", pcmk_strerror(rc), rc);
+        out->err(out, "Could not get new resource list: %s (%d)", pcmk_strerror(rc), rc);
         goto done;
     }
 
@@ -1371,7 +1369,7 @@ cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
                                            data_set, force);
     }
     if(rc != pcmk_rc_ok) {
-        fprintf(stderr, "Could not set target-role for %s: %s (%d)\n", rsc_id, pcmk_strerror(rc), rc);
+        out->err(out, "Could not set target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
         if (current_active) {
             g_list_free_full(current_active, free);
         }
@@ -1383,7 +1381,7 @@ cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
 
     rc = update_dataset(out, cib, data_set, TRUE);
     if(rc != pcmk_rc_ok) {
-        fprintf(stderr, "Could not determine which resources would be stopped\n");
+        out->err(out, "Could not determine which resources would be stopped");
         goto failure;
     }
 
@@ -1391,7 +1389,7 @@ cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
     dump_list(target_active, "Target");
 
     list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
-    fprintf(stdout, "Waiting for %d resources to stop:\n", g_list_length(list_delta));
+    out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
     display_list(out, list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
@@ -1410,7 +1408,7 @@ cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
             }
             rc = update_dataset(out, cib, data_set, FALSE);
             if(rc != pcmk_rc_ok) {
-                fprintf(stderr, "Could not determine which resources were stopped\n");
+                out->err(out, "Could not determine which resources were stopped");
                 goto failure;
             }
 
@@ -1427,7 +1425,7 @@ cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
         crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
         if(before == g_list_length(list_delta)) {
             /* aborted during stop phase, print the contents of list_delta */
-            fprintf(stderr, "Could not complete shutdown of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
+            out->info(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
             display_list(out, list_delta, " * ");
             rc = ETIME;
             goto failure;
@@ -1452,7 +1450,7 @@ cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
     }
 
     if(rc != pcmk_rc_ok) {
-        fprintf(stderr, "Could not unset target-role for %s: %s (%d)\n", rsc_id, pcmk_strerror(rc), rc);
+        out->err(out, "Could not unset target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
         goto done;
     }
 
@@ -1461,7 +1459,7 @@ cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
     }
     target_active = restart_target_active;
     list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
-    fprintf(stdout, "Waiting for %d resources to start again:\n", g_list_length(list_delta));
+    out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
     display_list(out, list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
@@ -1482,7 +1480,7 @@ cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
 
             rc = update_dataset(out, cib, data_set, FALSE);
             if(rc != pcmk_rc_ok) {
-                fprintf(stderr, "Could not determine which resources were started\n");
+                out->err(out, "Could not determine which resources were started");
                 goto failure;
             }
 
@@ -1502,7 +1500,7 @@ cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
 
         if(before == g_list_length(list_delta)) {
             /* aborted during start phase, print the contents of list_delta */
-            fprintf(stdout, "Could not complete restart of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
+            out->info(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
             display_list(out, list_delta, " * ");
             rc = ETIME;
             goto failure;
@@ -1578,29 +1576,23 @@ actions_are_pending(GListPtr actions)
     return FALSE;
 }
 
-/*!
- * \internal
- * \brief Print pending actions to stderr
- *
- * \param[in] actions   List of actions to check
- *
- * \return void
- */
 static void
 print_pending_actions(pcmk__output_t *out, GListPtr actions)
 {
     GListPtr action;
 
-    fprintf(stderr, "Pending actions:\n");
+    out->info(out, "Pending actions:");
     for (action = actions; action != NULL; action = action->next) {
         pe_action_t *a = (pe_action_t *) action->data;
 
-        if (action_is_pending(a)) {
-            fprintf(stderr, "\tAction %d: %s", a->id, a->uuid);
-            if (a->node) {
-                fprintf(stderr, "\ton %s", a->node->details->uname);
-            }
-            fprintf(stderr, "\n");
+        if (!action_is_pending(a)) {
+            continue;
+        }
+
+        if (a->node) {
+            out->info(out, "\tAction %d: %s\ton %s", a->id, a->uuid, a->node->details->uname);
+        } else {
+            out->info(out, "\tAction %d: %s", a->id, a->uuid);
         }
     }
 }
@@ -1678,8 +1670,8 @@ wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib)
                                                          "dc-version");
 
             if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
-                printf("warning: wait option may not work properly in "
-                       "mixed-version cluster\n");
+                out->info(out, "warning: wait option may not work properly in "
+                          "mixed-version cluster");
                 printed_version_warning = TRUE;
             }
         }
@@ -1702,8 +1694,8 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
     svc_action_t *op = NULL;
 
     if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
-        CMD_ERR("Sorry, the %s option doesn't support %s resources yet",
-                action, rsc_class);
+        out->err(out, "Sorry, the %s option doesn't support %s resources yet",
+                 action, rsc_class);
         crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
     }
 
@@ -1765,8 +1757,8 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
 
         g_hash_table_iter_init(&iter, override_hash);
         while (g_hash_table_iter_next(&iter, (gpointer *) & name, (gpointer *) & value)) {
-            printf("Overriding the cluster configuration for '%s' with '%s' = '%s'\n",
-                   rsc_name, name, value);
+            out->info(out, "Overriding the cluster configuration for '%s' with '%s' = '%s'",
+                      rsc_name, name, value);
             g_hash_table_replace(op->params, strdup(name), strdup(value));
         }
     }
@@ -1775,13 +1767,13 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
         exit_code = op->rc;
 
         if (op->status == PCMK_LRM_OP_DONE) {
-            printf("Operation %s for %s (%s:%s:%s) returned: '%s' (%d)\n",
-                   action, rsc_name, rsc_class, rsc_prov ? rsc_prov : "", rsc_type,
-                   services_ocf_exitcode_str(op->rc), op->rc);
+            out->info(out, "Operation %s for %s (%s:%s:%s) returned: '%s' (%d)",
+                      action, rsc_name, rsc_class, rsc_prov ? rsc_prov : "", rsc_type,
+                      services_ocf_exitcode_str(op->rc), op->rc);
         } else {
-            printf("Operation %s for %s (%s:%s:%s) failed: '%s' (%d)\n",
-                   action, rsc_name, rsc_class, rsc_prov ? rsc_prov : "", rsc_type,
-                   services_lrm_status_str(op->status), op->status);
+            out->info(out, "Operation %s for %s (%s:%s:%s) failed: '%s' (%d)",
+                      action, rsc_name, rsc_class, rsc_prov ? rsc_prov : "", rsc_type,
+                      services_lrm_status_str(op->status), op->status);
         }
 
         /* hide output for validate-all if not in verbose */
@@ -1833,11 +1825,11 @@ cli_resource_execute(pcmk__output_t *out, pe_resource_t *rsc,
         if(pe_rsc_is_clone(rsc)) {
             int rc = cli_resource_search(out, rsc, requested_name, data_set);
             if(rc > 0 && force == FALSE) {
-                CMD_ERR("It is not safe to %s %s here: the cluster claims it is already active",
-                        action, rsc->id);
-                CMD_ERR("Try setting target-role=Stopped first or specifying "
-                        "the force option");
-                crm_exit(CRM_EX_UNSAFE);
+                out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
+                         action, rsc->id);
+                out->err(out, "Try setting target-role=Stopped first or specifying "
+                         "the force option");
+                return CRM_EX_UNSAFE;
             }
         }
 
@@ -1851,9 +1843,8 @@ cli_resource_execute(pcmk__output_t *out, pe_resource_t *rsc,
     }
 
     if(rsc->variant == pe_group) {
-        CMD_ERR("Sorry, the %s option doesn't support group resources",
-                rsc_action);
-        crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
+        out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
+        return CRM_EX_UNIMPLEMENT_FEATURE;
     }
 
     rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
@@ -1895,12 +1886,12 @@ cli_resource_move(pcmk__output_t *out, pe_resource_t *rsc, const char *rsc_id,
         pe_resource_t *p = uber_parent(rsc);
 
         if (pcmk_is_set(p->flags, pe_rsc_promotable)) {
-            CMD_ERR("Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
+            out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
             rsc_id = p->id;
             rsc = p;
 
         } else {
-            CMD_ERR("Ignoring master option: %s is not promotable", rsc_id);
+            out->info(out, "Ignoring master option: %s is not promotable", rsc_id);
             promoted_role_only = FALSE;
         }
     }
@@ -1971,13 +1962,13 @@ cli_resource_move(pcmk__output_t *out, pe_resource_t *rsc, const char *rsc_id,
                                    NULL, cib, cib_options, promoted_role_only);
 
         } else if(count > 1) {
-            CMD_ERR("Resource '%s' is currently %s in %d locations. "
-                    "One may now move to %s",
-                    rsc_id, (promoted_role_only? "promoted" : "active"),
-                    count, dest->details->uname);
-            CMD_ERR("To prevent '%s' from being %s at a specific location, "
-                    "specify a node.",
-                    rsc_id, (promoted_role_only? "promoted" : "active"));
+            out->info(out, "Resource '%s' is currently %s in %d locations. "
+                      "One may now move to %s",
+                      rsc_id, (promoted_role_only? "promoted" : "active"),
+                      count, dest->details->uname);
+            out->info(out, "To prevent '%s' from being %s at a specific location, "
+                      "specify a node.",
+                      rsc_id, (promoted_role_only? "promoted" : "active"));
 
         } else {
             crm_trace("Not banning %s from its current location: not active", rsc_id);
-- 
1.8.3.1


From 8d94e547f6ae7a1bdeb6d28b4f9f9e102543e35f Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Thu, 3 Sep 2020 10:50:35 -0400
Subject: [PATCH 13/19] Feature: scheduler, tools: Add a new node-and-op output
 message.

This is used for the output of "crm_resource -O" and "crm_resource -o".
Each operation gets printed on a single line with a brief summary of the
node it's running on preceeding it.

There's a fair bit of overlap between this and the op-history message
but there's also a number of differences.  This new message is for use
in crm_resource, and attempting to combine the two is going to change
that program's output, which may not be acceptable to some users.
---
 include/crm/pengine/internal.h |   2 +
 lib/common/output_xml.c        |   1 +
 lib/pengine/pe_output.c        | 105 +++++++++++++++++++++++++++++++++++++++++
 tools/crm_resource_print.c     |  43 +++++------------
 4 files changed, 120 insertions(+), 31 deletions(-)

diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
index 00b6b4c..396d707 100644
--- a/include/crm/pengine/internal.h
+++ b/include/crm/pengine/internal.h
@@ -293,6 +293,8 @@ int pe__bundle_text(pcmk__output_t *out, va_list args);
 int pe__node_html(pcmk__output_t *out, va_list args);
 int pe__node_text(pcmk__output_t *out, va_list args);
 int pe__node_xml(pcmk__output_t *out, va_list args);
+int pe__node_and_op(pcmk__output_t *out, va_list args);
+int pe__node_and_op_xml(pcmk__output_t *out, va_list args);
 int pe__node_attribute_html(pcmk__output_t *out, va_list args);
 int pe__node_attribute_text(pcmk__output_t *out, va_list args);
 int pe__node_attribute_xml(pcmk__output_t *out, va_list args);
diff --git a/lib/common/output_xml.c b/lib/common/output_xml.c
index bba21e7..716d10f 100644
--- a/lib/common/output_xml.c
+++ b/lib/common/output_xml.c
@@ -61,6 +61,7 @@ static subst_t substitutions[] = {
     { "Negative Location Constraints",  "bans" },
     { "Node Attributes",                "node_attributes" },
     { "Resource Config",                "resource_config" },
+    { "Resource Operations",            "operations" },
 
     { NULL, NULL }
 };
diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
index dd3a880..1a3f93d 100644
--- a/lib/pengine/pe_output.c
+++ b/lib/pengine/pe_output.c
@@ -1284,6 +1284,109 @@ pe__node_attribute_html(pcmk__output_t *out, va_list args) {
     return pcmk_rc_ok;
 }
 
+PCMK__OUTPUT_ARGS("node-and-op", "pe_working_set_t *", "xmlNodePtr")
+int
+pe__node_and_op(pcmk__output_t *out, va_list args) {
+    pe_working_set_t *data_set = va_arg(args, pe_working_set_t *);
+    xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
+
+    pe_resource_t *rsc = NULL;
+    gchar *node_str = NULL;
+    char *last_change_str = NULL;
+
+    const char *op_rsc = crm_element_value(xml_op, "resource");
+    const char *status_s = crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS);
+    const char *op_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
+    int status = crm_parse_int(status_s, "0");
+    time_t last_change = 0;
+
+    rsc = pe_find_resource(data_set->resources, op_rsc);
+
+    if (rsc) {
+        pe_node_t *node = pe__current_node(rsc);
+        const char *target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
+        int opts = pe_print_rsconly | pe_print_pending;
+
+        if (node == NULL) {
+            node = rsc->pending_node;
+        }
+
+        node_str = pcmk__native_output_string(rsc, rsc_printable_id(rsc), node,
+                                              opts, target_role, false);
+    } else {
+        node_str = crm_strdup_printf("Unknown resource %s", op_rsc);
+    }
+
+    if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
+                                &last_change) == pcmk_ok) {
+        last_change_str = crm_strdup_printf(", %s=%s, exec=%sms",
+                                            XML_RSC_OP_LAST_CHANGE,
+                                            crm_strip_trailing_newline(ctime(&last_change)),
+                                            crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
+    }
+
+    out->list_item(out, NULL, "%s: %s (node=%s, call=%s, rc=%s%s): %s",
+                   node_str, op_key ? op_key : ID(xml_op),
+                   crm_element_value(xml_op, XML_ATTR_UNAME),
+                   crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
+                   crm_element_value(xml_op, XML_LRM_ATTR_RC),
+                   last_change_str ? last_change_str : "",
+                   services_lrm_status_str(status));
+
+    g_free(node_str);
+    free(last_change_str);
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("node-and-op", "pe_working_set_t *", "xmlNodePtr")
+int
+pe__node_and_op_xml(pcmk__output_t *out, va_list args) {
+    pe_working_set_t *data_set = va_arg(args, pe_working_set_t *);
+    xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
+
+    pe_resource_t *rsc = NULL;
+    const char *op_rsc = crm_element_value(xml_op, "resource");
+    const char *status_s = crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS);
+    const char *op_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
+    int status = crm_parse_int(status_s, "0");
+    time_t last_change = 0;
+
+    xmlNode *node = pcmk__output_create_xml_node(out, "operation");
+
+    rsc = pe_find_resource(data_set->resources, op_rsc);
+
+    if (rsc) {
+        const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
+        const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
+        char *agent_tuple = NULL;
+
+        agent_tuple = crm_strdup_printf("%s:%s:%s", class,
+                                        pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider) ? crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER) : "",
+                                        kind);
+
+        xmlSetProp(node, (pcmkXmlStr) "rsc", (pcmkXmlStr) rsc_printable_id(rsc));
+        xmlSetProp(node, (pcmkXmlStr) "agent", (pcmkXmlStr) agent_tuple);
+        free(agent_tuple);
+    }
+
+    xmlSetProp(node, (pcmkXmlStr) "op", (pcmkXmlStr) (op_key ? op_key : ID(xml_op)));
+    xmlSetProp(node, (pcmkXmlStr) "node", (pcmkXmlStr) crm_element_value(xml_op, XML_ATTR_UNAME));
+    xmlSetProp(node, (pcmkXmlStr) "call", (pcmkXmlStr) crm_element_value(xml_op, XML_LRM_ATTR_CALLID));
+    xmlSetProp(node, (pcmkXmlStr) "rc", (pcmkXmlStr) crm_element_value(xml_op, XML_LRM_ATTR_RC));
+
+    if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
+                                &last_change) == pcmk_ok) {
+        xmlSetProp(node, (pcmkXmlStr) XML_RSC_OP_LAST_CHANGE,
+                   (pcmkXmlStr) crm_strip_trailing_newline(ctime(&last_change)));
+        xmlSetProp(node, (pcmkXmlStr) XML_RSC_OP_T_EXEC,
+                   (pcmkXmlStr) crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
+    }
+
+    xmlSetProp(node, (pcmkXmlStr) "status", (pcmkXmlStr) services_lrm_status_str(status));
+
+    return pcmk_rc_ok;
+}
+
 PCMK__OUTPUT_ARGS("node-attribute", "const char *", "const char *", "gboolean", "int")
 int
 pe__node_attribute_xml(pcmk__output_t *out, va_list args) {
@@ -1878,6 +1981,8 @@ static pcmk__message_entry_t fmt_functions[] = {
     { "node", "log", pe__node_text },
     { "node", "text", pe__node_text },
     { "node", "xml", pe__node_xml },
+    { "node-and-op", "default", pe__node_and_op },
+    { "node-and-op", "xml", pe__node_and_op_xml },
     { "node-list", "html", pe__node_list_html },
     { "node-list", "log", pe__node_list_text },
     { "node-list", "text", pe__node_list_text },
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
index 447c57d..de2202e 100644
--- a/tools/crm_resource_print.c
+++ b/tools/crm_resource_print.c
@@ -89,42 +89,23 @@ cli_resource_print_operations(pcmk__output_t *out, const char *rsc_id,
                               const char *host_uname, bool active,
                               pe_working_set_t * data_set)
 {
-    pe_resource_t *rsc = NULL;
-    int opts = pe_print_printf | pe_print_rsconly | pe_print_suppres_nl | pe_print_pending;
+    int rc = pcmk_rc_no_output;
     GListPtr ops = find_operations(rsc_id, host_uname, active, data_set);
-    GListPtr lpc = NULL;
-
-    for (lpc = ops; lpc != NULL; lpc = lpc->next) {
-        xmlNode *xml_op = (xmlNode *) lpc->data;
 
-        const char *op_rsc = crm_element_value(xml_op, "resource");
-        const char *status_s = crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS);
-        const char *op_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
-        int status = crm_parse_int(status_s, "0");
-        time_t last_change = 0;
+    if (!ops) {
+        return rc;
+    }
 
-        rsc = pe_find_resource(data_set->resources, op_rsc);
-        if(rsc) {
-            rsc->fns->print(rsc, "", opts, stdout);
-        } else {
-            fprintf(stdout, "Unknown resource %s", op_rsc);
-        }
+    out->begin_list(out, NULL, NULL, "Resource Operations");
+    rc = pcmk_rc_ok;
 
-        fprintf(stdout, ": %s (node=%s, call=%s, rc=%s",
-                op_key ? op_key : ID(xml_op),
-                crm_element_value(xml_op, XML_ATTR_UNAME),
-                crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
-                crm_element_value(xml_op, XML_LRM_ATTR_RC));
-
-        if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
-                                    &last_change) == pcmk_ok) {
-            fprintf(stdout, ", " XML_RSC_OP_LAST_CHANGE "=%s, exec=%sms",
-                    crm_strip_trailing_newline(ctime(&last_change)),
-                    crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
-        }
-        fprintf(stdout, "): %s\n", services_lrm_status_str(status));
+    for (GListPtr lpc = ops; lpc != NULL; lpc = lpc->next) {
+        xmlNode *xml_op = (xmlNode *) lpc->data;
+        out->message(out, "node-and-op", data_set, xml_op);
     }
-    return pcmk_rc_ok;
+
+    out->end_list(out);
+    return rc;
 }
 
 void
-- 
1.8.3.1


From 768e2ae0efa626fce75e7fb84970c49617ca8448 Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Thu, 1 Oct 2020 11:21:40 -0400
Subject: [PATCH 14/19] Feature: tools: Use formatted output for CTS printing
 in crm_resource.

Note that this option only exists for internal use, passing information
to cts.  It's not exposed in the command line help.  I've implemented it
just using out->info so it will only print for the text output format.
I don't want XML output that someone might start to rely on, despite not
being in the schema.
---
 tools/crm_resource.c       |  2 ++
 tools/crm_resource_print.c | 33 ++++++++++++++++-----------------
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index 0532095..6e32982 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1813,12 +1813,14 @@ main(int argc, char **argv)
 
         case cmd_cts:
             rc = pcmk_rc_ok;
+
             for (GList *lpc = data_set->resources; lpc != NULL;
                  lpc = lpc->next) {
 
                 rsc = (pe_resource_t *) lpc->data;
                 cli_resource_print_cts(out, rsc);
             }
+
             cli_resource_print_cts_constraints(out, data_set);
             break;
 
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
index de2202e..4b3c42d 100644
--- a/tools/crm_resource_print.c
+++ b/tools/crm_resource_print.c
@@ -33,19 +33,18 @@ cli_resource_print_cts_constraints(pcmk__output_t *out, pe_working_set_t * data_
             continue;
         }
 
-        if (pcmk__str_eq(XML_CONS_TAG_RSC_DEPEND, crm_element_name(xml_obj), pcmk__str_casei)) {
-            printf("Constraint %s %s %s %s %s %s %s\n",
-                   crm_element_name(xml_obj),
-                   cons_string(crm_element_value(xml_obj, XML_ATTR_ID)),
-                   cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE)),
-                   cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET)),
-                   cons_string(crm_element_value(xml_obj, XML_RULE_ATTR_SCORE)),
-                   cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE)),
-                   cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE)));
-
-        } else if (pcmk__str_eq(XML_CONS_TAG_RSC_LOCATION, crm_element_name(xml_obj), pcmk__str_casei)) {
-            /* unpack_location(xml_obj, data_set); */
+        if (!pcmk__str_eq(XML_CONS_TAG_RSC_DEPEND, crm_element_name(xml_obj), pcmk__str_casei)) {
+            continue;
         }
+
+        out->info(out, "Constraint %s %s %s %s %s %s %s",
+                  crm_element_name(xml_obj),
+                  cons_string(crm_element_value(xml_obj, XML_ATTR_ID)),
+                  cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE)),
+                  cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET)),
+                  cons_string(crm_element_value(xml_obj, XML_RULE_ATTR_SCORE)),
+                  cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE)),
+                  cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE)));
     }
 }
 
@@ -70,11 +69,11 @@ cli_resource_print_cts(pcmk__output_t *out, pe_resource_t * rsc)
         host = node->details->uname;
     }
 
-    printf("Resource: %s %s %s %s %s %s %s %s %d %lld 0x%.16llx\n",
-           crm_element_name(rsc->xml), rsc->id,
-           rsc->clone_name ? rsc->clone_name : rsc->id, rsc->parent ? rsc->parent->id : "NA",
-           rprov ? rprov : "NA", rclass, rtype, host ? host : "NA", needs_quorum, rsc->flags,
-           rsc->flags);
+    out->info(out, "Resource: %s %s %s %s %s %s %s %s %d %lld 0x%.16llx",
+              crm_element_name(rsc->xml), rsc->id,
+              rsc->clone_name ? rsc->clone_name : rsc->id, rsc->parent ? rsc->parent->id : "NA",
+              rprov ? rprov : "NA", rclass, rtype, host ? host : "NA", needs_quorum, rsc->flags,
+              rsc->flags);
 
     for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
         pe_resource_t *child = (pe_resource_t *) lpc->data;
-- 
1.8.3.1


From 9ab0635ada9293456eed7e31a8bf2c6e9b44d833 Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Thu, 1 Oct 2020 14:33:47 -0400
Subject: [PATCH 15/19] Feature: tools: Use formatted output for crm_resource
 checks.

---
 cts/cli/regression.tools.exp |   4 +-
 tools/crm_resource.c         |   2 +-
 tools/crm_resource.h         |  20 +++-
 tools/crm_resource_print.c   | 260 +++++++++++++++++++++++++++++++++++++++++++
 tools/crm_resource_runtime.c | 170 +++++++---------------------
 5 files changed, 318 insertions(+), 138 deletions(-)

diff --git a/cts/cli/regression.tools.exp b/cts/cli/regression.tools.exp
index 935dce8..7166714 100644
--- a/cts/cli/regression.tools.exp
+++ b/cts/cli/regression.tools.exp
@@ -876,9 +876,7 @@ Set 'dummy' option: id=dummy-meta_attributes-target-role set=dummy-meta_attribut
 * Passed: crm_resource   - Create another resource meta attribute
 =#=#=#= Begin test: Show why a resource is not running =#=#=#=
 Resource dummy is not running
-
-  * Configuration specifies 'dummy' should remain stopped
-
+Configuration specifies 'dummy' should remain stopped
 =#=#=#= End test: Show why a resource is not running - OK (0) =#=#=#=
 * Passed: crm_resource   - Show why a resource is not running
 =#=#=#= Begin test: Remove another resource meta attribute =#=#=#=
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index 6e32982..f551059 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1868,7 +1868,7 @@ main(int argc, char **argv)
                         goto done;
                     }
                 }
-                cli_resource_why(out, cib_conn, data_set->resources, rsc, dest);
+                out->message(out, "resource-why", cib_conn, data_set->resources, rsc, dest);
                 rc = pcmk_rc_ok;
             }
             break;
diff --git a/tools/crm_resource.h b/tools/crm_resource.h
index 4fc7c71..377f7aa 100644
--- a/tools/crm_resource.h
+++ b/tools/crm_resource.h
@@ -23,6 +23,20 @@
 #include <crm/pengine/internal.h>
 #include <pacemaker-internal.h>
 
+enum resource_check_flags {
+    rsc_remain_stopped  = (1 << 0),
+    rsc_unpromotable    = (1 << 1),
+    rsc_unmanaged       = (1 << 2)
+};
+
+typedef struct resource_checks_s {
+    pe_resource_t *rsc;
+    unsigned int flags;
+    const char *lock_node;
+} resource_checks_t;
+
+resource_checks_t *cli_check_resource(pe_resource_t *rsc, char *role_s, char *managed);
+
 /* ban */
 int cli_resource_prefer(pcmk__output_t *out, const char *rsc_id, const char *host,
                         const char *move_lifetime, cib_t * cib_conn, int cib_options,
@@ -51,7 +65,7 @@ int cli_resource_print_operations(pcmk__output_t *out, const char *rsc_id,
                                   pe_working_set_t * data_set);
 
 /* runtime */
-void cli_resource_check(pcmk__output_t *out, cib_t * cib, pe_resource_t *rsc);
+int cli_resource_check(pcmk__output_t *out, cib_t * cib, pe_resource_t *rsc);
 int cli_resource_fail(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
                       const char *host_uname, const char *rsc_id,
                       pe_working_set_t *data_set);
@@ -98,7 +112,7 @@ int cli_resource_delete_attribute(pcmk__output_t *out, pe_resource_t *rsc,
 
 int update_working_set_xml(pe_working_set_t *data_set, xmlNode **xml);
 int wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib);
-void cli_resource_why(pcmk__output_t *out, cib_t *cib_conn, GListPtr resources,
-                      pe_resource_t *rsc, pe_node_t *node);
+
+bool resource_is_running_on(pe_resource_t *rsc, const char *host);
 
 void crm_resource_register_messages(pcmk__output_t *out);
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
index 4b3c42d..d2a2cc8 100644
--- a/tools/crm_resource_print.c
+++ b/tools/crm_resource_print.c
@@ -8,6 +8,7 @@
  */
 
 #include <crm_resource.h>
+#include <crm/common/lists_internal.h>
 #include <crm/common/xml_internal.h>
 #include <crm/common/output_internal.h>
 
@@ -287,6 +288,261 @@ property_text(pcmk__output_t *out, va_list args) {
     return pcmk_rc_ok;
 }
 
+PCMK__OUTPUT_ARGS("resource-check", "resource_checks_t *")
+static int
+resource_check_default(pcmk__output_t *out, va_list args) {
+    resource_checks_t *checks = va_arg(args, resource_checks_t *);
+
+    pe_resource_t *parent = uber_parent(checks->rsc);
+    int rc = pcmk_rc_no_output;
+    bool printed = false;
+
+    if (checks->flags != 0 || checks->lock_node != NULL) {
+        printed = true;
+        out->begin_list(out, NULL, NULL, "Resource Checks");
+    }
+
+    if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
+        out->list_item(out, "check", "Configuration specifies '%s' should remain stopped",
+                       parent->id);
+    }
+
+    if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
+        out->list_item(out, "check", "Configuration specifies '%s' should not be promoted",
+                       parent->id);
+    }
+
+    if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
+        out->list_item(out, "check", "Configuration prevents cluster from stopping or starting unmanaged '%s'",
+                       parent->id);
+    }
+
+    if (checks->lock_node) {
+        out->list_item(out, "check", "'%s' is locked to node %s due to shutdown",
+                       parent->id, checks->lock_node);
+    }
+
+    if (printed) {
+        out->end_list(out);
+        rc = pcmk_rc_ok;
+    }
+
+    return rc;
+}
+
+PCMK__OUTPUT_ARGS("resource-check", "resource_checks_t *")
+static int
+resource_check_xml(pcmk__output_t *out, va_list args) {
+    resource_checks_t *checks = va_arg(args, resource_checks_t *);
+
+    pe_resource_t *parent = uber_parent(checks->rsc);
+    int rc = pcmk_rc_no_output;
+
+    xmlNode *node = pcmk__output_create_xml_node(out, "check");
+
+    xmlSetProp(node, (pcmkXmlStr) "id", (pcmkXmlStr) parent->id);
+
+    if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
+        xmlSetProp(node, (pcmkXmlStr) "remain_stopped", (pcmkXmlStr) "true");
+    }
+
+    if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
+        xmlSetProp(node, (pcmkXmlStr) "promotable", (pcmkXmlStr) "false");
+    }
+
+    if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
+        xmlSetProp(node, (pcmkXmlStr) "unmanaged", (pcmkXmlStr) "true");
+    }
+
+    if (checks->lock_node) {
+        xmlSetProp(node, (pcmkXmlStr) "locked-to", (pcmkXmlStr) checks->lock_node);
+    }
+
+    return rc;
+}
+
+PCMK__OUTPUT_ARGS("resource-why", "cib_t *", "GListPtr", "pe_resource_t *",
+                  "pe_node_t *")
+static int
+resource_why_default(pcmk__output_t *out, va_list args)
+{
+    cib_t *cib_conn = va_arg(args, cib_t *);
+    GListPtr resources = va_arg(args, GListPtr);
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    pe_node_t *node = va_arg(args, pe_node_t *);
+
+    const char *host_uname = (node == NULL)? NULL : node->details->uname;
+
+    out->begin_list(out, NULL, NULL, "Resource Reasons");
+
+    if ((rsc == NULL) && (host_uname == NULL)) {
+        GListPtr lpc = NULL;
+        GListPtr hosts = NULL;
+
+        for (lpc = resources; lpc != NULL; lpc = lpc->next) {
+            pe_resource_t *rsc = (pe_resource_t *) lpc->data;
+            rsc->fns->location(rsc, &hosts, TRUE);
+
+            if (hosts == NULL) {
+                out->list_item(out, "reason", "Resource %s is not running", rsc->id);
+            } else {
+                out->list_item(out, "reason", "Resource %s is running", rsc->id);
+            }
+
+            cli_resource_check(out, cib_conn, rsc);
+            g_list_free(hosts);
+            hosts = NULL;
+        }
+
+    } else if ((rsc != NULL) && (host_uname != NULL)) {
+        if (resource_is_running_on(rsc, host_uname)) {
+            out->list_item(out, "reason", "Resource %s is running on host %s",
+                           rsc->id, host_uname);
+        } else {
+            out->list_item(out, "reason", "Resource %s is not running on host %s",
+                           rsc->id, host_uname);
+        }
+
+        cli_resource_check(out, cib_conn, rsc);
+
+    } else if ((rsc == NULL) && (host_uname != NULL)) {
+        const char* host_uname =  node->details->uname;
+        GListPtr allResources = node->details->allocated_rsc;
+        GListPtr activeResources = node->details->running_rsc;
+        GListPtr unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
+        GListPtr lpc = NULL;
+
+        for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
+            pe_resource_t *rsc = (pe_resource_t *) lpc->data;
+            out->list_item(out, "reason", "Resource %s is running on host %s",
+                           rsc->id, host_uname);
+            cli_resource_check(out, cib_conn, rsc);
+        }
+
+        for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
+            pe_resource_t *rsc = (pe_resource_t *) lpc->data;
+            out->list_item(out, "reason", "Resource %s is assigned to host %s but not running",
+                           rsc->id, host_uname);
+            cli_resource_check(out, cib_conn, rsc);
+        }
+
+        g_list_free(allResources);
+        g_list_free(activeResources);
+        g_list_free(unactiveResources);
+
+    } else if ((rsc != NULL) && (host_uname == NULL)) {
+        GListPtr hosts = NULL;
+
+        rsc->fns->location(rsc, &hosts, TRUE);
+        out->list_item(out, "reason", "Resource %s is %srunning",
+                       rsc->id, (hosts? "" : "not "));
+        cli_resource_check(out, cib_conn, rsc);
+        g_list_free(hosts);
+    }
+
+    out->end_list(out);
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("resource-why", "cib_t *", "GListPtr", "pe_resource_t *",
+                  "pe_node_t *")
+static int
+resource_why_xml(pcmk__output_t *out, va_list args)
+{
+    cib_t *cib_conn = va_arg(args, cib_t *);
+    GListPtr resources = va_arg(args, GListPtr);
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    pe_node_t *node = va_arg(args, pe_node_t *);
+
+    const char *host_uname = (node == NULL)? NULL : node->details->uname;
+
+    xmlNode *xml_node = pcmk__output_xml_create_parent(out, "reason");
+
+    if ((rsc == NULL) && (host_uname == NULL)) {
+        GListPtr lpc = NULL;
+        GListPtr hosts = NULL;
+
+        pcmk__output_xml_create_parent(out, "resources");
+
+        for (lpc = resources; lpc != NULL; lpc = lpc->next) {
+            pe_resource_t *rsc = (pe_resource_t *) lpc->data;
+            xmlNode *rsc_node = NULL;
+
+            rsc->fns->location(rsc, &hosts, TRUE);
+
+            rsc_node = pcmk__output_xml_create_parent(out, "resource");
+            xmlSetProp(rsc_node, (pcmkXmlStr) "id", (pcmkXmlStr) rsc->id);
+            xmlSetProp(rsc_node, (pcmkXmlStr) "running",
+                       (pcmkXmlStr) pcmk__btoa(hosts != NULL));
+
+            cli_resource_check(out, cib_conn, rsc);
+            pcmk__output_xml_pop_parent(out);
+            g_list_free(hosts);
+            hosts = NULL;
+        }
+
+        pcmk__output_xml_pop_parent(out);
+
+    } else if ((rsc != NULL) && (host_uname != NULL)) {
+        if (resource_is_running_on(rsc, host_uname)) {
+            xmlSetProp(xml_node, (pcmkXmlStr) "running_on", (pcmkXmlStr) host_uname);
+        }
+
+        cli_resource_check(out, cib_conn, rsc);
+
+    } else if ((rsc == NULL) && (host_uname != NULL)) {
+        const char* host_uname =  node->details->uname;
+        GListPtr allResources = node->details->allocated_rsc;
+        GListPtr activeResources = node->details->running_rsc;
+        GListPtr unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
+        GListPtr lpc = NULL;
+
+        pcmk__output_xml_create_parent(out, "resources");
+
+        for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
+            pe_resource_t *rsc = (pe_resource_t *) lpc->data;
+            xmlNode *rsc_node = NULL;
+
+            rsc_node = pcmk__output_xml_create_parent(out, "resource");
+            xmlSetProp(rsc_node, (pcmkXmlStr) "id", (pcmkXmlStr) rsc->id);
+            xmlSetProp(rsc_node, (pcmkXmlStr) "running", (pcmkXmlStr) "true");
+            xmlSetProp(rsc_node, (pcmkXmlStr) "host", (pcmkXmlStr) host_uname);
+
+            cli_resource_check(out, cib_conn, rsc);
+            pcmk__output_xml_pop_parent(out);
+        }
+
+        for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
+            pe_resource_t *rsc = (pe_resource_t *) lpc->data;
+            xmlNode *rsc_node = NULL;
+
+            rsc_node = pcmk__output_xml_create_parent(out, "resource");
+            xmlSetProp(rsc_node, (pcmkXmlStr) "id", (pcmkXmlStr) rsc->id);
+            xmlSetProp(rsc_node, (pcmkXmlStr) "running", (pcmkXmlStr) "false");
+            xmlSetProp(rsc_node, (pcmkXmlStr) "host", (pcmkXmlStr) host_uname);
+
+            cli_resource_check(out, cib_conn, rsc);
+            pcmk__output_xml_pop_parent(out);
+        }
+
+        pcmk__output_xml_pop_parent(out);
+        g_list_free(allResources);
+        g_list_free(activeResources);
+        g_list_free(unactiveResources);
+
+    } else if ((rsc != NULL) && (host_uname == NULL)) {
+        GListPtr hosts = NULL;
+
+        rsc->fns->location(rsc, &hosts, TRUE);
+        xmlSetProp(xml_node, (pcmkXmlStr) "running",
+                   (pcmkXmlStr) pcmk__btoa(hosts != NULL));
+        cli_resource_check(out, cib_conn, rsc);
+        g_list_free(hosts);
+    }
+
+    return pcmk_rc_ok;
+}
+
 static void
 add_resource_name(pcmk__output_t *out, pe_resource_t *rsc) {
     if (rsc->children == NULL) {
@@ -325,6 +581,10 @@ static pcmk__message_entry_t fmt_functions[] = {
     { "attribute", "text", attribute_text },
     { "property", "default", property_default },
     { "property", "text", property_text },
+    { "resource-check", "default", resource_check_default },
+    { "resource-check", "xml", resource_check_xml },
+    { "resource-why", "default", resource_why_default },
+    { "resource-why", "xml", resource_why_xml },
     { "resource-names-list", "default", resource_names },
 
     { NULL, NULL, NULL }
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index bd377a3..4f8287b 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -43,6 +43,35 @@ do_find_resource(pcmk__output_t *out, const char *rsc, pe_resource_t * the_rsc,
     return found;
 }
 
+resource_checks_t *
+cli_check_resource(pe_resource_t *rsc, char *role_s, char *managed)
+{
+    pe_resource_t *parent = uber_parent(rsc);
+    resource_checks_t *rc = calloc(1, sizeof(resource_checks_t));
+
+    if (role_s) {
+        enum rsc_role_e role = text2role(role_s);
+
+        if (role == RSC_ROLE_STOPPED) {
+            rc->flags |= rsc_remain_stopped;
+        } else if (pcmk_is_set(parent->flags, pe_rsc_promotable) &&
+                   role == RSC_ROLE_SLAVE) {
+            rc->flags |= rsc_unpromotable;
+        }
+    }
+
+    if (managed && !crm_is_true(managed)) {
+        rc->flags |= rsc_unmanaged;
+    }
+
+    if (rsc->lock_node) {
+        rc->lock_node = rsc->lock_node->details->uname;
+    }
+
+    rc->rsc = rsc;
+    return rc;
+}
+
 int
 cli_resource_search(pcmk__output_t *out, pe_resource_t *rsc, const char *requested_name,
                     pe_working_set_t *data_set)
@@ -878,13 +907,14 @@ cli_cleanup_all(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
     return rc;
 }
 
-void
+int
 cli_resource_check(pcmk__output_t *out, cib_t * cib_conn, pe_resource_t *rsc)
 {
-    bool printed = false;
     char *role_s = NULL;
     char *managed = NULL;
     pe_resource_t *parent = uber_parent(rsc);
+    int rc = pcmk_rc_no_output;
+    resource_checks_t *checks = NULL;
 
     find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
                        NULL, NULL, NULL, XML_RSC_ATTR_MANAGED, &managed);
@@ -892,41 +922,16 @@ cli_resource_check(pcmk__output_t *out, cib_t * cib_conn, pe_resource_t *rsc)
     find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
                        NULL, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, &role_s);
 
-    if(role_s) {
-        enum rsc_role_e role = text2role(role_s);
+    checks = cli_check_resource(rsc, role_s, managed);
 
-        free(role_s);
-        if(role == RSC_ROLE_UNKNOWN) {
-            // Treated as if unset
-
-        } else if(role == RSC_ROLE_STOPPED) {
-            printf("\n  * Configuration specifies '%s' should remain stopped\n",
-                   parent->id);
-            printed = true;
-
-        } else if (pcmk_is_set(parent->flags, pe_rsc_promotable)
-                   && (role == RSC_ROLE_SLAVE)) {
-            printf("\n  * Configuration specifies '%s' should not be promoted\n",
-                   parent->id);
-            printed = true;
-        }
+    if (checks->flags != 0 || checks->lock_node != NULL) {
+        rc = out->message(out, "resource-check", checks);
     }
 
-    if (managed && !crm_is_true(managed)) {
-        printf("%s  * Configuration prevents cluster from stopping or starting unmanaged '%s'\n",
-               (printed? "" : "\n"), parent->id);
-        printed = true;
-    }
+    free(role_s);
     free(managed);
-
-    if (rsc->lock_node) {
-        printf("%s  * '%s' is locked to node %s due to shutdown\n",
-               (printed? "" : "\n"), parent->id, rsc->lock_node->details->uname);
-    }
-
-    if (printed) {
-        printf("\n");
-    }
+    free(checks);
+    return rc;
 }
 
 // \return Standard Pacemaker return code
@@ -986,7 +991,7 @@ generate_resource_params(pe_resource_t * rsc, pe_working_set_t * data_set)
     return combined;
 }
 
-static bool resource_is_running_on(pe_resource_t *rsc, const char *host) 
+bool resource_is_running_on(pe_resource_t *rsc, const char *host)
 {
     bool found = TRUE;
     GListPtr hIter = NULL;
@@ -1977,100 +1982,3 @@ cli_resource_move(pcmk__output_t *out, pe_resource_t *rsc, const char *rsc_id,
 
     return rc;
 }
-
-static void
-cli_resource_why_without_rsc_and_host(pcmk__output_t *out, cib_t *cib_conn,
-                                      GListPtr resources)
-{
-    GListPtr lpc = NULL;
-    GListPtr hosts = NULL;
-
-    for (lpc = resources; lpc != NULL; lpc = lpc->next) {
-        pe_resource_t *rsc = (pe_resource_t *) lpc->data;
-        rsc->fns->location(rsc, &hosts, TRUE);
-
-        if (hosts == NULL) {
-            printf("Resource %s is not running\n", rsc->id);
-        } else {
-            printf("Resource %s is running\n", rsc->id);
-        }
-
-        cli_resource_check(out, cib_conn, rsc);
-        g_list_free(hosts);
-        hosts = NULL;
-     }
-
-}
-
-static void
-cli_resource_why_with_rsc_and_host(pcmk__output_t *out, cib_t *cib_conn,
-                                   GListPtr resources, pe_resource_t *rsc,
-                                   const char *host_uname)
-{
-    if (resource_is_running_on(rsc, host_uname)) {
-        printf("Resource %s is running on host %s\n",rsc->id,host_uname);
-    } else {
-        printf("Resource %s is not running on host %s\n", rsc->id, host_uname);
-    }
-    cli_resource_check(out, cib_conn, rsc);
-}
-
-static void
-cli_resource_why_without_rsc_with_host(pcmk__output_t *out, cib_t *cib_conn,
-                                       GListPtr resources, pe_node_t *node)
-{
-    const char* host_uname =  node->details->uname;
-    GListPtr allResources = node->details->allocated_rsc;
-    GListPtr activeResources = node->details->running_rsc;
-    GListPtr unactiveResources = pcmk__subtract_lists(allResources,activeResources,(GCompareFunc) strcmp);
-    GListPtr lpc = NULL;
-
-    for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
-        pe_resource_t *rsc = (pe_resource_t *) lpc->data;
-        printf("Resource %s is running on host %s\n",rsc->id,host_uname);
-        cli_resource_check(out, cib_conn, rsc);
-    }
-
-    for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
-        pe_resource_t *rsc = (pe_resource_t *) lpc->data;
-        printf("Resource %s is assigned to host %s but not running\n",
-               rsc->id, host_uname);
-        cli_resource_check(out, cib_conn, rsc);
-     }
-
-     g_list_free(allResources);
-     g_list_free(activeResources);
-     g_list_free(unactiveResources);
-}
-
-static void
-cli_resource_why_with_rsc_without_host(pcmk__output_t *out, cib_t *cib_conn,
-                                       GListPtr resources, pe_resource_t *rsc)
-{
-    GListPtr hosts = NULL;
-
-    rsc->fns->location(rsc, &hosts, TRUE);
-    printf("Resource %s is %srunning\n", rsc->id, (hosts? "" : "not "));
-    cli_resource_check(out, cib_conn, rsc);
-    g_list_free(hosts);
-}
-
-void cli_resource_why(pcmk__output_t *out, cib_t *cib_conn, GListPtr resources,
-                      pe_resource_t *rsc, pe_node_t *node)
-{
-    const char *host_uname = (node == NULL)? NULL : node->details->uname;
-
-    if ((rsc == NULL) && (host_uname == NULL)) {
-        cli_resource_why_without_rsc_and_host(out, cib_conn, resources);
-
-    } else if ((rsc != NULL) && (host_uname != NULL)) {
-        cli_resource_why_with_rsc_and_host(out, cib_conn, resources, rsc,
-                                           host_uname);
-
-    } else if ((rsc == NULL) && (host_uname != NULL)) {
-        cli_resource_why_without_rsc_with_host(out, cib_conn, resources, node);
-
-    } else if ((rsc != NULL) && (host_uname == NULL)) {
-        cli_resource_why_with_rsc_without_host(out, cib_conn, resources, rsc);
-    }
-}
-- 
1.8.3.1


From b5ce7803e3b072066458e93802688a1ec15875ce Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Fri, 2 Oct 2020 12:29:22 -0400
Subject: [PATCH 16/19] Feature: tools: Use formatted output for resource
 searching.

This reorganizes the cli_resource_search and do_find_resource functions
into more logical functions.  cli_resource_search now just puts together
a list of resources and returns it.  This list should not be freed
because it reuses existing lists.  Then, there is a formatted output
message to do the actual printing.
---
 tools/crm_resource.c         |  7 +++--
 tools/crm_resource.h         |  4 +--
 tools/crm_resource_print.c   | 72 ++++++++++++++++++++++++++++++++++++++++++++
 tools/crm_resource_runtime.c | 53 +++++++++-----------------------
 4 files changed, 92 insertions(+), 44 deletions(-)

diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index f551059..b341194 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1844,10 +1844,11 @@ main(int argc, char **argv)
                                                data_set);
             break;
 
-        case cmd_locate:
-            cli_resource_search(out, rsc, options.rsc_id, data_set);
-            rc = pcmk_rc_ok;
+        case cmd_locate: {
+            GListPtr resources = cli_resource_search(out, rsc, options.rsc_id, data_set);
+            rc = out->message(out, "resource-search", resources, rsc, options.rsc_id);
             break;
+        }
 
         case cmd_query_xml:
             rc = cli_resource_print(out, rsc, data_set, TRUE);
diff --git a/tools/crm_resource.h b/tools/crm_resource.h
index 377f7aa..6de2457 100644
--- a/tools/crm_resource.h
+++ b/tools/crm_resource.h
@@ -69,8 +69,8 @@ int cli_resource_check(pcmk__output_t *out, cib_t * cib, pe_resource_t *rsc);
 int cli_resource_fail(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
                       const char *host_uname, const char *rsc_id,
                       pe_working_set_t *data_set);
-int cli_resource_search(pcmk__output_t *out, pe_resource_t *rsc,
-                        const char *requested_name, pe_working_set_t *data_set);
+GListPtr cli_resource_search(pcmk__output_t *out, pe_resource_t *rsc,
+                             const char *requested_name, pe_working_set_t *data_set);
 int cli_resource_delete(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
                         const char *host_uname, pe_resource_t *rsc,
                         const char *operation, const char *interval_spec,
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
index d2a2cc8..5ff3e9b 100644
--- a/tools/crm_resource_print.c
+++ b/tools/crm_resource_print.c
@@ -361,6 +361,76 @@ resource_check_xml(pcmk__output_t *out, va_list args) {
     return rc;
 }
 
+PCMK__OUTPUT_ARGS("resource-search", "GListPtr", "pe_resource_t *", "gchar *")
+static int
+resource_search_default(pcmk__output_t *out, va_list args)
+{
+    GListPtr nodes = va_arg(args, GListPtr);
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    gchar *requested_name = va_arg(args, gchar *);
+
+    bool printed = false;
+    int rc = pcmk_rc_no_output;
+
+    if (!out->is_quiet(out) && nodes == NULL) {
+        out->err(out, "resource %s is NOT running", requested_name);
+        return rc;
+    }
+
+    for (GListPtr lpc = nodes; lpc != NULL; lpc = lpc->next) {
+        pe_node_t *node = (pe_node_t *) lpc->data;
+
+        if (!printed) {
+            out->begin_list(out, NULL, NULL, "Nodes");
+            printed = true;
+            rc = pcmk_rc_ok;
+        }
+
+        if (out->is_quiet(out)) {
+            out->list_item(out, "node", "%s", node->details->uname);
+        } else {
+            const char *state = "";
+
+            if (!pe_rsc_is_clone(rsc) && rsc->fns->state(rsc, TRUE) == RSC_ROLE_MASTER) {
+                state = " Master";
+            }
+            out->list_item(out, "node", "resource %s is running on: %s%s",
+                           requested_name, node->details->uname, state);
+        }
+    }
+
+    if (printed) {
+        out->end_list(out);
+    }
+
+    return rc;
+}
+
+
+PCMK__OUTPUT_ARGS("resource-search", "GListPtr", "pe_resource_t *", "gchar *")
+static int
+resource_search_xml(pcmk__output_t *out, va_list args)
+{
+    GListPtr nodes = va_arg(args, GListPtr);
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    gchar *requested_name = va_arg(args, gchar *);
+
+    xmlNode *xml_node = pcmk__output_xml_create_parent(out, "nodes");
+
+    xmlSetProp(xml_node, (pcmkXmlStr) "resource", (pcmkXmlStr) requested_name);
+
+    for (GListPtr lpc = nodes; lpc != NULL; lpc = lpc->next) {
+        pe_node_t *node = (pe_node_t *) lpc->data;
+        xmlNode *sub_node = pcmk__output_create_xml_text_node(out, "node", node->details->uname);
+
+        if (!pe_rsc_is_clone(rsc) && rsc->fns->state(rsc, TRUE) == RSC_ROLE_MASTER) {
+            xmlSetProp(sub_node, (pcmkXmlStr) "state", (pcmkXmlStr) "promoted");
+        }
+    }
+
+    return pcmk_rc_ok;
+}
+
 PCMK__OUTPUT_ARGS("resource-why", "cib_t *", "GListPtr", "pe_resource_t *",
                   "pe_node_t *")
 static int
@@ -583,6 +653,8 @@ static pcmk__message_entry_t fmt_functions[] = {
     { "property", "text", property_text },
     { "resource-check", "default", resource_check_default },
     { "resource-check", "xml", resource_check_xml },
+    { "resource-search", "default", resource_search_default },
+    { "resource-search", "xml", resource_search_xml },
     { "resource-why", "default", resource_why_default },
     { "resource-why", "xml", resource_why_xml },
     { "resource-names-list", "default", resource_names },
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 4f8287b..bbd8bc1 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -12,37 +12,6 @@
 #include <crm/common/lists_internal.h>
 #include <crm/common/xml_internal.h>
 
-static int
-do_find_resource(pcmk__output_t *out, const char *rsc, pe_resource_t * the_rsc,
-                 pe_working_set_t * data_set)
-{
-    int found = 0;
-    GListPtr lpc = NULL;
-
-    for (lpc = the_rsc->running_on; lpc != NULL; lpc = lpc->next) {
-        pe_node_t *node = (pe_node_t *) lpc->data;
-
-        if (out->is_quiet(out)) {
-            out->info(out, "%s", node->details->uname);
-        } else {
-            const char *state = "";
-
-            if (!pe_rsc_is_clone(the_rsc) && the_rsc->fns->state(the_rsc, TRUE) == RSC_ROLE_MASTER) {
-                state = "Master";
-            }
-            out->info(out, "resource %s is running on: %s %s", rsc, node->details->uname, state);
-        }
-
-        found++;
-    }
-
-    if (!out->is_quiet(out) && found == 0) {
-        out->err(out, "resource %s is NOT running", rsc);
-    }
-
-    return found;
-}
-
 resource_checks_t *
 cli_check_resource(pe_resource_t *rsc, char *role_s, char *managed)
 {
@@ -72,16 +41,19 @@ cli_check_resource(pe_resource_t *rsc, char *role_s, char *managed)
     return rc;
 }
 
-int
+GListPtr
 cli_resource_search(pcmk__output_t *out, pe_resource_t *rsc, const char *requested_name,
                     pe_working_set_t *data_set)
 {
-    int found = 0;
+    GListPtr found = NULL;
     pe_resource_t *parent = uber_parent(rsc);
 
     if (pe_rsc_is_clone(rsc)) {
         for (GListPtr iter = rsc->children; iter != NULL; iter = iter->next) {
-            found += do_find_resource(out, requested_name, iter->data, data_set);
+            GListPtr extra = ((pe_resource_t *) iter->data)->running_on;
+            if (extra != NULL) {
+                found = g_list_concat(found, extra);
+            }
         }
 
     /* The anonymous clone children's common ID is supplied */
@@ -92,11 +64,14 @@ cli_resource_search(pcmk__output_t *out, pe_resource_t *rsc, const char *request
                && !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {
 
         for (GListPtr iter = parent->children; iter; iter = iter->next) {
-            found += do_find_resource(out, requested_name, iter->data, data_set);
+            GListPtr extra = ((pe_resource_t *) iter->data)->running_on;
+            if (extra != NULL) {
+                found = g_list_concat(found, extra);
+            }
         }
 
-    } else {
-        found += do_find_resource(out, requested_name, rsc, data_set);
+    } else if (rsc->running_on != NULL) {
+        found = g_list_concat(found, rsc->running_on);
     }
 
     return found;
@@ -1828,8 +1803,8 @@ cli_resource_execute(pcmk__output_t *out, pe_resource_t *rsc,
         action = rsc_action+6;
 
         if(pe_rsc_is_clone(rsc)) {
-            int rc = cli_resource_search(out, rsc, requested_name, data_set);
-            if(rc > 0 && force == FALSE) {
+            GListPtr rscs = cli_resource_search(out, rsc, requested_name, data_set);
+            if(rscs != NULL && force == FALSE) {
                 out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
                          action, rsc->id);
                 out->err(out, "Try setting target-role=Stopped first or specifying "
-- 
1.8.3.1


From a74e1193e005994ec4cfe1991e13bb61fff64de9 Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Mon, 5 Oct 2020 13:54:11 -0400
Subject: [PATCH 17/19] Feature: tools: Use formatted output for stacks and
 constraints.

This also changes the format for text output.  The new output uses the
underlying text list code for handling indentation and nesting, while
the old code passed around an indentation prefix.  However, this does
mean we end up with a little of the text list fanciness as well.
Hopefully that will not be a problem.
---
 cts/cli/regression.tools.exp |   7 +-
 include/pcmki/pcmki_output.h |   7 +
 lib/pacemaker/pcmk_output.c  | 328 ++++++++++++++++++++++++++++++++++++++++++-
 tools/crm_resource.c         |  40 +-----
 tools/crm_resource.h         |   4 -
 tools/crm_resource_print.c   |  91 ------------
 6 files changed, 343 insertions(+), 134 deletions(-)

diff --git a/cts/cli/regression.tools.exp b/cts/cli/regression.tools.exp
index 7166714..221730d 100644
--- a/cts/cli/regression.tools.exp
+++ b/cts/cli/regression.tools.exp
@@ -2001,12 +2001,13 @@ WARNING: Creating rsc_location constraint 'cli-ban-dummy-on-node1' with a score
 =#=#=#= End test: Ban dummy from node1 - OK (0) =#=#=#=
 * Passed: crm_resource   - Ban dummy from node1
 =#=#=#= Begin test: Show where a resource is running =#=#=#=
-resource dummy is running on: node1 
+resource dummy is running on: node1
 =#=#=#= End test: Show where a resource is running - OK (0) =#=#=#=
 * Passed: crm_resource   - Show where a resource is running
 =#=#=#= Begin test: Show constraints on a resource =#=#=#=
-* dummy
-  : Node node1                                                                   (score=-INFINITY, id=cli-ban-dummy-on-node1)
+dummy:
+  * Locations:
+    * Node node1 (score=-INFINITY, id=cli-ban-dummy-on-node1)
 =#=#=#= End test: Show constraints on a resource - OK (0) =#=#=#=
 * Passed: crm_resource   - Show constraints on a resource
 =#=#=#= Begin test: Ban dummy from node2 =#=#=#=
diff --git a/include/pcmki/pcmki_output.h b/include/pcmki/pcmki_output.h
index 2b750fb..0faef35 100644
--- a/include/pcmki/pcmki_output.h
+++ b/include/pcmki/pcmki_output.h
@@ -21,6 +21,13 @@ extern pcmk__supported_format_t pcmk__out_formats[];
 int pcmk__out_prologue(pcmk__output_t **out, xmlNodePtr *xml);
 void pcmk__out_epilogue(pcmk__output_t *out, xmlNodePtr *xml, int retval);
 
+/* This function registers only the formatted output messages that are a part
+ * of libpacemaker.  It is not to be confused with pcmk__register_messages,
+ * which is a part of formatted output support and registers a whole table of
+ * messages at a time.
+ */
+void pcmk__register_lib_messages(pcmk__output_t *out);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/pacemaker/pcmk_output.c b/lib/pacemaker/pcmk_output.c
index adf4c34..306e561 100644
--- a/lib/pacemaker/pcmk_output.c
+++ b/lib/pacemaker/pcmk_output.c
@@ -13,7 +13,7 @@
 #include <crm/stonith-ng.h>
 #include <crm/fencing/internal.h>
 #include <libxml/tree.h>
-#include <pcmki/pcmki_output.h>
+#include <pacemaker-internal.h>
 
 pcmk__supported_format_t pcmk__out_formats[] = {
     PCMK__SUPPORTED_FORMAT_XML,
@@ -46,3 +46,329 @@ pcmk__out_epilogue(pcmk__output_t *out, xmlNodePtr *xml, int retval) {
 
     pcmk__output_free(out);
 }
+
+PCMK__OUTPUT_ARGS("colocations-list", "pe_resource_t *", "gboolean", "gboolean")
+static int colocations_list(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    gboolean dependents = va_arg(args, gboolean);
+    gboolean recursive = va_arg(args, gboolean);
+
+    GListPtr lpc = NULL;
+    GListPtr list = rsc->rsc_cons;
+    bool printed_header = false;
+
+    if (dependents) {
+        list = rsc->rsc_cons_lhs;
+    }
+
+    if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
+        return pcmk_rc_no_output;
+    }
+
+    pe__set_resource_flags(rsc, pe_rsc_allocating);
+    for (lpc = list; lpc != NULL; lpc = lpc->next) {
+        rsc_colocation_t *cons = (rsc_colocation_t *) lpc->data;
+
+        char *score = NULL;
+        pe_resource_t *peer = cons->rsc_rh;
+
+        if (dependents) {
+            peer = cons->rsc_lh;
+        }
+
+        if (pcmk_is_set(peer->flags, pe_rsc_allocating)) {
+            if (dependents == FALSE) {
+                if (!printed_header) {
+                    out->begin_list(out, NULL, NULL, "Colocations");
+                    printed_header = true;
+                }
+
+                out->list_item(out, NULL, "%s (id=%s - loop)", peer->id, cons->id);
+            }
+            continue;
+        }
+
+        if (dependents && recursive) {
+            if (!printed_header) {
+                out->begin_list(out, NULL, NULL, "Colocations");
+                printed_header = true;
+            }
+
+            out->message(out, "colocations-list", rsc, dependents, recursive);
+        }
+
+        if (!printed_header) {
+            out->begin_list(out, NULL, NULL, "Colocations");
+            printed_header = true;
+        }
+
+        score = score2char(cons->score);
+        if (cons->role_rh > RSC_ROLE_STARTED) {
+            out->list_item(out, NULL, "%s (score=%s, %s role=%s, id=%s",
+                           peer->id, score, dependents ? "needs" : "with",
+                           role2text(cons->role_rh), cons->id);
+        } else {
+            out->list_item(out, NULL, "%s (score=%s, id=%s",
+                           peer->id, score, cons->id);
+        }
+
+        free(score);
+        out->message(out, "locations-list", peer);
+
+        if (!dependents && recursive) {
+            out->message(out, "colocations-list", rsc, dependents, recursive);
+        }
+    }
+
+    if (printed_header) {
+        out->end_list(out);
+    }
+
+    return pcmk_rc_no_output;
+}
+
+PCMK__OUTPUT_ARGS("colocations-list", "pe_resource_t *", "gboolean", "gboolean")
+static int colocations_list_xml(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    gboolean dependents = va_arg(args, gboolean);
+    gboolean recursive = va_arg(args, gboolean);
+
+    GListPtr lpc = NULL;
+    GListPtr list = rsc->rsc_cons;
+    bool printed_header = false;
+
+    if (dependents) {
+        list = rsc->rsc_cons_lhs;
+    }
+
+    if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
+        return pcmk_rc_ok;
+    }
+
+    pe__set_resource_flags(rsc, pe_rsc_allocating);
+    for (lpc = list; lpc != NULL; lpc = lpc->next) {
+        rsc_colocation_t *cons = (rsc_colocation_t *) lpc->data;
+        pe_resource_t *peer = cons->rsc_rh;
+        char *score = NULL;
+
+        if (dependents) {
+            peer = cons->rsc_lh;
+        }
+
+        if (pcmk_is_set(peer->flags, pe_rsc_allocating)) {
+            if (dependents == FALSE) {
+                xmlNodePtr node;
+
+                if (!printed_header) {
+                    pcmk__output_xml_create_parent(out, "colocations");
+                    printed_header = true;
+                }
+
+                node = pcmk__output_create_xml_node(out, "colocation");
+                xmlSetProp(node, (pcmkXmlStr) "peer", (pcmkXmlStr) peer->id);
+                xmlSetProp(node, (pcmkXmlStr) "id", (pcmkXmlStr) cons->id);
+            }
+            continue;
+        }
+
+        if (dependents && recursive) {
+            if (!printed_header) {
+                pcmk__output_xml_create_parent(out, "colocations");
+                printed_header = true;
+            }
+
+            out->message(out, "colocations-list", rsc, dependents, recursive);
+        }
+
+        if (!printed_header) {
+            pcmk__output_xml_create_parent(out, "colocations");
+            printed_header = true;
+        }
+
+        score = score2char(cons->score);
+        if (cons->role_rh > RSC_ROLE_STARTED) {
+            xmlNodePtr node = pcmk__output_create_xml_node(out, "colocation");
+            xmlSetProp(node, (pcmkXmlStr) "peer", (pcmkXmlStr) peer->id);
+            xmlSetProp(node, (pcmkXmlStr) "id", (pcmkXmlStr) cons->id);
+            xmlSetProp(node, (pcmkXmlStr) "score", (pcmkXmlStr) score);
+            xmlSetProp(node, (pcmkXmlStr) "dependents",
+                       (pcmkXmlStr) (dependents ? "needs" : "with"));
+            xmlSetProp(node, (pcmkXmlStr) "role", (pcmkXmlStr) role2text(cons->role_rh));
+        } else {
+            xmlNodePtr node = pcmk__output_create_xml_node(out, "colocation");
+            xmlSetProp(node, (pcmkXmlStr) "peer", (pcmkXmlStr) peer->id);
+            xmlSetProp(node, (pcmkXmlStr) "id", (pcmkXmlStr) cons->id);
+            xmlSetProp(node, (pcmkXmlStr) "score", (pcmkXmlStr) score);
+        }
+
+        free(score);
+        out->message(out, "locations-list", peer);
+
+        if (!dependents && recursive) {
+            out->message(out, "colocations-list", rsc, dependents, recursive);
+        }
+    }
+
+    if (printed_header) {
+        pcmk__output_xml_pop_parent(out);
+    }
+
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("locations-list", "pe_resource_t *")
+static int locations_list(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc G_GNUC_UNUSED = va_arg(args, pe_resource_t *);
+
+    GListPtr lpc = NULL;
+    GListPtr list = rsc->rsc_location;
+
+    out->begin_list(out, NULL, NULL, "Locations");
+
+    for (lpc = list; lpc != NULL; lpc = lpc->next) {
+        pe__location_t *cons = lpc->data;
+
+        GListPtr lpc2 = NULL;
+
+        for (lpc2 = cons->node_list_rh; lpc2 != NULL; lpc2 = lpc2->next) {
+            pe_node_t *node = (pe_node_t *) lpc2->data;
+            char *score = score2char(node->weight);
+
+            out->list_item(out, NULL, "Node %s (score=%s, id=%s)",
+                           node->details->uname, score, cons->id);
+            free(score);
+        }
+    }
+
+    out->end_list(out);
+
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("locations-list", "pe_resource_t *")
+static int locations_list_xml(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+
+    GListPtr lpc = NULL;
+    GListPtr list = rsc->rsc_location;
+
+    pcmk__output_xml_create_parent(out, "locations");
+
+    for (lpc = list; lpc != NULL; lpc = lpc->next) {
+        pe__location_t *cons = lpc->data;
+
+        GListPtr lpc2 = NULL;
+
+        for (lpc2 = cons->node_list_rh; lpc2 != NULL; lpc2 = lpc2->next) {
+            pe_node_t *node = (pe_node_t *) lpc2->data;
+            char *score = score2char(node->weight);
+
+            xmlNodePtr xml_node = pcmk__output_create_xml_node(out, "location");
+            xmlSetProp(xml_node, (pcmkXmlStr) "host", (pcmkXmlStr) node->details->uname);
+            xmlSetProp(xml_node, (pcmkXmlStr) "id", (pcmkXmlStr) cons->id);
+            xmlSetProp(xml_node, (pcmkXmlStr) "score", (pcmkXmlStr) score);
+
+            free(score);
+        }
+    }
+
+    pcmk__output_xml_pop_parent(out);
+
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("stacks-constraints", "pe_resource_t *", "pe_working_set_t *", "gboolean")
+static int
+stacks_and_constraints(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc G_GNUC_UNUSED = va_arg(args, pe_resource_t *);
+    pe_working_set_t *data_set G_GNUC_UNUSED = va_arg(args, pe_working_set_t *);
+    gboolean recursive G_GNUC_UNUSED = va_arg(args, gboolean);
+
+    GListPtr lpc = NULL;
+    xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS,
+                                               data_set->input);
+
+    unpack_constraints(cib_constraints, data_set);
+
+    // Constraints apply to group/clone, not member/instance
+    rsc = uber_parent(rsc);
+
+    for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
+        pe_resource_t *r = (pe_resource_t *) lpc->data;
+
+        pe__clear_resource_flags(r, pe_rsc_allocating);
+    }
+
+    out->message(out, "colocations-list", rsc, TRUE, recursive);
+
+    out->begin_list(out, NULL, NULL, "%s", rsc->id);
+    out->message(out, "locations-list", rsc);
+    out->end_list(out);
+
+    for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
+        pe_resource_t *r = (pe_resource_t *) lpc->data;
+
+        pe__clear_resource_flags(r, pe_rsc_allocating);
+    }
+
+    out->message(out, "colocations-list", rsc, FALSE, recursive);
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("stacks-constraints", "pe_resource_t *", "pe_working_set_t *", "gboolean")
+static int
+stacks_and_constraints_xml(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    pe_working_set_t *data_set = va_arg(args, pe_working_set_t *);
+    gboolean recursive = va_arg(args, gboolean);
+
+    GListPtr lpc = NULL;
+    xmlNodePtr node = NULL;
+    xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS,
+                                               data_set->input);
+
+    unpack_constraints(cib_constraints, data_set);
+
+    // Constraints apply to group/clone, not member/instance
+    rsc = uber_parent(rsc);
+
+    for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
+        pe_resource_t *r = (pe_resource_t *) lpc->data;
+
+        pe__clear_resource_flags(r, pe_rsc_allocating);
+    }
+
+    pcmk__output_xml_create_parent(out, "constraints");
+
+    out->message(out, "colocations-list", rsc, TRUE, recursive);
+
+    node = pcmk__output_xml_create_parent(out, "resource");
+    xmlSetProp(node, (pcmkXmlStr) "id", (pcmkXmlStr) rsc->id);
+    out->message(out, "locations-list", rsc);
+    pcmk__output_xml_pop_parent(out);
+
+    for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
+        pe_resource_t *r = (pe_resource_t *) lpc->data;
+
+        pe__clear_resource_flags(r, pe_rsc_allocating);
+    }
+
+    out->message(out, "colocations-list", rsc, FALSE, recursive);
+    return pcmk_rc_ok;
+}
+
+static pcmk__message_entry_t fmt_functions[] = {
+    { "colocations-list", "default", colocations_list },
+    { "colocations-list", "xml", colocations_list_xml },
+    { "locations-list", "default", locations_list },
+    { "locations-list", "xml", locations_list_xml },
+    { "stacks-constraints", "default", stacks_and_constraints },
+    { "stacks-constraints", "xml", stacks_and_constraints_xml },
+
+    { NULL, NULL, NULL }
+};
+
+void
+pcmk__register_lib_messages(pcmk__output_t *out) {
+    pcmk__register_messages(out, fmt_functions);
+}
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index b341194..2c62ff6 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1194,38 +1194,6 @@ list_providers(pcmk__output_t *out, const char *agent_spec, crm_exit_t *exit_cod
     return rc;
 }
 
-static void
-list_stacks_and_constraints(pcmk__output_t *out, pe_resource_t *rsc, bool recursive)
-{
-    GListPtr lpc = NULL;
-    xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS,
-                                               data_set->input);
-
-    unpack_constraints(cib_constraints, data_set);
-
-    // Constraints apply to group/clone, not member/instance
-    rsc = uber_parent(rsc);
-
-    for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
-        pe_resource_t *r = (pe_resource_t *) lpc->data;
-
-        pe__clear_resource_flags(r, pe_rsc_allocating);
-    }
-
-    cli_resource_print_colocation(out, rsc, TRUE, recursive, 1);
-
-    fprintf(stdout, "* %s\n", rsc->id);
-    cli_resource_print_location(out, rsc, NULL);
-
-    for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
-        pe_resource_t *r = (pe_resource_t *) lpc->data;
-
-        pe__clear_resource_flags(r, pe_rsc_allocating);
-    }
-
-    cli_resource_print_colocation(out, rsc, FALSE, recursive, 1);
-}
-
 static int
 populate_working_set(xmlNodePtr *cib_xml_copy)
 {
@@ -1629,7 +1597,8 @@ main(int argc, char **argv)
             pcmk__force_args(context, &error, "%s --xml-substitute", g_get_prgname());
         }
     } else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches)) {
-        if (options.rsc_cmd == cmd_list_resources) {
+        if (options.rsc_cmd == cmd_colocations || options.rsc_cmd == cmd_colocations_deep ||
+            options.rsc_cmd == cmd_list_resources) {
             pcmk__force_args(context, &error, "%s --text-fancy", g_get_prgname());
         }
     }
@@ -1637,6 +1606,7 @@ main(int argc, char **argv)
     pe__register_messages(out);
     crm_resource_register_messages(out);
     lrmd__register_messages(out);
+    pcmk__register_lib_messages(out);
 
     if (args->version) {
         out->version(out, false);
@@ -1804,11 +1774,11 @@ main(int argc, char **argv)
             break;
 
         case cmd_colocations:
-            list_stacks_and_constraints(out, rsc, false);
+            rc = out->message(out, "stacks-constraints", rsc, data_set, false);
             break;
 
         case cmd_colocations_deep:
-            list_stacks_and_constraints(out, rsc, true);
+            rc = out->message(out, "stacks-constraints", rsc, data_set, true);
             break;
 
         case cmd_cts:
diff --git a/tools/crm_resource.h b/tools/crm_resource.h
index 6de2457..5bfadb7 100644
--- a/tools/crm_resource.h
+++ b/tools/crm_resource.h
@@ -53,10 +53,6 @@ int cli_resource_clear_all_expired(xmlNode *root, cib_t *cib_conn, int cib_optio
 void cli_resource_print_cts(pcmk__output_t *out, pe_resource_t * rsc);
 void cli_resource_print_raw(pcmk__output_t *out, pe_resource_t * rsc);
 void cli_resource_print_cts_constraints(pcmk__output_t *out, pe_working_set_t * data_set);
-void cli_resource_print_location(pcmk__output_t *out, pe_resource_t * rsc,
-                                 const char *prefix);
-void cli_resource_print_colocation(pcmk__output_t *out, pe_resource_t * rsc,
-                                   bool dependents, bool recursive, int offset);
 
 int cli_resource_print(pcmk__output_t *out, pe_resource_t *rsc, pe_working_set_t *data_set,
                        bool expanded);
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
index 5ff3e9b..6303863 100644
--- a/tools/crm_resource_print.c
+++ b/tools/crm_resource_print.c
@@ -108,97 +108,6 @@ cli_resource_print_operations(pcmk__output_t *out, const char *rsc_id,
     return rc;
 }
 
-void
-cli_resource_print_location(pcmk__output_t *out, pe_resource_t * rsc, const char *prefix)
-{
-    GListPtr lpc = NULL;
-    GListPtr list = rsc->rsc_location;
-    int offset = 0;
-
-    if (prefix) {
-        offset = strlen(prefix) - 2;
-    }
-
-    for (lpc = list; lpc != NULL; lpc = lpc->next) {
-        pe__location_t *cons = lpc->data;
-
-        GListPtr lpc2 = NULL;
-
-        for (lpc2 = cons->node_list_rh; lpc2 != NULL; lpc2 = lpc2->next) {
-            pe_node_t *node = (pe_node_t *) lpc2->data;
-            char *score = score2char(node->weight);
-
-            fprintf(stdout, "%s: Node %-*s (score=%s, id=%s)\n",
-                    prefix ? prefix : "  ", 71 - offset, node->details->uname, score, cons->id);
-            free(score);
-        }
-    }
-}
-
-void
-cli_resource_print_colocation(pcmk__output_t *out, pe_resource_t * rsc,
-                              bool dependents, bool recursive, int offset)
-{
-    char *prefix = NULL;
-    GListPtr lpc = NULL;
-    GListPtr list = rsc->rsc_cons;
-
-    prefix = calloc(1, (offset * 4) + 1);
-    memset(prefix, ' ', offset * 4);
-
-    if (dependents) {
-        list = rsc->rsc_cons_lhs;
-    }
-
-    if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
-        /* Break colocation loops */
-        printf("loop %s\n", rsc->id);
-        free(prefix);
-        return;
-    }
-
-    pe__set_resource_flags(rsc, pe_rsc_allocating);
-    for (lpc = list; lpc != NULL; lpc = lpc->next) {
-        rsc_colocation_t *cons = (rsc_colocation_t *) lpc->data;
-
-        char *score = NULL;
-        pe_resource_t *peer = cons->rsc_rh;
-
-        if (dependents) {
-            peer = cons->rsc_lh;
-        }
-
-        if (pcmk_is_set(peer->flags, pe_rsc_allocating)) {
-            if (dependents == FALSE) {
-                fprintf(stdout, "%s%-*s (id=%s - loop)\n", prefix, 80 - (4 * offset), peer->id,
-                        cons->id);
-            }
-            continue;
-        }
-
-        if (dependents && recursive) {
-            cli_resource_print_colocation(out, peer, dependents, recursive, offset + 1);
-        }
-
-        score = score2char(cons->score);
-        if (cons->role_rh > RSC_ROLE_STARTED) {
-            fprintf(stdout, "%s%-*s (score=%s, %s role=%s, id=%s)\n", prefix, 80 - (4 * offset),
-                    peer->id, score, dependents ? "needs" : "with", role2text(cons->role_rh),
-                    cons->id);
-        } else {
-            fprintf(stdout, "%s%-*s (score=%s, id=%s)\n", prefix, 80 - (4 * offset),
-                    peer->id, score, cons->id);
-        }
-        cli_resource_print_location(out, peer, prefix);
-        free(score);
-
-        if (!dependents && recursive) {
-            cli_resource_print_colocation(out, peer, dependents, recursive, offset + 1);
-        }
-    }
-    free(prefix);
-}
-
 // \return Standard Pacemaker return code
 int
 cli_resource_print(pcmk__output_t *out, pe_resource_t *rsc,
-- 
1.8.3.1


From c718c8798254732122771552bef59bf4cd45d24a Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Wed, 4 Nov 2020 11:17:43 -0500
Subject: [PATCH 18/19] Feature: xml: Add a schema for new crm_resource output.

---
 xml/Makefile.am              |   2 +-
 xml/api/crm_resource-2.4.rng | 255 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 256 insertions(+), 1 deletion(-)
 create mode 100644 xml/api/crm_resource-2.4.rng

diff --git a/xml/Makefile.am b/xml/Makefile.am
index 2f99f1c..a56258d 100644
--- a/xml/Makefile.am
+++ b/xml/Makefile.am
@@ -50,7 +50,7 @@ version_pairs_last = $(wordlist \
 # problems.
 
 # Names of API schemas that form the choices for pacemaker-result content
-API_request_base	= command-output crm_mon crmadmin stonith_admin version
+API_request_base	= command-output crm_mon crm_resource crmadmin stonith_admin version
 
 # Names of CIB schemas that form the choices for cib/configuration content
 CIB_cfg_base		= options nodes resources constraints fencing acls tags alerts
diff --git a/xml/api/crm_resource-2.4.rng b/xml/api/crm_resource-2.4.rng
new file mode 100644
index 0000000..1bcb969
--- /dev/null
+++ b/xml/api/crm_resource-2.4.rng
@@ -0,0 +1,255 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="element-crm-resource"/>
+    </start>
+
+    <define name="element-crm-resource">
+        <choice>
+            <ref name="agents-list" />
+            <ref name="alternatives-list" />
+            <ref name="constraints-list" />
+            <externalRef href="generic-list-2.4.rng"/>
+            <element name="metadata"> <text/> </element>
+            <ref name="locate-list" />
+            <ref name="operations-list" />
+            <ref name="providers-list" />
+            <ref name="reasons-list" />
+            <ref name="resource-check" />
+            <ref name="resource-config" />
+            <ref name="resources-list" />
+        </choice>
+    </define>
+
+    <define name="agents-list">
+        <element name="agents">
+            <attribute name="standard"> <text/> </attribute>
+            <optional>
+                <attribute name="provider"> <text/> </attribute>
+            </optional>
+            <zeroOrMore>
+                <element name="agent"> <text/> </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="alternatives-list">
+        <element name="providers">
+            <attribute name="for"> <text/> </attribute>
+            <zeroOrMore>
+                <element name="provider"> <text/> </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="colocations-list">
+        <element name="colocations">
+            <oneOrMore>
+                <ref name="element-colocation-list"/>
+            </oneOrMore>
+        </element>
+    </define>
+
+    <define name="constraints-list">
+        <element name="constraints">
+            <optional>
+                <ref name="colocations-list"/>
+            </optional>
+            <element name="resource">
+                <attribute name="id"> <text/> </attribute>
+                <ref name="locations-list"/>
+            </element>
+            <optional>
+                <ref name="colocations-list"/>
+            </optional>
+        </element>
+    </define>
+
+    <define name="locate-list">
+        <element name="nodes">
+            <attribute name="resource"> <text/> </attribute>
+            <zeroOrMore>
+                <element name="node">
+                    <optional>
+                        <attribute name="state"><value>promoted</value></attribute>
+                    </optional>
+                    <text/>
+                </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="locations-list">
+        <element name="locations">
+            <zeroOrMore>
+                <element name="location">
+                    <attribute name="host"> <text/> </attribute>
+                    <attribute name="id"> <text/> </attribute>
+                    <attribute name="score"> <text/> </attribute>
+                </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="operations-list">
+        <element name="operations">
+            <oneOrMore>
+                <ref name="element-operation-list" />
+            </oneOrMore>
+        </element>
+    </define>
+
+    <define name="providers-list">
+        <element name="providers">
+            <attribute name="standard"> <value>ocf</value> </attribute>
+            <optional>
+                <attribute name="agent"> <text/> </attribute>
+            </optional>
+            <zeroOrMore>
+                <element name="provider"> <text/> </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="reasons-list">
+        <choice>
+            <ref name="no-resource-or-uname"/>
+            <ref name="resource-and-uname"/>
+            <ref name="no-resource-but-uname"/>
+            <ref name="resource-but-no-uname"/>
+        </choice>
+    </define>
+
+    <define name="no-resource-or-uname">
+        <element name="reason">
+            <element name="resources">
+                <zeroOrMore>
+                    <element name="resource">
+                        <attribute name="id"> <text/> </attribute>
+                        <attribute name="running"> <data type="boolean"/> </attribute>
+                        <ref name="resource-check"/>
+                    </element>
+                </zeroOrMore>
+            </element>
+        </element>
+    </define>
+
+    <define name="resource-and-uname">
+        <element name="reason">
+            <attribute name="running_on"> <text/> </attribute>
+            <ref name="resource-check"/>
+        </element>
+    </define>
+
+    <define name="no-resource-but-uname">
+        <element name="reason">
+            <element name="resources">
+                <zeroOrMore>
+                    <element name="resource">
+                        <attribute name="id"> <text/> </attribute>
+                        <attribute name="running"> <data type="boolean"/> </attribute>
+                        <attribute name="host"> <text/> </attribute>
+                        <ref name="resource-check"/>
+                    </element>
+                </zeroOrMore>
+            </element>
+        </element>
+    </define>
+
+    <define name="resource-but-no-uname">
+        <element name="reason">
+            <attribute name="running"> <data type="boolean"/> </attribute>
+            <ref name="resource-check"/>
+        </element>
+    </define>
+
+    <define name="resource-config">
+        <element name="resource_config">
+            <externalRef href="resources-2.4.rng" />
+            <element name="xml"> <text/> </element>
+        </element>
+    </define>
+
+    <define name="resource-check">
+        <element name="check">
+            <attribute name="id"> <text/> </attribute>
+            <optional>
+                <choice>
+                    <attribute name="remain_stopped"><value>true</value></attribute>
+                    <attribute name="promotable"><value>false</value></attribute>
+                </choice>
+            </optional>
+            <optional>
+                <attribute name="unmanaged"><value>true</value></attribute>
+            </optional>
+            <optional>
+                <attribute name="locked-to"> <text/> </attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="resources-list">
+        <element name="resources">
+            <zeroOrMore>
+                <externalRef href="resources-2.4.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="element-colocation-list">
+        <optional>
+            <element name="colocation">
+                <attribute name="peer"> <text/> </attribute>
+                <attribute name="id"> <text/> </attribute>
+            </element>
+        </optional>
+        <optional>
+            <ref name="colocations-list" />
+        </optional>
+        <choice>
+            <element name="colocation">
+                <attribute name="peer"> <text/> </attribute>
+                <attribute name="id"> <text/> </attribute>
+                <attribute name="score"> <text/> </attribute>
+                <attribute name="dependends">
+                    <choice>
+                        <value>needs</value>
+                        <value>with</value>
+                    </choice>
+                </attribute>
+                <attribute name="role"> <text/> </attribute>
+            </element>
+            <element name="colocation">
+                <attribute name="peer"> <text/> </attribute>
+                <attribute name="id"> <text/> </attribute>
+                <attribute name="score"> <text/> </attribute>
+            </element>
+        </choice>
+        <ref name="locations-list" />
+        <optional>
+            <ref name="colocations-list" />
+        </optional>
+    </define>
+
+    <define name="element-operation-list">
+        <element name="operation">
+            <optional>
+                <group>
+                    <attribute name="rsc"> <text/> </attribute>
+                    <attribute name="agent"> <text/> </attribute>
+                </group>
+            </optional>
+            <attribute name="op"> <text/> </attribute>
+            <attribute name="node"> <text/> </attribute>
+            <attribute name="call"> <data type="nonNegativeInteger" /> </attribute>
+            <attribute name="rc"> <data type="nonNegativeInteger" /> </attribute>
+            <optional>
+                <attribute name="last-rc-change"> <text/> </attribute>
+                <attribute name="exec-time"> <data type="nonNegativeInteger" /> </attribute>
+            </optional>
+            <attribute name="status"> <text/> </attribute>
+        </element>
+    </define>
+</grammar>
-- 
1.8.3.1


From a4bb7ae6a2a9ea3016539f3cdd5002536217b39b Mon Sep 17 00:00:00 2001
From: Chris Lumens <clumens@redhat.com>
Date: Wed, 21 Oct 2020 13:50:15 -0400
Subject: [PATCH 19/19] Fix: include: Bump CRM_FEATURE_SET to 3.6.3.

This is being bumped due to the addition of the --output-as= and
--output-to= arguments to crm_resource for formatted output.  In
addition, there are various minor differences in the crm_resource text
output.  It is hoped that over time, these differences won't matter as
much because consumers can use the XML output instead.
---
 include/crm/crm.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/crm/crm.h b/include/crm/crm.h
index 4eca278..b07152c 100644
--- a/include/crm/crm.h
+++ b/include/crm/crm.h
@@ -51,7 +51,7 @@ extern "C" {
  * >=3.0.13: Fail counts include operation name and interval
  * >=3.2.0:  DC supports PCMK_LRM_OP_INVALID and PCMK_LRM_OP_NOT_CONNECTED
  */
-#  define CRM_FEATURE_SET		"3.6.2"
+#  define CRM_FEATURE_SET		"3.6.3"
 
 #  define EOS		'\0'
 #  define DIMOF(a)	((int) (sizeof(a)/sizeof(a[0])) )
-- 
1.8.3.1