From f4e3d77c94906a062641c7bf34243049de521a87 Mon Sep 17 00:00:00 2001

From: Klaus Wenninger <klaus.wenninger@aon.at>

Date: Wed, 3 Feb 2021 13:25:22 +0100

Subject: [PATCH] Fix: crm_mon: detect when run on remote-node

---

 daemons/execd/remoted_proxy.c     | 17 +++++++

 daemons/pacemakerd/pacemakerd.c   |  6 +--

 include/crm/common/ipc_internal.h |  2 +

 lib/common/ipc_server.c           | 26 ++++++++++

 tools/crm_mon.c                   | 99 ++++++++++++++++++++++++---------------

 5 files changed, 106 insertions(+), 44 deletions(-)

diff --git a/daemons/execd/remoted_proxy.c b/daemons/execd/remoted_proxy.c

index 9329fa6..0fe39bf 100644

--- a/daemons/execd/remoted_proxy.c

+++ b/daemons/execd/remoted_proxy.c

@@ -29,6 +29,7 @@ static qb_ipcs_service_t *cib_shm = NULL;

 static qb_ipcs_service_t *attrd_ipcs = NULL;

 static qb_ipcs_service_t *crmd_ipcs = NULL;

 static qb_ipcs_service_t *stonith_ipcs = NULL;

+static qb_ipcs_service_t *pacemakerd_ipcs = NULL;

 // An IPC provider is a cluster node controller connecting as a client

 static GList *ipc_providers = NULL;

@@ -126,6 +127,12 @@ stonith_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)

 }

 static int32_t

+pacemakerd_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)

+{

+    return -EREMOTEIO;

+}

+

+static int32_t

 cib_proxy_accept_rw(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)

 {

     return ipc_proxy_accept(c, uid, gid, PCMK__SERVER_BASED_RW);

@@ -356,6 +363,14 @@ static struct qb_ipcs_service_handlers stonith_proxy_callbacks = {

     .connection_destroyed = ipc_proxy_destroy

 };

+static struct qb_ipcs_service_handlers pacemakerd_proxy_callbacks = {

+    .connection_accept = pacemakerd_proxy_accept,

+    .connection_created = NULL,

+    .msg_process = NULL,

+    .connection_closed = NULL,

+    .connection_destroyed = NULL

+};

+

 static struct qb_ipcs_service_handlers cib_proxy_callbacks_ro = {

     .connection_accept = cib_proxy_accept_ro,

     .connection_created = NULL,

@@ -422,6 +437,7 @@ ipc_proxy_init(void)

                           &cib_proxy_callbacks_rw);

     pcmk__serve_attrd_ipc(&attrd_ipcs, &attrd_proxy_callbacks);

     pcmk__serve_fenced_ipc(&stonith_ipcs, &stonith_proxy_callbacks);

+    pcmk__serve_pacemakerd_ipc(&pacemakerd_ipcs, &pacemakerd_proxy_callbacks);

     crmd_ipcs = pcmk__serve_controld_ipc(&crmd_proxy_callbacks);

     if (crmd_ipcs == NULL) {

         crm_err("Failed to create controller: exiting and inhibiting respawn");

@@ -444,6 +460,7 @@ ipc_proxy_cleanup(void)

     pcmk__stop_based_ipc(cib_ro, cib_rw, cib_shm);

     qb_ipcs_destroy(attrd_ipcs);

     qb_ipcs_destroy(stonith_ipcs);

+    qb_ipcs_destroy(pacemakerd_ipcs);

     qb_ipcs_destroy(crmd_ipcs);

     cib_ro = NULL;

     cib_rw = NULL;

diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c

index 509b0f8..4572b70 100644

--- a/daemons/pacemakerd/pacemakerd.c

+++ b/daemons/pacemakerd/pacemakerd.c

@@ -1287,11 +1287,7 @@ main(int argc, char **argv)

     // Don't build CRM_RSCTMP_DIR, pacemaker-execd will do it

-    ipcs = mainloop_add_ipc_server(CRM_SYSTEM_MCP, QB_IPC_NATIVE, &mcp_ipc_callbacks);

-    if (ipcs == NULL) {

-        crm_err("Couldn't start IPC server");

-        crm_exit(CRM_EX_OSERR);

-    }

+    pcmk__serve_pacemakerd_ipc(&ipcs, &mcp_ipc_callbacks);

 #ifdef SUPPORT_COROSYNC

     /* Allows us to block shutdown */

diff --git a/include/crm/common/ipc_internal.h b/include/crm/common/ipc_internal.h

index cf935f3..fb82ce1 100644

--- a/include/crm/common/ipc_internal.h

+++ b/include/crm/common/ipc_internal.h

@@ -221,6 +221,8 @@ void pcmk__serve_attrd_ipc(qb_ipcs_service_t **ipcs,

                            struct qb_ipcs_service_handlers *cb);

 void pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs,

                             struct qb_ipcs_service_handlers *cb);

+void pcmk__serve_pacemakerd_ipc(qb_ipcs_service_t **ipcs,

+                                struct qb_ipcs_service_handlers *cb);

 qb_ipcs_service_t *pcmk__serve_controld_ipc(struct qb_ipcs_service_handlers *cb);

 void pcmk__serve_based_ipc(qb_ipcs_service_t **ipcs_ro,

diff --git a/lib/common/ipc_server.c b/lib/common/ipc_server.c

index 4d3e954..b3aaf8e 100644

--- a/lib/common/ipc_server.c

+++ b/lib/common/ipc_server.c

@@ -922,6 +922,32 @@ pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs,

 }

 /*!

+ * \internal

+ * \brief Add an IPC server to the main loop for the pacemakerd API

+ *

+ * \param[in] cb  IPC callbacks

+ *

+ * \note This function exits with CRM_EX_OSERR if unable to create the servers.

+ */

+void

+pcmk__serve_pacemakerd_ipc(qb_ipcs_service_t **ipcs,

+                       struct qb_ipcs_service_handlers *cb)

+{

+    *ipcs = mainloop_add_ipc_server(CRM_SYSTEM_MCP, QB_IPC_NATIVE, cb);

+

+    if (*ipcs == NULL) {

+        crm_err("Couldn't start pacemakerd IPC server");

+        crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");

+        /* sub-daemons are observed by pacemakerd. Thus we exit CRM_EX_FATAL

+         * if we want to prevent pacemakerd from restarting them.

+         * With pacemakerd we leave the exit-code shown to e.g. systemd

+         * to what it was prior to moving the code here from pacemakerd.c

+         */

+        crm_exit(CRM_EX_OSERR);

+    }

+}

+

+/*!

  * \brief Check whether string represents a client name used by cluster daemons

  *

  * \param[in] name  String to check

diff --git a/tools/crm_mon.c b/tools/crm_mon.c

index d4d4ac3..e58fed2 100644

--- a/tools/crm_mon.c

+++ b/tools/crm_mon.c

@@ -83,6 +83,8 @@ static gchar **processed_args = NULL;

 static time_t last_refresh = 0;

 crm_trigger_t *refresh_trigger = NULL;

+static gboolean on_remote_node = FALSE;

+

 int interactive_fence_level = 0;

 static pcmk__supported_format_t formats[] = {

@@ -988,48 +990,63 @@ pacemakerd_status(void)

     }

     pcmk_register_ipc_callback(pacemakerd_api, pacemakerd_event_cb, (void *) &state);

     rc = pcmk_connect_ipc(pacemakerd_api, pcmk_ipc_dispatch_poll);

-    if (rc == pcmk_rc_ok) {

-        rc = pcmk_pacemakerd_api_ping(pacemakerd_api, crm_system_name);

-        if (rc == pcmk_rc_ok) {

-            rc = pcmk_poll_ipc(pacemakerd_api, options.reconnect_msec/2);

+    switch (rc) {

+        case pcmk_rc_ok:

+            rc = pcmk_pacemakerd_api_ping(pacemakerd_api, crm_system_name);

             if (rc == pcmk_rc_ok) {

-                pcmk_dispatch_ipc(pacemakerd_api);

-                rc = ENOTCONN;

-                if ((output_format == mon_output_console) ||

-                    (output_format == mon_output_plain)) {

-                    switch (state) {

-                        case pcmk_pacemakerd_state_running:

-                            rc = pcmk_rc_ok;

-                            break;

-                        case pcmk_pacemakerd_state_starting_daemons:

-                            print_as(output_format ,"Pacemaker daemons starting ...\n");

-                            break;

-                        case pcmk_pacemakerd_state_wait_for_ping:

-                            print_as(output_format ,"Waiting for startup-trigger from SBD ...\n");

-                            break;

-                        case pcmk_pacemakerd_state_shutting_down:

-                            print_as(output_format ,"Pacemaker daemons shutting down ...\n");

-                            break;

-                        case pcmk_pacemakerd_state_shutdown_complete:

-                            /* assuming pacemakerd doesn't dispatch any pings after entering

-                            * that state unless it is waiting for SBD

-                            */

-                            print_as(output_format ,"Pacemaker daemons shut down - reporting to SBD ...\n");

-                            break;

-                        default:

-                            break;

-                    }

-                } else {

-                    switch (state) {

-                        case pcmk_pacemakerd_state_running:

-                            rc = pcmk_rc_ok;

-                            break;

-                        default:

-                            break;

+                rc = pcmk_poll_ipc(pacemakerd_api, options.reconnect_msec/2);

+                if (rc == pcmk_rc_ok) {

+                    pcmk_dispatch_ipc(pacemakerd_api);

+                    rc = ENOTCONN;

+                    if ((output_format == mon_output_console) ||

+                        (output_format == mon_output_plain)) {

+                        switch (state) {

+                            case pcmk_pacemakerd_state_running:

+                                rc = pcmk_rc_ok;

+                                break;

+                            case pcmk_pacemakerd_state_starting_daemons:

+                                print_as(output_format ,"Pacemaker daemons starting ...\n");

+                                break;

+                            case pcmk_pacemakerd_state_wait_for_ping:

+                                print_as(output_format ,"Waiting for startup-trigger from SBD ...\n");

+                                break;

+                            case pcmk_pacemakerd_state_shutting_down:

+                                print_as(output_format ,"Pacemaker daemons shutting down ...\n");

+                                break;

+                            case pcmk_pacemakerd_state_shutdown_complete:

+                                /* assuming pacemakerd doesn't dispatch any pings after entering

+                                * that state unless it is waiting for SBD

+                                */

+                                print_as(output_format ,"Pacemaker daemons shut down - reporting to SBD ...\n");

+                                break;

+                            default:

+                                break;

+                        }

+                    } else {

+                        switch (state) {

+                            case pcmk_pacemakerd_state_running:

+                                rc = pcmk_rc_ok;

+                                break;

+                            default:

+                                break;

+                        }

                     }

                 }

             }

-        }

+            break;

+        case EREMOTEIO:

+            rc = pcmk_rc_ok;

+            on_remote_node = TRUE;

+#if CURSES_ENABLED

+            /* just show this if refresh is gonna remove all traces */

+            if (output_format == mon_output_console) {

+                print_as(output_format ,

+                    "Running on remote-node waiting to be connected by cluster ...\n");

+            }

+#endif

+            break;

+        default:

+            break;

     }

     pcmk_free_ipc_api(pacemakerd_api);

     /* returning with ENOTCONN triggers a retry */

@@ -1348,7 +1365,11 @@ handle_connection_failures(int rc)

                     pcmk_rc_str(rc));

         rc = MON_STATUS_CRIT;

     } else if (rc == ENOTCONN) {

-        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: cluster is not available on this node");

+        if (on_remote_node) {

+            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: remote-node not connected to cluster");

+        } else {

+            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: cluster is not available on this node");

+        }

         rc = pcmk_rc2exitc(rc);

     } else {

         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Connection to cluster failed: %s", pcmk_rc_str(rc));

-- 

1.8.3.1