From 6016283dfdcb45bf750f96715fc653a4c0904bca Mon Sep 17 00:00:00 2001
From: Damien Ciabrini <dciabrin@redhat.com>
Date: Fri, 28 Jun 2019 13:34:40 +0200
Subject: [PATCH] podman: only use exec to manage container's lifecycle

Under heavy IO load, podman may be impacted and take a long time
to execute some actions. If that takes more than the default
20s container monitoring timeout, containers will restart unexpectedly.

Replace all IO-sensitive podman calls (inspect, exists...) by
equivalent "podman exec" calls, because the latter command seems
less prone to performance degradation under IO load.

With this commit, the resource agent now requires podman 1.0.2+,
because it relies on of two different patches [1,2] that improve
IO performance and enable to distinguish "container stopped"
"container doesn't exist" error codes.

Tested on an OpenStack environment with podman 1.0.2, with the
following scenario:
  . regular start/stop/monitor operations
  . probe operations (pcs resource cleanup/refresh)
  . unmanage/manage operations
  . reboot

[1] https://github.com/containers/libpod/commit/90b835db69d589de559462d988cb3fae5cf1ef49
[2] https://github.com/containers/libpod/commit/a19975f96d2ee7efe186d9aa0be42285cfafa3f4
---
 heartbeat/podman | 75 ++++++++++++++++++++++++------------------------
 1 file changed, 37 insertions(+), 38 deletions(-)

diff --git a/heartbeat/podman b/heartbeat/podman
index 51f6ba883..8fc2c4695 100755
--- a/heartbeat/podman
+++ b/heartbeat/podman
@@ -129,9 +129,6 @@ the health of the container. This command must return 0 to indicate that
 the container is healthy. A non-zero return code will indicate that the
 container has failed and should be recovered.
 
-If 'podman exec' is supported, it is used to execute the command. If not,
-nsenter is used.
-
 Note: Using this method for monitoring processes inside a container
 is not recommended, as containerd tries to track processes running
 inside the container and does not deal well with many short-lived
@@ -192,17 +189,13 @@ monitor_cmd_exec()
 	local rc=$OCF_SUCCESS
 	local out
 
-	if [ -z "$OCF_RESKEY_monitor_cmd" ]; then
-		return $rc
-	fi
-
 	out=$(podman exec ${CONTAINER} $OCF_RESKEY_monitor_cmd 2>&1)
 	rc=$?
-	if [ $rc -eq 127 ]; then
-		ocf_log err "monitor cmd failed (rc=$rc), output: $out"
-		ocf_exit_reason "monitor_cmd, ${OCF_RESKEY_monitor_cmd} , not found within container."
-		# there is no recovering from this, exit immediately
-		exit $OCF_ERR_ARGS
+	# 125: no container with name or ID ${CONTAINER} found
+	# 126: container state improper (not running)
+	# 127: any other error
+	if [ $rc -eq 125 ] || [ $rc -eq 126 ]; then
+		rc=$OCF_NOT_RUNNING
 	elif [ $rc -ne 0 ]; then
 		ocf_exit_reason "monitor cmd failed (rc=$rc), output: $out"
 		rc=$OCF_ERR_GENERIC
@@ -215,7 +208,16 @@ monitor_cmd_exec()
 
 container_exists()
 {
-	podman inspect --format {{.State.Running}} $CONTAINER | egrep '(true|false)' >/dev/null 2>&1
+	local rc
+	local out
+
+	out=$(podman exec ${CONTAINER} $OCF_RESKEY_monitor_cmd 2>&1)
+	rc=$?
+	# 125: no container with name or ID ${CONTAINER} found
+	if [ $rc -ne 125 ]; then
+		return 0
+	fi
+	return 1
 }
 
 remove_container()
@@ -236,30 +238,30 @@ remove_container()
 
 podman_simple_status()
 {
-	local val
-
-	# retrieve the 'Running' attribute for the container
-	val=$(podman inspect --format {{.State.Running}} $CONTAINER 2>/dev/null)
-	if [ $? -ne 0 ]; then
-		#not running as a result of container not being found
-		return $OCF_NOT_RUNNING
-	fi
+	local rc
 
-	if ocf_is_true "$val"; then
-		# container exists and is running
-		return $OCF_SUCCESS
+	# simple status is implemented via podman exec
+	# everything besides success is considered "not running"
+	monitor_cmd_exec
+	rc=$?
+	if [ $rc -ne $OCF_SUCCESS ]; then
+		rc=$OCF_NOT_RUNNING;
 	fi
-
-	return $OCF_NOT_RUNNING
+	return $rc
 }
 
 podman_monitor()
 {
-	if [ -z "$OCF_RESKEY_monitor_cmd" ]; then
-		podman_simple_status
-		return $?
-	fi
+	# We rely on running podman exec to monitor the container
+	# state because that command seems to be less prone to
+	# performance issue under IO load.
+	#
+	# For probes to work, we expect cmd_exec to be able to report
+	# when a container is not running. Here, we're not interested
+	# in distinguishing whether it's stopped or non existing
+	# (there's function container_exists for that)
 	monitor_cmd_exec
+	return $?
 }
 
 podman_create_mounts() {
@@ -416,14 +418,6 @@ podman_validate()
 		exit $OCF_ERR_CONFIGURED
 	fi
 
-	if [ -n "$OCF_RESKEY_monitor_cmd" ]; then
-		podman exec --help >/dev/null 2>&1
-		if [ ! $? ]; then
-			ocf_log info "checking for nsenter, which is required when 'monitor_cmd' is specified"
-			check_binary nsenter
-		fi
-	fi
-
 	image_exists
 	if [ $? -ne 0 ]; then
 		ocf_exit_reason "base image, ${OCF_RESKEY_image}, could not be found."
@@ -457,6 +451,11 @@ fi
 
 CONTAINER=$OCF_RESKEY_name
 
+# Note: we currently monitor podman containers by with the "podman exec"
+# command, so make sure that invocation is always valid by enforcing the
+# exec command to be non-empty
+: ${OCF_RESKEY_monitor_cmd:=/bin/true}
+
 case $__OCF_ACTION in
 meta-data) meta_data
 		exit $OCF_SUCCESS;;