From 764dacb6195f8940f13b9c322b1bc8189c5619fc Mon Sep 17 00:00:00 2001

From: Lars Ellenberg <lars.ellenberg@linbit.com>

Date: Mon, 6 Sep 2021 12:13:42 +0200

Subject: [PATCH 1/6] Fix NFSv4 lock failover: set NFS Server Scope

Problem: https://github.com/ClusterLabs/resource-agents/issues/1644

RFC8881, 8.4.2.1 State Reclaim:

| If the server scope is different, the client should not attempt to

| reclaim locks. In this situation, no lock reclaim is possible.

| Any attempt to re-obtain the locks with non-reclaim operations is

| problematic since there is no guarantee that the existing

| filehandles will be recognized by the new server, or that if

| recognized, they denote the same objects. It is best to treat the

| locks as having been revoked by the reconfiguration event.

That's why for lock reclaim to even be attempted, we have to define and set

the same server scope for NFSD on all cluster nodes in the NFS failover

cluster. And in linux, that is done by setting the uts nodename for the

command that starts the nfsd kernel threads.

For "init scripts", just set it directly using unshare --uts.

For systemd units, add NFS_SERVER_SCOPE to some environment files

and inject the "unshare --uts" into the ExecStart command lines

using override drop-in files.

---

 heartbeat/nfsserver | 120 +++++++++++++++++++++++++++++++++++++++++++-

 1 file changed, 119 insertions(+), 1 deletion(-)

diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver

index 96b19abe36..0888378645 100755

--- a/heartbeat/nfsserver

+++ b/heartbeat/nfsserver

@@ -5,6 +5,18 @@

 # by hxinwei@gmail.com

 # License: GNU General Public License v2 (GPLv2) and later

+

+# I don't know for certain whether all services actuall _need_ this,

+# I know that at least nfs-server needs it.

+# The rgmanager resource agent in rgmanager/src/resources/nfsserver.sh.in

+# did the unshare for gssd and idmapd as well, even though it seems unclear why.

+# Let's start with just the nfs-server, and add others if/when we have clear

+# indication they need it.

+#NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-idmapd.service nfs-mountd.service nfs-server.service nfsdcld.service rpc-gssd.service rpc-statd.service rpcbind.service"

+NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-server.service"

+SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE=/run/sysconfig/nfs-server-scope

+SYSTEMD_UNSHARE_UTS_DROPIN=51-resource-agents-unshare-uts.conf

+

 if [ -n "$OCF_DEBUG_LIBRARY" ]; then

     . $OCF_DEBUG_LIBRARY

 else

@@ -99,6 +111,31 @@ Specifies the length of sm-notify retry time (minutes).

 <content type="integer" default="" />

 </parameter>

+<parameter name="nfs_server_scope" unique="0" required="0">

+<longdesc lang="en">

+RFC8881, 8.4.2.1 State Reclaim:

+

+If the server scope is different, the client should not attempt to

+reclaim locks. In this situation, no lock reclaim is possible.

+Any attempt to re-obtain the locks with non-reclaim operations is

+problematic since there is no guarantee that the existing

+filehandles will be recognized by the new server, or that if

+recognized, they denote the same objects. It is best to treat the

+locks as having been revoked by the reconfiguration event.

+

+For lock reclaim to even be attempted, we have to define and set the same

+server scope for NFSD on all cluster nodes in the NFS failover cluster.

+

+This agent won't "guess" a suitable server scope name for you, you need to

+explicitly specify this. But without it, NFSv4 lock reclaim after failover

+won't work properly.  Suggested value: the failover "service IP".

+</longdesc>

+<shortdesc lang="en">

+RFC8881 NFS server scope for (lock) state reclaim after failover.

+</shortdesc>

+<content type="string"/>

+</parameter>

+

 <parameter name="nfs_ip" unique="0" required="0">

 <longdesc lang="en">

 Comma separated list of floating IP addresses used to access the nfs service

@@ -269,7 +306,11 @@ nfs_exec()

 	set_exec_mode

 	case $EXEC_MODE in 

-		1) ${OCF_RESKEY_nfs_init_script} $cmd;;

+		1) if [ -z "$OCF_RESKEY_nfs_server_scope" ] ; then

+			${OCF_RESKEY_nfs_init_script} $cmd

+		   else

+			unshare -u /bin/sh -c "hostname ${OCF_RESKEY_nfs_server_scope}; exec ${OCF_RESKEY_nfs_init_script} $cmd"

+		   fi ;;

 		2) if ! echo $svc | grep -q "\."; then

 			svc="${svc}.service"

 		   fi

@@ -623,6 +664,74 @@ notify_locks()

 	fi

 }

+# Problem: https://github.com/ClusterLabs/resource-agents/issues/1644

+# RFC8881, 8.4.2.1 State Reclaim:

+#

+# | If the server scope is different, the client should not attempt to

+# | reclaim locks. In this situation, no lock reclaim is possible.

+# | Any attempt to re-obtain the locks with non-reclaim operations is

+# | problematic since there is no guarantee that the existing

+# | filehandles will be recognized by the new server, or that if

+# | recognized, they denote the same objects. It is best to treat the

+# | locks as having been revoked by the reconfiguration event.

+#

+# That's why for lock reclaim to even be attempted, we have to define and set

+# the same server scope for NFSD on all cluster nodes in the NFS failover

+# cluster. And in linux, that is done by setting the uts nodename for the

+# command that starts the nfsd kernel threads.

+#

+inject_unshare_uts_name_into_systemd_units ()

+{

+	local END_TAG="# END OF DROP-IN FOR NFS SERVER SCOPE"

+	local services

+	services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE | cut -d ' ' -f1)

+

+	local svc dir dropin edited_exec_start do_reload=false

+	for svc in $services ; do

+		dir=/run/systemd/system/$svc.d

+		dropin=$dir/$SYSTEMD_UNSHARE_UTS_DROPIN

+		grep -sqF "$END_TAG" "$dropin" && continue

+

+		test -d "$dir" || mkdir -p "$dir"

+		test -e "$dropin" && rm -f "$dropin"

+

+		edited_exec_start=$(systemctl cat $svc | sed -ne "s#^ExecStart=\\(.*\\)#ExecStart=/usr/bin/unshare --uts /bin/sh -ec 'hostname \${NFS_SERVER_SCOPE}; exec \"\$@\"' -- \\1#p")

+		cat > "$dropin" <<___

+[Service]

+EnvironmentFile=$SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE

+# reset list of exec start, then re-populate with unshared uts namespace

+ExecStart=

+$edited_exec_start

+$END_TAG

+___

+		do_reload=true

+		ocf_log debug "injected unshare --uts into $dropin"

+	done

+

+	mkdir -p "${SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE%/*}"

+	echo "NFS_SERVER_SCOPE=$OCF_RESKEY_nfs_server_scope" > "$SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE"

+

+	$do_reload && systemctl daemon-reload

+}

+

+remove_unshare_uts_dropins ()

+{

+	local services

+	services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE)

+

+	local svc dir dropin do_reload=false

+	for svc in $services ; do

+		dir=/run/systemd/system/$svc.d

+		dropin=$dir/$SYSTEMD_UNSHARE_UTS_DROPIN

+		test -e "$dropin" || continue

+		rm -f "$dropin"

+		do_reload=true

+		ocf_log debug "removed unshare --uts from $svc"

+	done

+	rm -f "${SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE}"

+	$do_reload && systemctl daemon-reload

+}

+

 nfsserver_start ()

 {

 	local rc;

@@ -636,6 +745,13 @@ nfsserver_start ()

 	is_redhat_based && set_env_args

 	bind_tree

 	prepare_directory

+	case $EXEC_MODE in [23])

+		if [ -z "$OCF_RESKEY_nfs_server_scope" ] ; then

+			remove_unshare_uts_dropins

+		else

+			inject_unshare_uts_name_into_systemd_units

+		fi ;;

+	esac

 	if ! `mount | grep -q " on $OCF_RESKEY_rpcpipefs_dir "`; then

 		mount -t rpc_pipefs sunrpc $OCF_RESKEY_rpcpipefs_dir

@@ -854,6 +970,8 @@ nfsserver_stop ()

 		ocf_log info "NFS server stopped"

 	fi

+	case $EXEC_MODE in [23]) remove_unshare_uts_dropins;; esac

+

 	return $rc

 }

From 515697b53c1614d05d39491c9af83e8d8b844b17 Mon Sep 17 00:00:00 2001

From: Lars Ellenberg <lars.ellenberg@linbit.com>

Date: Fri, 8 Oct 2021 12:01:41 +0200

Subject: [PATCH 2/6] Fix NFSv4 lock failover: set NFS Server Scope, regardless

 of EXEC_MODE

Debian (and other systems) may provide "init scripts",

which will only redirect back to systemd.

If we just unshare --uts the init script invocation,

the uts namespace is useless in that case.

If systemd is running, mangle the nfs-server.service unit,

independent of the "EXEC_MODE".

---

 heartbeat/nfsserver | 18 ++++++++++++++----

 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver

index 0888378645..054aabbaf6 100755

--- a/heartbeat/nfsserver

+++ b/heartbeat/nfsserver

@@ -745,13 +745,20 @@ nfsserver_start ()

 	is_redhat_based && set_env_args

 	bind_tree

 	prepare_directory

-	case $EXEC_MODE in [23])

+

+	# Debian (and other systems) may provide "init scripts",

+	# which will only redirect back to systemd.

+	# If we just unshare --uts the init script invocation,

+	# the uts namespace is useless in that case.

+	# If systemd is running, mangle the nfs-server.service unit,

+	# independent of the "EXEC_MODE" we detected.

+	if $systemd_is_running ; then

 		if [ -z "$OCF_RESKEY_nfs_server_scope" ] ; then

 			remove_unshare_uts_dropins

 		else

 			inject_unshare_uts_name_into_systemd_units

-		fi ;;

-	esac

+		fi

+	fi

 	if ! `mount | grep -q " on $OCF_RESKEY_rpcpipefs_dir "`; then

 		mount -t rpc_pipefs sunrpc $OCF_RESKEY_rpcpipefs_dir

@@ -970,7 +977,9 @@ nfsserver_stop ()

 		ocf_log info "NFS server stopped"

 	fi

-	case $EXEC_MODE in [23]) remove_unshare_uts_dropins;; esac

+	if $systemd_is_running; then

+		remove_unshare_uts_dropins

+	fi

 	return $rc

 }

@@ -1008,6 +1017,7 @@ nfsserver_validate ()

 }

 nfsserver_validate

+systemd_is_running && systemd_is_running=true || systemd_is_running=false

 case $__OCF_ACTION in

 	start)      nfsserver_start

From e83c20d88f404f9f9d829c654883d60eb6cc9ff3 Mon Sep 17 00:00:00 2001

From: Lars Ellenberg <lars.ellenberg@linbit.com>

Date: Fri, 8 Oct 2021 17:06:18 +0200

Subject: [PATCH 3/6] Fix NFSv4 lock failover: add missing "|cut -f1" in

 remove_unshare_uts_dropins

---

 heartbeat/nfsserver | 2 +-

 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver

index 054aabbaf6..d3db89a537 100755

--- a/heartbeat/nfsserver

+++ b/heartbeat/nfsserver

@@ -717,7 +717,7 @@ ___

 remove_unshare_uts_dropins ()

 {

 	local services

-	services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE)

+	services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE | cut -d ' ' -f1)

 	local svc dir dropin do_reload=false

 	for svc in $services ; do

From b5b0e4a0b60d285af576b2d8ecfbe95e5a177a87 Mon Sep 17 00:00:00 2001

From: Lars Ellenberg <lars.ellenberg@linbit.com>

Date: Fri, 8 Oct 2021 17:07:13 +0200

Subject: [PATCH 4/6] Fix NFSv4 lock failover: get rid of "world-inaccessible"

 warning

by temporarily changing the umask before generating the dropins

---

 heartbeat/nfsserver | 3 +++

 1 file changed, 3 insertions(+)

diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver

index d3db89a537..447e0302b2 100755

--- a/heartbeat/nfsserver

+++ b/heartbeat/nfsserver

@@ -687,6 +687,8 @@ inject_unshare_uts_name_into_systemd_units ()

 	services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE | cut -d ' ' -f1)

 	local svc dir dropin edited_exec_start do_reload=false

+	local old_umask=$(umask)

+	umask 0022

 	for svc in $services ; do

 		dir=/run/systemd/system/$svc.d

 		dropin=$dir/$SYSTEMD_UNSHARE_UTS_DROPIN

@@ -710,6 +712,7 @@ ___

 	mkdir -p "${SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE%/*}"

 	echo "NFS_SERVER_SCOPE=$OCF_RESKEY_nfs_server_scope" > "$SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE"

+	umask $old_umask

 	$do_reload && systemctl daemon-reload

 }

From 3c6c91ce5a00eeef9cd766389d73a0b42580a1e6 Mon Sep 17 00:00:00 2001

From: Lars Ellenberg <lars.ellenberg@linbit.com>

Date: Fri, 8 Oct 2021 17:08:09 +0200

Subject: [PATCH 5/6] Fix NFSv4 lock failover: deal with "special executable

 prefix" chars in ExecStart

---

 heartbeat/nfsserver | 2 +-

 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver

index 447e0302b2..5326bd2c6e 100755

--- a/heartbeat/nfsserver

+++ b/heartbeat/nfsserver

@@ -697,7 +697,7 @@ inject_unshare_uts_name_into_systemd_units ()

 		test -d "$dir" || mkdir -p "$dir"

 		test -e "$dropin" && rm -f "$dropin"

-		edited_exec_start=$(systemctl cat $svc | sed -ne "s#^ExecStart=\\(.*\\)#ExecStart=/usr/bin/unshare --uts /bin/sh -ec 'hostname \${NFS_SERVER_SCOPE}; exec \"\$@\"' -- \\1#p")

+		edited_exec_start=$(systemctl cat $svc | sed -ne "s#^ExecStart=\\([-+:!@]*\\)\\(.*\\)#ExecStart=\\1/usr/bin/unshare --uts /bin/sh -c 'hostname \${NFS_SERVER_SCOPE}; exec \"\$@\"' -- \\2#p")

 		cat > "$dropin" <<___

 [Service]

 EnvironmentFile=$SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE

From 512fbaf61e6d24a1236ef50e323ea17a62485c36 Mon Sep 17 00:00:00 2001

From: Lars Ellenberg <lars.ellenberg@linbit.com>

Date: Fri, 8 Oct 2021 17:08:59 +0200

Subject: [PATCH 6/6] Fix NFSv4 lock failover: add rpc-statd-notify to the

 comment list of potentially interesting services

---

 heartbeat/nfsserver | 2 +-

 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver

index 5326bd2c6e..240dd1a76c 100755

--- a/heartbeat/nfsserver

+++ b/heartbeat/nfsserver

@@ -12,7 +12,7 @@

 # did the unshare for gssd and idmapd as well, even though it seems unclear why.

 # Let's start with just the nfs-server, and add others if/when we have clear

 # indication they need it.

-#NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-idmapd.service nfs-mountd.service nfs-server.service nfsdcld.service rpc-gssd.service rpc-statd.service rpcbind.service"

+#NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-idmapd.service nfs-mountd.service nfs-server.service nfsdcld.service rpc-gssd.service rpc-statd.service rpc-statd-notify.service rpcbind.service"

 NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-server.service"

 SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE=/run/sysconfig/nfs-server-scope

 SYSTEMD_UNSHARE_UTS_DROPIN=51-resource-agents-unshare-uts.conf