From 764dacb6195f8940f13b9c322b1bc8189c5619fc Mon Sep 17 00:00:00 2001
From: Lars Ellenberg <lars.ellenberg@linbit.com>
Date: Mon, 6 Sep 2021 12:13:42 +0200
Subject: [PATCH 1/6] Fix NFSv4 lock failover: set NFS Server Scope
Problem: https://github.com/ClusterLabs/resource-agents/issues/1644
RFC8881, 8.4.2.1 State Reclaim:
| If the server scope is different, the client should not attempt to
| reclaim locks. In this situation, no lock reclaim is possible.
| Any attempt to re-obtain the locks with non-reclaim operations is
| problematic since there is no guarantee that the existing
| filehandles will be recognized by the new server, or that if
| recognized, they denote the same objects. It is best to treat the
| locks as having been revoked by the reconfiguration event.
That's why for lock reclaim to even be attempted, we have to define and set
the same server scope for NFSD on all cluster nodes in the NFS failover
cluster. And in linux, that is done by setting the uts nodename for the
command that starts the nfsd kernel threads.
For "init scripts", just set it directly using unshare --uts.
For systemd units, add NFS_SERVER_SCOPE to some environment files
and inject the "unshare --uts" into the ExecStart command lines
using override drop-in files.
---
heartbeat/nfsserver | 120 +++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 119 insertions(+), 1 deletion(-)
diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver
index 96b19abe36..0888378645 100755
--- a/heartbeat/nfsserver
+++ b/heartbeat/nfsserver
@@ -5,6 +5,18 @@
# by hxinwei@gmail.com
# License: GNU General Public License v2 (GPLv2) and later
+
+# I don't know for certain whether all services actuall _need_ this,
+# I know that at least nfs-server needs it.
+# The rgmanager resource agent in rgmanager/src/resources/nfsserver.sh.in
+# did the unshare for gssd and idmapd as well, even though it seems unclear why.
+# Let's start with just the nfs-server, and add others if/when we have clear
+# indication they need it.
+#NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-idmapd.service nfs-mountd.service nfs-server.service nfsdcld.service rpc-gssd.service rpc-statd.service rpcbind.service"
+NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-server.service"
+SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE=/run/sysconfig/nfs-server-scope
+SYSTEMD_UNSHARE_UTS_DROPIN=51-resource-agents-unshare-uts.conf
+
if [ -n "$OCF_DEBUG_LIBRARY" ]; then
. $OCF_DEBUG_LIBRARY
else
@@ -99,6 +111,31 @@ Specifies the length of sm-notify retry time (minutes).
<content type="integer" default="" />
</parameter>
+<parameter name="nfs_server_scope" unique="0" required="0">
+<longdesc lang="en">
+RFC8881, 8.4.2.1 State Reclaim:
+
+If the server scope is different, the client should not attempt to
+reclaim locks. In this situation, no lock reclaim is possible.
+Any attempt to re-obtain the locks with non-reclaim operations is
+problematic since there is no guarantee that the existing
+filehandles will be recognized by the new server, or that if
+recognized, they denote the same objects. It is best to treat the
+locks as having been revoked by the reconfiguration event.
+
+For lock reclaim to even be attempted, we have to define and set the same
+server scope for NFSD on all cluster nodes in the NFS failover cluster.
+
+This agent won't "guess" a suitable server scope name for you, you need to
+explicitly specify this. But without it, NFSv4 lock reclaim after failover
+won't work properly. Suggested value: the failover "service IP".
+</longdesc>
+<shortdesc lang="en">
+RFC8881 NFS server scope for (lock) state reclaim after failover.
+</shortdesc>
+<content type="string"/>
+</parameter>
+
<parameter name="nfs_ip" unique="0" required="0">
<longdesc lang="en">
Comma separated list of floating IP addresses used to access the nfs service
@@ -269,7 +306,11 @@ nfs_exec()
set_exec_mode
case $EXEC_MODE in
- 1) ${OCF_RESKEY_nfs_init_script} $cmd;;
+ 1) if [ -z "$OCF_RESKEY_nfs_server_scope" ] ; then
+ ${OCF_RESKEY_nfs_init_script} $cmd
+ else
+ unshare -u /bin/sh -c "hostname ${OCF_RESKEY_nfs_server_scope}; exec ${OCF_RESKEY_nfs_init_script} $cmd"
+ fi ;;
2) if ! echo $svc | grep -q "\."; then
svc="${svc}.service"
fi
@@ -623,6 +664,74 @@ notify_locks()
fi
}
+# Problem: https://github.com/ClusterLabs/resource-agents/issues/1644
+# RFC8881, 8.4.2.1 State Reclaim:
+#
+# | If the server scope is different, the client should not attempt to
+# | reclaim locks. In this situation, no lock reclaim is possible.
+# | Any attempt to re-obtain the locks with non-reclaim operations is
+# | problematic since there is no guarantee that the existing
+# | filehandles will be recognized by the new server, or that if
+# | recognized, they denote the same objects. It is best to treat the
+# | locks as having been revoked by the reconfiguration event.
+#
+# That's why for lock reclaim to even be attempted, we have to define and set
+# the same server scope for NFSD on all cluster nodes in the NFS failover
+# cluster. And in linux, that is done by setting the uts nodename for the
+# command that starts the nfsd kernel threads.
+#
+inject_unshare_uts_name_into_systemd_units ()
+{
+ local END_TAG="# END OF DROP-IN FOR NFS SERVER SCOPE"
+ local services
+ services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE | cut -d ' ' -f1)
+
+ local svc dir dropin edited_exec_start do_reload=false
+ for svc in $services ; do
+ dir=/run/systemd/system/$svc.d
+ dropin=$dir/$SYSTEMD_UNSHARE_UTS_DROPIN
+ grep -sqF "$END_TAG" "$dropin" && continue
+
+ test -d "$dir" || mkdir -p "$dir"
+ test -e "$dropin" && rm -f "$dropin"
+
+ edited_exec_start=$(systemctl cat $svc | sed -ne "s#^ExecStart=\\(.*\\)#ExecStart=/usr/bin/unshare --uts /bin/sh -ec 'hostname \${NFS_SERVER_SCOPE}; exec \"\$@\"' -- \\1#p")
+ cat > "$dropin" <<___
+[Service]
+EnvironmentFile=$SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE
+# reset list of exec start, then re-populate with unshared uts namespace
+ExecStart=
+$edited_exec_start
+$END_TAG
+___
+ do_reload=true
+ ocf_log debug "injected unshare --uts into $dropin"
+ done
+
+ mkdir -p "${SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE%/*}"
+ echo "NFS_SERVER_SCOPE=$OCF_RESKEY_nfs_server_scope" > "$SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE"
+
+ $do_reload && systemctl daemon-reload
+}
+
+remove_unshare_uts_dropins ()
+{
+ local services
+ services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE)
+
+ local svc dir dropin do_reload=false
+ for svc in $services ; do
+ dir=/run/systemd/system/$svc.d
+ dropin=$dir/$SYSTEMD_UNSHARE_UTS_DROPIN
+ test -e "$dropin" || continue
+ rm -f "$dropin"
+ do_reload=true
+ ocf_log debug "removed unshare --uts from $svc"
+ done
+ rm -f "${SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE}"
+ $do_reload && systemctl daemon-reload
+}
+
nfsserver_start ()
{
local rc;
@@ -636,6 +745,13 @@ nfsserver_start ()
is_redhat_based && set_env_args
bind_tree
prepare_directory
+ case $EXEC_MODE in [23])
+ if [ -z "$OCF_RESKEY_nfs_server_scope" ] ; then
+ remove_unshare_uts_dropins
+ else
+ inject_unshare_uts_name_into_systemd_units
+ fi ;;
+ esac
if ! `mount | grep -q " on $OCF_RESKEY_rpcpipefs_dir "`; then
mount -t rpc_pipefs sunrpc $OCF_RESKEY_rpcpipefs_dir
@@ -854,6 +970,8 @@ nfsserver_stop ()
ocf_log info "NFS server stopped"
fi
+ case $EXEC_MODE in [23]) remove_unshare_uts_dropins;; esac
+
return $rc
}
From 515697b53c1614d05d39491c9af83e8d8b844b17 Mon Sep 17 00:00:00 2001
From: Lars Ellenberg <lars.ellenberg@linbit.com>
Date: Fri, 8 Oct 2021 12:01:41 +0200
Subject: [PATCH 2/6] Fix NFSv4 lock failover: set NFS Server Scope, regardless
of EXEC_MODE
Debian (and other systems) may provide "init scripts",
which will only redirect back to systemd.
If we just unshare --uts the init script invocation,
the uts namespace is useless in that case.
If systemd is running, mangle the nfs-server.service unit,
independent of the "EXEC_MODE".
---
heartbeat/nfsserver | 18 ++++++++++++++----
1 file changed, 14 insertions(+), 4 deletions(-)
diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver
index 0888378645..054aabbaf6 100755
--- a/heartbeat/nfsserver
+++ b/heartbeat/nfsserver
@@ -745,13 +745,20 @@ nfsserver_start ()
is_redhat_based && set_env_args
bind_tree
prepare_directory
- case $EXEC_MODE in [23])
+
+ # Debian (and other systems) may provide "init scripts",
+ # which will only redirect back to systemd.
+ # If we just unshare --uts the init script invocation,
+ # the uts namespace is useless in that case.
+ # If systemd is running, mangle the nfs-server.service unit,
+ # independent of the "EXEC_MODE" we detected.
+ if $systemd_is_running ; then
if [ -z "$OCF_RESKEY_nfs_server_scope" ] ; then
remove_unshare_uts_dropins
else
inject_unshare_uts_name_into_systemd_units
- fi ;;
- esac
+ fi
+ fi
if ! `mount | grep -q " on $OCF_RESKEY_rpcpipefs_dir "`; then
mount -t rpc_pipefs sunrpc $OCF_RESKEY_rpcpipefs_dir
@@ -970,7 +977,9 @@ nfsserver_stop ()
ocf_log info "NFS server stopped"
fi
- case $EXEC_MODE in [23]) remove_unshare_uts_dropins;; esac
+ if $systemd_is_running; then
+ remove_unshare_uts_dropins
+ fi
return $rc
}
@@ -1008,6 +1017,7 @@ nfsserver_validate ()
}
nfsserver_validate
+systemd_is_running && systemd_is_running=true || systemd_is_running=false
case $__OCF_ACTION in
start) nfsserver_start
From e83c20d88f404f9f9d829c654883d60eb6cc9ff3 Mon Sep 17 00:00:00 2001
From: Lars Ellenberg <lars.ellenberg@linbit.com>
Date: Fri, 8 Oct 2021 17:06:18 +0200
Subject: [PATCH 3/6] Fix NFSv4 lock failover: add missing "|cut -f1" in
remove_unshare_uts_dropins
---
heartbeat/nfsserver | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver
index 054aabbaf6..d3db89a537 100755
--- a/heartbeat/nfsserver
+++ b/heartbeat/nfsserver
@@ -717,7 +717,7 @@ ___
remove_unshare_uts_dropins ()
{
local services
- services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE)
+ services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE | cut -d ' ' -f1)
local svc dir dropin do_reload=false
for svc in $services ; do
From b5b0e4a0b60d285af576b2d8ecfbe95e5a177a87 Mon Sep 17 00:00:00 2001
From: Lars Ellenberg <lars.ellenberg@linbit.com>
Date: Fri, 8 Oct 2021 17:07:13 +0200
Subject: [PATCH 4/6] Fix NFSv4 lock failover: get rid of "world-inaccessible"
warning
by temporarily changing the umask before generating the dropins
---
heartbeat/nfsserver | 3 +++
1 file changed, 3 insertions(+)
diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver
index d3db89a537..447e0302b2 100755
--- a/heartbeat/nfsserver
+++ b/heartbeat/nfsserver
@@ -687,6 +687,8 @@ inject_unshare_uts_name_into_systemd_units ()
services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE | cut -d ' ' -f1)
local svc dir dropin edited_exec_start do_reload=false
+ local old_umask=$(umask)
+ umask 0022
for svc in $services ; do
dir=/run/systemd/system/$svc.d
dropin=$dir/$SYSTEMD_UNSHARE_UTS_DROPIN
@@ -710,6 +712,7 @@ ___
mkdir -p "${SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE%/*}"
echo "NFS_SERVER_SCOPE=$OCF_RESKEY_nfs_server_scope" > "$SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE"
+ umask $old_umask
$do_reload && systemctl daemon-reload
}
From 3c6c91ce5a00eeef9cd766389d73a0b42580a1e6 Mon Sep 17 00:00:00 2001
From: Lars Ellenberg <lars.ellenberg@linbit.com>
Date: Fri, 8 Oct 2021 17:08:09 +0200
Subject: [PATCH 5/6] Fix NFSv4 lock failover: deal with "special executable
prefix" chars in ExecStart
---
heartbeat/nfsserver | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver
index 447e0302b2..5326bd2c6e 100755
--- a/heartbeat/nfsserver
+++ b/heartbeat/nfsserver
@@ -697,7 +697,7 @@ inject_unshare_uts_name_into_systemd_units ()
test -d "$dir" || mkdir -p "$dir"
test -e "$dropin" && rm -f "$dropin"
- edited_exec_start=$(systemctl cat $svc | sed -ne "s#^ExecStart=\\(.*\\)#ExecStart=/usr/bin/unshare --uts /bin/sh -ec 'hostname \${NFS_SERVER_SCOPE}; exec \"\$@\"' -- \\1#p")
+ edited_exec_start=$(systemctl cat $svc | sed -ne "s#^ExecStart=\\([-+:!@]*\\)\\(.*\\)#ExecStart=\\1/usr/bin/unshare --uts /bin/sh -c 'hostname \${NFS_SERVER_SCOPE}; exec \"\$@\"' -- \\2#p")
cat > "$dropin" <<___
[Service]
EnvironmentFile=$SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE
From 512fbaf61e6d24a1236ef50e323ea17a62485c36 Mon Sep 17 00:00:00 2001
From: Lars Ellenberg <lars.ellenberg@linbit.com>
Date: Fri, 8 Oct 2021 17:08:59 +0200
Subject: [PATCH 6/6] Fix NFSv4 lock failover: add rpc-statd-notify to the
comment list of potentially interesting services
---
heartbeat/nfsserver | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver
index 5326bd2c6e..240dd1a76c 100755
--- a/heartbeat/nfsserver
+++ b/heartbeat/nfsserver
@@ -12,7 +12,7 @@
# did the unshare for gssd and idmapd as well, even though it seems unclear why.
# Let's start with just the nfs-server, and add others if/when we have clear
# indication they need it.
-#NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-idmapd.service nfs-mountd.service nfs-server.service nfsdcld.service rpc-gssd.service rpc-statd.service rpcbind.service"
+#NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-idmapd.service nfs-mountd.service nfs-server.service nfsdcld.service rpc-gssd.service rpc-statd.service rpc-statd-notify.service rpcbind.service"
NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-server.service"
SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE=/run/sysconfig/nfs-server-scope
SYSTEMD_UNSHARE_UTS_DROPIN=51-resource-agents-unshare-uts.conf