diff --git a/Makefile b/Makefile
index f3834b8..b25ca77 100644
--- a/Makefile
+++ b/Makefile
@@ -16,7 +16,7 @@ VERSION := $(if $(VERSION),$(VERSION),$(LAST_VERSION))
 DEFAULT_SIZE := 0
 
 
-all: spec src srpm
+all: spec src srpm patch
 
 pre: ## Get the last remote hash for the specified branch
 	$(eval REMOTEHASH := $(shell git ls-remote $(BASE) $(BRANCH) | awk '{print $$1}'))
@@ -34,12 +34,7 @@ echo: pre  ## Print the passed (or computed) parameters
 	@echo RELEASE $(RELEASE)
 
 patch: pre ## Build the latest patch
-	@git diff $(PWD)/SOURCES/cephadm > $(PWD)/SOURCES/cephadm-$(VERSION)-$(RELEASE).patch \
-	$(eval SIZE := $(shell find SOURCES -name cephadm-$(VERSION)-$(RELEASE).patch -printf "%k"))
-	@SIZE=$(if $(SIZE),$(SIZE),$(DEFAULT_SIZE))
-	@if [[ "$(SIZE)" -eq 0 ]]; then \
-	   rm -f $(PWD)/SOURCES/cephadm-$(VERSION)-$(RELEASE).patch; \
-	fi \
+	@git diff $(PWD)/SOURCES/cephadm > $(PWD)/SOURCES/cephadm-$(VERSION)-$(RELEASE).patch
 
 spec: pre  ## Patch and build the cephadm spec file
 	sed SPECS/cephadm.spec.in \
diff --git a/SOURCES/cephadm b/SOURCES/cephadm
index 524c038..a7b664c 100644
--- a/SOURCES/cephadm
+++ b/SOURCES/cephadm
@@ -61,6 +61,7 @@ DATA_DIR = '/var/lib/ceph'
 LOG_DIR = '/var/log/ceph'
 LOCK_DIR = '/run/cephadm'
 LOGROTATE_DIR = '/etc/logrotate.d'
+SYSCTL_DIR = '/usr/lib/sysctl.d'
 UNIT_DIR = '/etc/systemd/system'
 LOG_DIR_MODE = 0o770
 DATA_DIR_MODE = 0o700
@@ -109,6 +110,7 @@ class BaseConfig:
         self.data_dir: str = DATA_DIR
         self.log_dir: str = LOG_DIR
         self.logrotate_dir: str = LOGROTATE_DIR
+        self.sysctl_dir: str = SYSCTL_DIR
         self.unit_dir: str = UNIT_DIR
         self.verbose: bool = False
         self.timeout: Optional[int] = DEFAULT_TIMEOUT
@@ -210,11 +212,9 @@ logging_config = {
         },
         'log_file': {
             'level': 'DEBUG',
-            'class': 'logging.handlers.RotatingFileHandler',
+            'class': 'logging.handlers.WatchedFileHandler',
             'formatter': 'cephadm',
             'filename': '%s/cephadm.log' % LOG_DIR,
-            'maxBytes': 1024000,
-            'backupCount': 1,
         }
     },
     'loggers': {
@@ -249,6 +249,18 @@ class Ceph(object):
 ##################################
 
 
+class OSD(object):
+    @staticmethod
+    def get_sysctl_settings() -> List[str]:
+        return [
+            '# allow a large number of OSDs',
+            'fs.aio-max-nr = 1048576',
+            'kernel.pid_max = 4194304',
+        ]
+
+##################################
+
+
 class Monitoring(object):
     """Define the configs for the monitoring containers"""
 
@@ -349,7 +361,7 @@ def populate_files(config_dir, config_files, uid, gid):
         config_file = os.path.join(config_dir, fname)
         config_content = dict_get_join(config_files, fname)
         logger.info('Write file: %s' % (config_file))
-        with open(config_file, 'w') as f:
+        with open(config_file, 'w', encoding='utf-8') as f:
             os.fchown(f.fileno(), uid, gid)
             os.fchmod(f.fileno(), 0o600)
             f.write(config_content)
@@ -491,36 +503,6 @@ class NFSGanesha(object):
                 os.fchown(f.fileno(), uid, gid)
                 f.write(self.rgw.get('keyring', ''))
 
-    def get_rados_grace_container(self, action):
-        # type: (str) -> CephContainer
-        """Container for a ganesha action on the grace db"""
-        entrypoint = '/usr/bin/ganesha-rados-grace'
-
-        assert self.pool
-        args = ['--pool', self.pool]
-        if self.namespace:
-            args += ['--ns', self.namespace]
-        if self.userid:
-            args += ['--userid', self.userid]
-        args += [action, self.get_daemon_name()]
-
-        data_dir = get_data_dir(self.fsid, self.ctx.data_dir,
-                                self.daemon_type, self.daemon_id)
-        volume_mounts = self.get_container_mounts(data_dir)
-        envs = self.get_container_envs()
-
-        logger.info('Creating RADOS grace for action: %s' % action)
-        c = CephContainer(
-            self.ctx,
-            image=self.image,
-            entrypoint=entrypoint,
-            args=args,
-            volume_mounts=volume_mounts,
-            cname=self.get_container_name(desc='grace-%s' % action),
-            envs=envs
-        )
-        return c
-
 ##################################
 
 
@@ -732,6 +714,13 @@ class HAproxy(object):
         mounts[os.path.join(data_dir, 'haproxy')] = '/var/lib/haproxy'
         return mounts
 
+    @staticmethod
+    def get_sysctl_settings() -> List[str]:
+        return [
+            '# IP forwarding',
+            'net.ipv4.ip_forward = 1',
+        ]
+
 ##################################
 
 
@@ -811,12 +800,12 @@ class Keepalived(object):
         return envs
 
     @staticmethod
-    def get_prestart():
-        return (
-            '# keepalived needs IP forwarding and non-local bind\n'
-            'sysctl net.ipv4.ip_forward=1\n'
-            'sysctl net.ipv4.ip_nonlocal_bind=1\n'
-        )
+    def get_sysctl_settings() -> List[str]:
+        return [
+            '# IP forwarding and non-local bind',
+            'net.ipv4.ip_forward = 1',
+            'net.ipv4.ip_nonlocal_bind = 1',
+        ]
 
     def extract_uid_gid_keepalived(self):
         # better directory for this?
@@ -1190,7 +1179,7 @@ class FileLock(object):
                         lock_id, lock_filename, poll_intervall
                     )
                     time.sleep(poll_intervall)
-        except:  # noqa
+        except Exception:
             # Something did go wrong, so decrement the counter.
             self._lock_counter = max(0, self._lock_counter - 1)
 
@@ -2061,10 +2050,11 @@ def check_units(ctx, units, enabler=None):
 
 
 def is_container_running(ctx: CephadmContext, name: str) -> bool:
-    out, err, ret = call_throws(ctx, [
-        ctx.container_engine.path, 'ps',
-        '--format', '{{.Names}}'])
-    return name in out
+    out, err, ret = call(ctx, [
+        ctx.container_engine.path, 'container', 'inspect',
+        '--format', '{{.State.Status}}', name
+    ])
+    return out == 'running'
 
 
 def get_legacy_config_fsid(cluster, legacy_dir=None):
@@ -2163,7 +2153,6 @@ def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid,
 
     if daemon_type in Monitoring.components.keys():
         config_json: Dict[str, Any] = get_parm(ctx.config_json)
-        required_files = Monitoring.components[daemon_type].get('config-json-files', list())
 
         # Set up directories specific to the monitoring component
         config_dir = ''
@@ -2192,10 +2181,14 @@ def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid,
             makedirs(os.path.join(data_dir_root, config_dir, 'data'), uid, gid, 0o755)
 
         # populate the config directory for the component from the config-json
-        for fname in required_files:
-            if 'files' in config_json:  # type: ignore
+        if 'files' in config_json:
+            for fname in config_json['files']:
                 content = dict_get_join(config_json['files'], fname)
-                with open(os.path.join(data_dir_root, config_dir, fname), 'w') as f:
+                if os.path.isabs(fname):
+                    fpath = os.path.join(data_dir_root, fname.lstrip(os.path.sep))
+                else:
+                    fpath = os.path.join(data_dir_root, config_dir, fname)
+                with open(fpath, 'w', encoding='utf-8') as f:
                     os.fchown(f.fileno(), uid, gid)
                     os.fchmod(f.fileno(), 0o600)
                     f.write(content)
@@ -2267,8 +2260,8 @@ def get_config_and_keyring(ctx):
         try:
             with open(ctx.config, 'r') as f:
                 config = f.read()
-        except FileNotFoundError:
-            raise Error('config file: %s does not exist' % ctx.config)
+        except FileNotFoundError as e:
+            raise Error(e)
 
     if 'key' in ctx and ctx.key:
         keyring = '[%s]\n\tkey = %s\n' % (ctx.name, ctx.key)
@@ -2276,8 +2269,8 @@ def get_config_and_keyring(ctx):
         try:
             with open(ctx.keyring, 'r') as f:
                 keyring = f.read()
-        except FileNotFoundError:
-            raise Error('keyring file: %s does not exist' % ctx.keyring)
+        except FileNotFoundError as e:
+            raise Error(e)
 
     return config, keyring
 
@@ -2682,7 +2675,7 @@ def deploy_daemon_units(
     # cmd
     data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id)
     with open(data_dir + '/unit.run.new', 'w') as f, \
-         open(data_dir + '/unit.meta.new', 'w') as metaf:
+            open(data_dir + '/unit.meta.new', 'w') as metaf:
         f.write('set -e\n')
 
         if daemon_type in Ceph.daemons:
@@ -2718,18 +2711,11 @@ def deploy_daemon_units(
                     memory_limit=ctx.memory_limit,
                 )
                 _write_container_cmd_to_bash(ctx, f, prestart, 'LVM OSDs use ceph-volume lvm activate')
-        elif daemon_type == NFSGanesha.daemon_type:
-            # add nfs to the rados grace db
-            nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id)
-            prestart = nfs_ganesha.get_rados_grace_container('add')
-            _write_container_cmd_to_bash(ctx, f, prestart, 'add daemon to rados grace')
         elif daemon_type == CephIscsi.daemon_type:
             f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n')
             ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id)
             tcmu_container = ceph_iscsi.get_tcmu_runner_container()
             _write_container_cmd_to_bash(ctx, f, tcmu_container, 'iscsi tcmu-runnter container', background=True)
-        elif daemon_type == Keepalived.daemon_type:
-            f.write(Keepalived.get_prestart())
 
         _write_container_cmd_to_bash(ctx, f, c, '%s.%s' % (daemon_type, str(daemon_id)))
 
@@ -2771,11 +2757,6 @@ def deploy_daemon_units(
                                                     daemon_id),
             )
             _write_container_cmd_to_bash(ctx, f, poststop, 'deactivate osd')
-        elif daemon_type == NFSGanesha.daemon_type:
-            # remove nfs from the rados grace db
-            nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id)
-            poststop = nfs_ganesha.get_rados_grace_container('remove')
-            _write_container_cmd_to_bash(ctx, f, poststop, 'remove daemon from rados grace')
         elif daemon_type == CephIscsi.daemon_type:
             # make sure we also stop the tcmu container
             ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id)
@@ -2793,6 +2774,9 @@ def deploy_daemon_units(
             os.rename(data_dir + '/unit.image.new',
                       data_dir + '/unit.image')
 
+    # sysctl
+    install_sysctl(ctx, fsid, daemon_type)
+
     # systemd
     install_base_units(ctx, fsid)
     unit = get_unit_file(ctx, fsid)
@@ -2935,6 +2919,36 @@ def update_firewalld(ctx, daemon_type):
     firewall.apply_rules()
 
 
+def install_sysctl(ctx: CephadmContext, fsid: str, daemon_type: str) -> None:
+    """
+    Set up sysctl settings
+    """
+    def _write(conf: Path, lines: List[str]) -> None:
+        lines = [
+            '# created by cephadm',
+            '',
+            *lines,
+            '',
+        ]
+        with open(conf, 'w') as f:
+            f.write('\n'.join(lines))
+
+    conf = Path(ctx.sysctl_dir).joinpath(f'90-ceph-{fsid}-{daemon_type}.conf')
+    lines: Optional[List] = None
+
+    if daemon_type == 'osd':
+        lines = OSD.get_sysctl_settings()
+    elif daemon_type == 'haproxy':
+        lines = HAproxy.get_sysctl_settings()
+    elif daemon_type == 'keepalived':
+        lines = Keepalived.get_sysctl_settings()
+
+    # apply the sysctl settings
+    if lines:
+        _write(conf, lines)
+        call_throws(ctx, ['sysctl', '--system'])
+
+
 def install_base_units(ctx, fsid):
     # type: (CephadmContext, str) -> None
     """
@@ -3102,13 +3116,13 @@ class CephContainer:
             'run',
             '--rm',
             '--ipc=host',
+            # some containers (ahem, haproxy) override this, but we want a fast
+            # shutdown always (and, more importantly, a successful exit even if we
+            # fall back to SIGKILL).
+            '--stop-signal=SIGTERM',
         ]
 
         if isinstance(self.ctx.container_engine, Podman):
-            # podman adds the container *name* to /etc/hosts (for 127.0.1.1)
-            # by default, which makes python's socket.getfqdn() return that
-            # instead of a valid hostname.
-            cmd_args.append('--no-hosts')
             if os.path.exists('/etc/ceph/podman-auth.json'):
                 cmd_args.append('--authfile=/etc/ceph/podman-auth.json')
 
@@ -3966,11 +3980,7 @@ def command_bootstrap(ctx):
             except PermissionError:
                 raise Error(f'Unable to create {dirname} due to permissions failure. Retry with root, or sudo or preallocate the directory.')
 
-    if ctx.config and os.path.exists(ctx.config):
-        with open(ctx.config) as f:
-            user_conf = f.read()
-    else:
-        user_conf = None
+    (user_conf, _) = get_config_and_keyring(ctx)
 
     if not ctx.skip_prepare_host:
         command_prepare_host(ctx)
@@ -4072,7 +4082,7 @@ def command_bootstrap(ctx):
         # the mgr (e.g. mgr/cephadm/container_image_prometheus)
         # they don't seem to be stored if there isn't a mgr yet.
         # Since re-assimilating the same conf settings should be
-        # idempotent we can just do it aain here.
+        # idempotent we can just do it again here.
         with tempfile.NamedTemporaryFile(buffering=0) as tmp:
             tmp.write(user_conf.encode('utf-8'))
             cli(['config', 'assimilate-conf',
@@ -4329,8 +4339,8 @@ def command_deploy(ctx):
                       ports=daemon_ports)
 
     elif daemon_type == NFSGanesha.daemon_type:
-        if not ctx.reconfig and not redeploy:
-            daemon_ports.extend(NFSGanesha.port_map.values())
+        if not ctx.reconfig and not redeploy and not daemon_ports:
+            daemon_ports = list(NFSGanesha.port_map.values())
 
         config, keyring = get_config_and_keyring(ctx)
         # TODO: extract ganesha uid/gid (997, 994) ?
@@ -4489,6 +4499,10 @@ def command_shell(ctx):
                                             os.path.join(home, f))
             mounts[home] = '/root'
 
+    for i in ctx.volume:
+        a, b = i.split(':', 1)
+        mounts[a] = b
+
     c = CephContainer(
         ctx,
         image=ctx.image,
@@ -5507,10 +5521,10 @@ def command_rm_cluster(ctx):
         _zap_osds(ctx)
 
     # rm units
-    call_throws(ctx, ['rm', '-f', ctx.unit_dir +  # noqa: W504
-                      '/ceph-%s@.service' % ctx.fsid])
-    call_throws(ctx, ['rm', '-f', ctx.unit_dir +  # noqa: W504
-                      '/ceph-%s.target' % ctx.fsid])
+    call_throws(ctx, ['rm', '-f', ctx.unit_dir
+                      + '/ceph-%s@.service' % ctx.fsid])
+    call_throws(ctx, ['rm', '-f', ctx.unit_dir
+                      + '/ceph-%s.target' % ctx.fsid])
     call_throws(ctx, ['rm', '-rf',
                       ctx.unit_dir + '/ceph-%s.target.wants' % ctx.fsid])
     # rm data
@@ -5519,12 +5533,21 @@ def command_rm_cluster(ctx):
     if not ctx.keep_logs:
         # rm logs
         call_throws(ctx, ['rm', '-rf', ctx.log_dir + '/' + ctx.fsid])
-        call_throws(ctx, ['rm', '-rf', ctx.log_dir +  # noqa: W504
-                          '/*.wants/ceph-%s@*' % ctx.fsid])
+        call_throws(ctx, ['rm', '-rf', ctx.log_dir
+                          + '/*.wants/ceph-%s@*' % ctx.fsid])
 
     # rm logrotate config
     call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/ceph-%s' % ctx.fsid])
 
+    # rm cephadm logrotate config if last cluster on host
+    if not os.listdir(ctx.data_dir):
+        call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/cephadm'])
+
+    # rm sysctl settings
+    sysctl_dir = Path(ctx.sysctl_dir)
+    for p in sysctl_dir.glob(f'90-ceph-{ctx.fsid}-*.conf'):
+        p.unlink()
+
     # clean up config, keyring, and pub key files
     files = ['/etc/ceph/ceph.conf', '/etc/ceph/ceph.pub', '/etc/ceph/ceph.client.admin.keyring']
 
@@ -6712,16 +6735,6 @@ def command_gather_facts(ctx: CephadmContext):
     host = HostFacts(ctx)
     print(host.dump())
 
-##################################
-
-
-def command_verify_prereqs(ctx: CephadmContext):
-    if ctx.service_type == 'haproxy' or ctx.service_type == 'keepalived':
-        out, err, code = call(
-            ctx, ['sysctl', '-n', 'net.ipv4.ip_nonlocal_bind']
-        )
-        if out.strip() != '1':
-            raise Error('net.ipv4.ip_nonlocal_bind not set to 1')
 
 ##################################
 
@@ -6803,12 +6816,14 @@ class CephadmDaemonHandler(BaseHTTPRequestHandler):
             The token is installed at deployment time and must be provided to
             ensure we only respond to callers who know our token i.e. mgr
             """
+
             def wrapper(self, *args, **kwargs):
                 auth = self.headers.get('Authorization', None)
                 if auth != 'Bearer ' + self.server.token:
                     self.send_error(401)
                     return
                 f(self, *args, **kwargs)
+
             return wrapper
 
     def _help_page(self):
@@ -7484,6 +7499,10 @@ def _get_parser():
         default=LOGROTATE_DIR,
         help='location of logrotate configuration files')
     parser.add_argument(
+        '--sysctl-dir',
+        default=SYSCTL_DIR,
+        help='location of sysctl configuration files')
+    parser.add_argument(
         '--unit-dir',
         default=UNIT_DIR,
         help='base directory for systemd units')
@@ -7663,6 +7682,11 @@ def _get_parser():
         default=[],
         help='set environment variable')
     parser_shell.add_argument(
+        '--volume', '-v',
+        action='append',
+        default=[],
+        help='set environment variable')
+    parser_shell.add_argument(
         'command', nargs=argparse.REMAINDER,
         help='command (optional)')
     parser_shell.add_argument(
@@ -8083,15 +8107,6 @@ def _get_parser():
         help='Maintenance action - enter maintenance, or exit maintenance')
     parser_maintenance.set_defaults(func=command_maintenance)
 
-    parser_verify_prereqs = subparsers.add_parser(
-        'verify-prereqs',
-        help='verify system prerequisites for a given service are met on this host')
-    parser_verify_prereqs.set_defaults(func=command_verify_prereqs)
-    parser_verify_prereqs.add_argument(
-        '--daemon-type',
-        required=True,
-        help='service type of service to whose prereqs will be checked')
-
     return parser
 
 
@@ -8135,6 +8150,18 @@ def cephadm_init(args: List[str]) -> Optional[CephadmContext]:
     dictConfig(logging_config)
     logger = logging.getLogger()
 
+    if not os.path.exists(ctx.logrotate_dir + '/cephadm'):
+        with open(ctx.logrotate_dir + '/cephadm', 'w') as f:
+            f.write("""# created by cephadm
+/var/log/ceph/cephadm.log {
+    rotate 7
+    daily
+    compress
+    missingok
+    notifempty
+}
+""")
+
     if ctx.verbose:
         for handler in logger.handlers:
             if handler.name == 'console':
diff --git a/SOURCES/cephadm-16.2.4-3.patch b/SOURCES/cephadm-16.2.4-3.patch
new file mode 100644
index 0000000..24e8ef7
--- /dev/null
+++ b/SOURCES/cephadm-16.2.4-3.patch
@@ -0,0 +1,482 @@
+diff --git a/SOURCES/cephadm b/SOURCES/cephadm
+index 524c038..a7b664c 100644
+--- a/SOURCES/cephadm
++++ b/SOURCES/cephadm
+@@ -61,6 +61,7 @@ DATA_DIR = '/var/lib/ceph'
+ LOG_DIR = '/var/log/ceph'
+ LOCK_DIR = '/run/cephadm'
+ LOGROTATE_DIR = '/etc/logrotate.d'
++SYSCTL_DIR = '/usr/lib/sysctl.d'
+ UNIT_DIR = '/etc/systemd/system'
+ LOG_DIR_MODE = 0o770
+ DATA_DIR_MODE = 0o700
+@@ -109,6 +110,7 @@ class BaseConfig:
+         self.data_dir: str = DATA_DIR
+         self.log_dir: str = LOG_DIR
+         self.logrotate_dir: str = LOGROTATE_DIR
++        self.sysctl_dir: str = SYSCTL_DIR
+         self.unit_dir: str = UNIT_DIR
+         self.verbose: bool = False
+         self.timeout: Optional[int] = DEFAULT_TIMEOUT
+@@ -210,11 +212,9 @@ logging_config = {
+         },
+         'log_file': {
+             'level': 'DEBUG',
+-            'class': 'logging.handlers.RotatingFileHandler',
++            'class': 'logging.handlers.WatchedFileHandler',
+             'formatter': 'cephadm',
+             'filename': '%s/cephadm.log' % LOG_DIR,
+-            'maxBytes': 1024000,
+-            'backupCount': 1,
+         }
+     },
+     'loggers': {
+@@ -249,6 +249,18 @@ class Ceph(object):
+ ##################################
+ 
+ 
++class OSD(object):
++    @staticmethod
++    def get_sysctl_settings() -> List[str]:
++        return [
++            '# allow a large number of OSDs',
++            'fs.aio-max-nr = 1048576',
++            'kernel.pid_max = 4194304',
++        ]
++
++##################################
++
++
+ class Monitoring(object):
+     """Define the configs for the monitoring containers"""
+ 
+@@ -349,7 +361,7 @@ def populate_files(config_dir, config_files, uid, gid):
+         config_file = os.path.join(config_dir, fname)
+         config_content = dict_get_join(config_files, fname)
+         logger.info('Write file: %s' % (config_file))
+-        with open(config_file, 'w') as f:
++        with open(config_file, 'w', encoding='utf-8') as f:
+             os.fchown(f.fileno(), uid, gid)
+             os.fchmod(f.fileno(), 0o600)
+             f.write(config_content)
+@@ -491,36 +503,6 @@ class NFSGanesha(object):
+                 os.fchown(f.fileno(), uid, gid)
+                 f.write(self.rgw.get('keyring', ''))
+ 
+-    def get_rados_grace_container(self, action):
+-        # type: (str) -> CephContainer
+-        """Container for a ganesha action on the grace db"""
+-        entrypoint = '/usr/bin/ganesha-rados-grace'
+-
+-        assert self.pool
+-        args = ['--pool', self.pool]
+-        if self.namespace:
+-            args += ['--ns', self.namespace]
+-        if self.userid:
+-            args += ['--userid', self.userid]
+-        args += [action, self.get_daemon_name()]
+-
+-        data_dir = get_data_dir(self.fsid, self.ctx.data_dir,
+-                                self.daemon_type, self.daemon_id)
+-        volume_mounts = self.get_container_mounts(data_dir)
+-        envs = self.get_container_envs()
+-
+-        logger.info('Creating RADOS grace for action: %s' % action)
+-        c = CephContainer(
+-            self.ctx,
+-            image=self.image,
+-            entrypoint=entrypoint,
+-            args=args,
+-            volume_mounts=volume_mounts,
+-            cname=self.get_container_name(desc='grace-%s' % action),
+-            envs=envs
+-        )
+-        return c
+-
+ ##################################
+ 
+ 
+@@ -732,6 +714,13 @@ class HAproxy(object):
+         mounts[os.path.join(data_dir, 'haproxy')] = '/var/lib/haproxy'
+         return mounts
+ 
++    @staticmethod
++    def get_sysctl_settings() -> List[str]:
++        return [
++            '# IP forwarding',
++            'net.ipv4.ip_forward = 1',
++        ]
++
+ ##################################
+ 
+ 
+@@ -811,12 +800,12 @@ class Keepalived(object):
+         return envs
+ 
+     @staticmethod
+-    def get_prestart():
+-        return (
+-            '# keepalived needs IP forwarding and non-local bind\n'
+-            'sysctl net.ipv4.ip_forward=1\n'
+-            'sysctl net.ipv4.ip_nonlocal_bind=1\n'
+-        )
++    def get_sysctl_settings() -> List[str]:
++        return [
++            '# IP forwarding and non-local bind',
++            'net.ipv4.ip_forward = 1',
++            'net.ipv4.ip_nonlocal_bind = 1',
++        ]
+ 
+     def extract_uid_gid_keepalived(self):
+         # better directory for this?
+@@ -1190,7 +1179,7 @@ class FileLock(object):
+                         lock_id, lock_filename, poll_intervall
+                     )
+                     time.sleep(poll_intervall)
+-        except:  # noqa
++        except Exception:
+             # Something did go wrong, so decrement the counter.
+             self._lock_counter = max(0, self._lock_counter - 1)
+ 
+@@ -2061,10 +2050,11 @@ def check_units(ctx, units, enabler=None):
+ 
+ 
+ def is_container_running(ctx: CephadmContext, name: str) -> bool:
+-    out, err, ret = call_throws(ctx, [
+-        ctx.container_engine.path, 'ps',
+-        '--format', '{{.Names}}'])
+-    return name in out
++    out, err, ret = call(ctx, [
++        ctx.container_engine.path, 'container', 'inspect',
++        '--format', '{{.State.Status}}', name
++    ])
++    return out == 'running'
+ 
+ 
+ def get_legacy_config_fsid(cluster, legacy_dir=None):
+@@ -2163,7 +2153,6 @@ def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid,
+ 
+     if daemon_type in Monitoring.components.keys():
+         config_json: Dict[str, Any] = get_parm(ctx.config_json)
+-        required_files = Monitoring.components[daemon_type].get('config-json-files', list())
+ 
+         # Set up directories specific to the monitoring component
+         config_dir = ''
+@@ -2192,10 +2181,14 @@ def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid,
+             makedirs(os.path.join(data_dir_root, config_dir, 'data'), uid, gid, 0o755)
+ 
+         # populate the config directory for the component from the config-json
+-        for fname in required_files:
+-            if 'files' in config_json:  # type: ignore
++        if 'files' in config_json:
++            for fname in config_json['files']:
+                 content = dict_get_join(config_json['files'], fname)
+-                with open(os.path.join(data_dir_root, config_dir, fname), 'w') as f:
++                if os.path.isabs(fname):
++                    fpath = os.path.join(data_dir_root, fname.lstrip(os.path.sep))
++                else:
++                    fpath = os.path.join(data_dir_root, config_dir, fname)
++                with open(fpath, 'w', encoding='utf-8') as f:
+                     os.fchown(f.fileno(), uid, gid)
+                     os.fchmod(f.fileno(), 0o600)
+                     f.write(content)
+@@ -2267,8 +2260,8 @@ def get_config_and_keyring(ctx):
+         try:
+             with open(ctx.config, 'r') as f:
+                 config = f.read()
+-        except FileNotFoundError:
+-            raise Error('config file: %s does not exist' % ctx.config)
++        except FileNotFoundError as e:
++            raise Error(e)
+ 
+     if 'key' in ctx and ctx.key:
+         keyring = '[%s]\n\tkey = %s\n' % (ctx.name, ctx.key)
+@@ -2276,8 +2269,8 @@ def get_config_and_keyring(ctx):
+         try:
+             with open(ctx.keyring, 'r') as f:
+                 keyring = f.read()
+-        except FileNotFoundError:
+-            raise Error('keyring file: %s does not exist' % ctx.keyring)
++        except FileNotFoundError as e:
++            raise Error(e)
+ 
+     return config, keyring
+ 
+@@ -2682,7 +2675,7 @@ def deploy_daemon_units(
+     # cmd
+     data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id)
+     with open(data_dir + '/unit.run.new', 'w') as f, \
+-         open(data_dir + '/unit.meta.new', 'w') as metaf:
++            open(data_dir + '/unit.meta.new', 'w') as metaf:
+         f.write('set -e\n')
+ 
+         if daemon_type in Ceph.daemons:
+@@ -2718,18 +2711,11 @@ def deploy_daemon_units(
+                     memory_limit=ctx.memory_limit,
+                 )
+                 _write_container_cmd_to_bash(ctx, f, prestart, 'LVM OSDs use ceph-volume lvm activate')
+-        elif daemon_type == NFSGanesha.daemon_type:
+-            # add nfs to the rados grace db
+-            nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id)
+-            prestart = nfs_ganesha.get_rados_grace_container('add')
+-            _write_container_cmd_to_bash(ctx, f, prestart, 'add daemon to rados grace')
+         elif daemon_type == CephIscsi.daemon_type:
+             f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n')
+             ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id)
+             tcmu_container = ceph_iscsi.get_tcmu_runner_container()
+             _write_container_cmd_to_bash(ctx, f, tcmu_container, 'iscsi tcmu-runnter container', background=True)
+-        elif daemon_type == Keepalived.daemon_type:
+-            f.write(Keepalived.get_prestart())
+ 
+         _write_container_cmd_to_bash(ctx, f, c, '%s.%s' % (daemon_type, str(daemon_id)))
+ 
+@@ -2771,11 +2757,6 @@ def deploy_daemon_units(
+                                                     daemon_id),
+             )
+             _write_container_cmd_to_bash(ctx, f, poststop, 'deactivate osd')
+-        elif daemon_type == NFSGanesha.daemon_type:
+-            # remove nfs from the rados grace db
+-            nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id)
+-            poststop = nfs_ganesha.get_rados_grace_container('remove')
+-            _write_container_cmd_to_bash(ctx, f, poststop, 'remove daemon from rados grace')
+         elif daemon_type == CephIscsi.daemon_type:
+             # make sure we also stop the tcmu container
+             ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id)
+@@ -2793,6 +2774,9 @@ def deploy_daemon_units(
+             os.rename(data_dir + '/unit.image.new',
+                       data_dir + '/unit.image')
+ 
++    # sysctl
++    install_sysctl(ctx, fsid, daemon_type)
++
+     # systemd
+     install_base_units(ctx, fsid)
+     unit = get_unit_file(ctx, fsid)
+@@ -2935,6 +2919,36 @@ def update_firewalld(ctx, daemon_type):
+     firewall.apply_rules()
+ 
+ 
++def install_sysctl(ctx: CephadmContext, fsid: str, daemon_type: str) -> None:
++    """
++    Set up sysctl settings
++    """
++    def _write(conf: Path, lines: List[str]) -> None:
++        lines = [
++            '# created by cephadm',
++            '',
++            *lines,
++            '',
++        ]
++        with open(conf, 'w') as f:
++            f.write('\n'.join(lines))
++
++    conf = Path(ctx.sysctl_dir).joinpath(f'90-ceph-{fsid}-{daemon_type}.conf')
++    lines: Optional[List] = None
++
++    if daemon_type == 'osd':
++        lines = OSD.get_sysctl_settings()
++    elif daemon_type == 'haproxy':
++        lines = HAproxy.get_sysctl_settings()
++    elif daemon_type == 'keepalived':
++        lines = Keepalived.get_sysctl_settings()
++
++    # apply the sysctl settings
++    if lines:
++        _write(conf, lines)
++        call_throws(ctx, ['sysctl', '--system'])
++
++
+ def install_base_units(ctx, fsid):
+     # type: (CephadmContext, str) -> None
+     """
+@@ -3102,13 +3116,13 @@ class CephContainer:
+             'run',
+             '--rm',
+             '--ipc=host',
++            # some containers (ahem, haproxy) override this, but we want a fast
++            # shutdown always (and, more importantly, a successful exit even if we
++            # fall back to SIGKILL).
++            '--stop-signal=SIGTERM',
+         ]
+ 
+         if isinstance(self.ctx.container_engine, Podman):
+-            # podman adds the container *name* to /etc/hosts (for 127.0.1.1)
+-            # by default, which makes python's socket.getfqdn() return that
+-            # instead of a valid hostname.
+-            cmd_args.append('--no-hosts')
+             if os.path.exists('/etc/ceph/podman-auth.json'):
+                 cmd_args.append('--authfile=/etc/ceph/podman-auth.json')
+ 
+@@ -3966,11 +3980,7 @@ def command_bootstrap(ctx):
+             except PermissionError:
+                 raise Error(f'Unable to create {dirname} due to permissions failure. Retry with root, or sudo or preallocate the directory.')
+ 
+-    if ctx.config and os.path.exists(ctx.config):
+-        with open(ctx.config) as f:
+-            user_conf = f.read()
+-    else:
+-        user_conf = None
++    (user_conf, _) = get_config_and_keyring(ctx)
+ 
+     if not ctx.skip_prepare_host:
+         command_prepare_host(ctx)
+@@ -4072,7 +4082,7 @@ def command_bootstrap(ctx):
+         # the mgr (e.g. mgr/cephadm/container_image_prometheus)
+         # they don't seem to be stored if there isn't a mgr yet.
+         # Since re-assimilating the same conf settings should be
+-        # idempotent we can just do it aain here.
++        # idempotent we can just do it again here.
+         with tempfile.NamedTemporaryFile(buffering=0) as tmp:
+             tmp.write(user_conf.encode('utf-8'))
+             cli(['config', 'assimilate-conf',
+@@ -4329,8 +4339,8 @@ def command_deploy(ctx):
+                       ports=daemon_ports)
+ 
+     elif daemon_type == NFSGanesha.daemon_type:
+-        if not ctx.reconfig and not redeploy:
+-            daemon_ports.extend(NFSGanesha.port_map.values())
++        if not ctx.reconfig and not redeploy and not daemon_ports:
++            daemon_ports = list(NFSGanesha.port_map.values())
+ 
+         config, keyring = get_config_and_keyring(ctx)
+         # TODO: extract ganesha uid/gid (997, 994) ?
+@@ -4489,6 +4499,10 @@ def command_shell(ctx):
+                                             os.path.join(home, f))
+             mounts[home] = '/root'
+ 
++    for i in ctx.volume:
++        a, b = i.split(':', 1)
++        mounts[a] = b
++
+     c = CephContainer(
+         ctx,
+         image=ctx.image,
+@@ -5507,10 +5521,10 @@ def command_rm_cluster(ctx):
+         _zap_osds(ctx)
+ 
+     # rm units
+-    call_throws(ctx, ['rm', '-f', ctx.unit_dir +  # noqa: W504
+-                      '/ceph-%s@.service' % ctx.fsid])
+-    call_throws(ctx, ['rm', '-f', ctx.unit_dir +  # noqa: W504
+-                      '/ceph-%s.target' % ctx.fsid])
++    call_throws(ctx, ['rm', '-f', ctx.unit_dir
++                      + '/ceph-%s@.service' % ctx.fsid])
++    call_throws(ctx, ['rm', '-f', ctx.unit_dir
++                      + '/ceph-%s.target' % ctx.fsid])
+     call_throws(ctx, ['rm', '-rf',
+                       ctx.unit_dir + '/ceph-%s.target.wants' % ctx.fsid])
+     # rm data
+@@ -5519,12 +5533,21 @@ def command_rm_cluster(ctx):
+     if not ctx.keep_logs:
+         # rm logs
+         call_throws(ctx, ['rm', '-rf', ctx.log_dir + '/' + ctx.fsid])
+-        call_throws(ctx, ['rm', '-rf', ctx.log_dir +  # noqa: W504
+-                          '/*.wants/ceph-%s@*' % ctx.fsid])
++        call_throws(ctx, ['rm', '-rf', ctx.log_dir
++                          + '/*.wants/ceph-%s@*' % ctx.fsid])
+ 
+     # rm logrotate config
+     call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/ceph-%s' % ctx.fsid])
+ 
++    # rm cephadm logrotate config if last cluster on host
++    if not os.listdir(ctx.data_dir):
++        call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/cephadm'])
++
++    # rm sysctl settings
++    sysctl_dir = Path(ctx.sysctl_dir)
++    for p in sysctl_dir.glob(f'90-ceph-{ctx.fsid}-*.conf'):
++        p.unlink()
++
+     # clean up config, keyring, and pub key files
+     files = ['/etc/ceph/ceph.conf', '/etc/ceph/ceph.pub', '/etc/ceph/ceph.client.admin.keyring']
+ 
+@@ -6712,16 +6735,6 @@ def command_gather_facts(ctx: CephadmContext):
+     host = HostFacts(ctx)
+     print(host.dump())
+ 
+-##################################
+-
+-
+-def command_verify_prereqs(ctx: CephadmContext):
+-    if ctx.service_type == 'haproxy' or ctx.service_type == 'keepalived':
+-        out, err, code = call(
+-            ctx, ['sysctl', '-n', 'net.ipv4.ip_nonlocal_bind']
+-        )
+-        if out.strip() != '1':
+-            raise Error('net.ipv4.ip_nonlocal_bind not set to 1')
+ 
+ ##################################
+ 
+@@ -6803,12 +6816,14 @@ class CephadmDaemonHandler(BaseHTTPRequestHandler):
+             The token is installed at deployment time and must be provided to
+             ensure we only respond to callers who know our token i.e. mgr
+             """
++
+             def wrapper(self, *args, **kwargs):
+                 auth = self.headers.get('Authorization', None)
+                 if auth != 'Bearer ' + self.server.token:
+                     self.send_error(401)
+                     return
+                 f(self, *args, **kwargs)
++
+             return wrapper
+ 
+     def _help_page(self):
+@@ -7483,6 +7498,10 @@ def _get_parser():
+         '--logrotate-dir',
+         default=LOGROTATE_DIR,
+         help='location of logrotate configuration files')
++    parser.add_argument(
++        '--sysctl-dir',
++        default=SYSCTL_DIR,
++        help='location of sysctl configuration files')
+     parser.add_argument(
+         '--unit-dir',
+         default=UNIT_DIR,
+@@ -7662,6 +7681,11 @@ def _get_parser():
+         action='append',
+         default=[],
+         help='set environment variable')
++    parser_shell.add_argument(
++        '--volume', '-v',
++        action='append',
++        default=[],
++        help='set environment variable')
+     parser_shell.add_argument(
+         'command', nargs=argparse.REMAINDER,
+         help='command (optional)')
+@@ -8083,15 +8107,6 @@ def _get_parser():
+         help='Maintenance action - enter maintenance, or exit maintenance')
+     parser_maintenance.set_defaults(func=command_maintenance)
+ 
+-    parser_verify_prereqs = subparsers.add_parser(
+-        'verify-prereqs',
+-        help='verify system prerequisites for a given service are met on this host')
+-    parser_verify_prereqs.set_defaults(func=command_verify_prereqs)
+-    parser_verify_prereqs.add_argument(
+-        '--daemon-type',
+-        required=True,
+-        help='service type of service to whose prereqs will be checked')
+-
+     return parser
+ 
+ 
+@@ -8135,6 +8150,18 @@ def cephadm_init(args: List[str]) -> Optional[CephadmContext]:
+     dictConfig(logging_config)
+     logger = logging.getLogger()
+ 
++    if not os.path.exists(ctx.logrotate_dir + '/cephadm'):
++        with open(ctx.logrotate_dir + '/cephadm', 'w') as f:
++            f.write("""# created by cephadm
++/var/log/ceph/cephadm.log {
++    rotate 7
++    daily
++    compress
++    missingok
++    notifempty
++}
++""")
++
+     if ctx.verbose:
+         for handler in logger.handlers:
+             if handler.name == 'console':
diff --git a/SPECS/cephadm.spec.in b/SPECS/cephadm.spec.in
index dec43bc..0c53c9f 100644
--- a/SPECS/cephadm.spec.in
+++ b/SPECS/cephadm.spec.in
@@ -1,14 +1,14 @@
 # Upstream ceph commit upon which this package is based:
-# patches_base=d3d607667e58de33790b10a32b08f2b29d5aa567
+# patches_base=e132babf2904505731f0d45d5168c7e0e12c06be
 
 Name:    cephadm
 Epoch:   2
 Version: 16.2.4
-Release: 2%{?dist}
+Release: 3%{?dist}
 Summary: Utility to bootstrap Ceph clusters
 License: LGPL-2.1
 URL:     https://ceph.io
-Source0: https://github.com/ceph/ceph/raw/d3d607667e58de33790b10a32b08f2b29d5aa567/src/cephadm/cephadm
+Source0: https://github.com/ceph/ceph/raw/e132babf2904505731f0d45d5168c7e0e12c06be/src/cephadm/cephadm
 Source1: COPYING-LGPL2.1
 
 BuildArch: noarch
@@ -54,5 +54,8 @@ exit 0
 
 %changelog
 
+* Thu Jun 10 2021 Francesco Pantano <fpantano@redhat.com> - 2:16.2.4-3
+- 16.2.4-3 GA
+
 * Thu Mar 25 2021 Francesco Pantano <fpantano@redhat.com> - 2:16.1.0-2
 - 16.1.0-2 GA