diff --git a/SOURCES/cephadm b/SOURCES/cephadm index 524c038..a7b664c 100644 --- a/SOURCES/cephadm +++ b/SOURCES/cephadm @@ -61,6 +61,7 @@ DATA_DIR = '/var/lib/ceph' LOG_DIR = '/var/log/ceph' LOCK_DIR = '/run/cephadm' LOGROTATE_DIR = '/etc/logrotate.d' +SYSCTL_DIR = '/usr/lib/sysctl.d' UNIT_DIR = '/etc/systemd/system' LOG_DIR_MODE = 0o770 DATA_DIR_MODE = 0o700 @@ -109,6 +110,7 @@ class BaseConfig: self.data_dir: str = DATA_DIR self.log_dir: str = LOG_DIR self.logrotate_dir: str = LOGROTATE_DIR + self.sysctl_dir: str = SYSCTL_DIR self.unit_dir: str = UNIT_DIR self.verbose: bool = False self.timeout: Optional[int] = DEFAULT_TIMEOUT @@ -210,11 +212,9 @@ logging_config = { }, 'log_file': { 'level': 'DEBUG', - 'class': 'logging.handlers.RotatingFileHandler', + 'class': 'logging.handlers.WatchedFileHandler', 'formatter': 'cephadm', 'filename': '%s/cephadm.log' % LOG_DIR, - 'maxBytes': 1024000, - 'backupCount': 1, } }, 'loggers': { @@ -249,6 +249,18 @@ class Ceph(object): ################################## +class OSD(object): + @staticmethod + def get_sysctl_settings() -> List[str]: + return [ + '# allow a large number of OSDs', + 'fs.aio-max-nr = 1048576', + 'kernel.pid_max = 4194304', + ] + +################################## + + class Monitoring(object): """Define the configs for the monitoring containers""" @@ -349,7 +361,7 @@ def populate_files(config_dir, config_files, uid, gid): config_file = os.path.join(config_dir, fname) config_content = dict_get_join(config_files, fname) logger.info('Write file: %s' % (config_file)) - with open(config_file, 'w') as f: + with open(config_file, 'w', encoding='utf-8') as f: os.fchown(f.fileno(), uid, gid) os.fchmod(f.fileno(), 0o600) f.write(config_content) @@ -491,36 +503,6 @@ class NFSGanesha(object): os.fchown(f.fileno(), uid, gid) f.write(self.rgw.get('keyring', '')) - def get_rados_grace_container(self, action): - # type: (str) -> CephContainer - """Container for a ganesha action on the grace db""" - entrypoint = '/usr/bin/ganesha-rados-grace' - - assert self.pool - args = ['--pool', self.pool] - if self.namespace: - args += ['--ns', self.namespace] - if self.userid: - args += ['--userid', self.userid] - args += [action, self.get_daemon_name()] - - data_dir = get_data_dir(self.fsid, self.ctx.data_dir, - self.daemon_type, self.daemon_id) - volume_mounts = self.get_container_mounts(data_dir) - envs = self.get_container_envs() - - logger.info('Creating RADOS grace for action: %s' % action) - c = CephContainer( - self.ctx, - image=self.image, - entrypoint=entrypoint, - args=args, - volume_mounts=volume_mounts, - cname=self.get_container_name(desc='grace-%s' % action), - envs=envs - ) - return c - ################################## @@ -732,6 +714,13 @@ class HAproxy(object): mounts[os.path.join(data_dir, 'haproxy')] = '/var/lib/haproxy' return mounts + @staticmethod + def get_sysctl_settings() -> List[str]: + return [ + '# IP forwarding', + 'net.ipv4.ip_forward = 1', + ] + ################################## @@ -811,12 +800,12 @@ class Keepalived(object): return envs @staticmethod - def get_prestart(): - return ( - '# keepalived needs IP forwarding and non-local bind\n' - 'sysctl net.ipv4.ip_forward=1\n' - 'sysctl net.ipv4.ip_nonlocal_bind=1\n' - ) + def get_sysctl_settings() -> List[str]: + return [ + '# IP forwarding and non-local bind', + 'net.ipv4.ip_forward = 1', + 'net.ipv4.ip_nonlocal_bind = 1', + ] def extract_uid_gid_keepalived(self): # better directory for this? @@ -1190,7 +1179,7 @@ class FileLock(object): lock_id, lock_filename, poll_intervall ) time.sleep(poll_intervall) - except: # noqa + except Exception: # Something did go wrong, so decrement the counter. self._lock_counter = max(0, self._lock_counter - 1) @@ -2061,10 +2050,11 @@ def check_units(ctx, units, enabler=None): def is_container_running(ctx: CephadmContext, name: str) -> bool: - out, err, ret = call_throws(ctx, [ - ctx.container_engine.path, 'ps', - '--format', '{{.Names}}']) - return name in out + out, err, ret = call(ctx, [ + ctx.container_engine.path, 'container', 'inspect', + '--format', '{{.State.Status}}', name + ]) + return out == 'running' def get_legacy_config_fsid(cluster, legacy_dir=None): @@ -2163,7 +2153,6 @@ def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid, if daemon_type in Monitoring.components.keys(): config_json: Dict[str, Any] = get_parm(ctx.config_json) - required_files = Monitoring.components[daemon_type].get('config-json-files', list()) # Set up directories specific to the monitoring component config_dir = '' @@ -2192,10 +2181,14 @@ def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid, makedirs(os.path.join(data_dir_root, config_dir, 'data'), uid, gid, 0o755) # populate the config directory for the component from the config-json - for fname in required_files: - if 'files' in config_json: # type: ignore + if 'files' in config_json: + for fname in config_json['files']: content = dict_get_join(config_json['files'], fname) - with open(os.path.join(data_dir_root, config_dir, fname), 'w') as f: + if os.path.isabs(fname): + fpath = os.path.join(data_dir_root, fname.lstrip(os.path.sep)) + else: + fpath = os.path.join(data_dir_root, config_dir, fname) + with open(fpath, 'w', encoding='utf-8') as f: os.fchown(f.fileno(), uid, gid) os.fchmod(f.fileno(), 0o600) f.write(content) @@ -2267,8 +2260,8 @@ def get_config_and_keyring(ctx): try: with open(ctx.config, 'r') as f: config = f.read() - except FileNotFoundError: - raise Error('config file: %s does not exist' % ctx.config) + except FileNotFoundError as e: + raise Error(e) if 'key' in ctx and ctx.key: keyring = '[%s]\n\tkey = %s\n' % (ctx.name, ctx.key) @@ -2276,8 +2269,8 @@ def get_config_and_keyring(ctx): try: with open(ctx.keyring, 'r') as f: keyring = f.read() - except FileNotFoundError: - raise Error('keyring file: %s does not exist' % ctx.keyring) + except FileNotFoundError as e: + raise Error(e) return config, keyring @@ -2682,7 +2675,7 @@ def deploy_daemon_units( # cmd data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) with open(data_dir + '/unit.run.new', 'w') as f, \ - open(data_dir + '/unit.meta.new', 'w') as metaf: + open(data_dir + '/unit.meta.new', 'w') as metaf: f.write('set -e\n') if daemon_type in Ceph.daemons: @@ -2718,18 +2711,11 @@ def deploy_daemon_units( memory_limit=ctx.memory_limit, ) _write_container_cmd_to_bash(ctx, f, prestart, 'LVM OSDs use ceph-volume lvm activate') - elif daemon_type == NFSGanesha.daemon_type: - # add nfs to the rados grace db - nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id) - prestart = nfs_ganesha.get_rados_grace_container('add') - _write_container_cmd_to_bash(ctx, f, prestart, 'add daemon to rados grace') elif daemon_type == CephIscsi.daemon_type: f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n') ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id) tcmu_container = ceph_iscsi.get_tcmu_runner_container() _write_container_cmd_to_bash(ctx, f, tcmu_container, 'iscsi tcmu-runnter container', background=True) - elif daemon_type == Keepalived.daemon_type: - f.write(Keepalived.get_prestart()) _write_container_cmd_to_bash(ctx, f, c, '%s.%s' % (daemon_type, str(daemon_id))) @@ -2771,11 +2757,6 @@ def deploy_daemon_units( daemon_id), ) _write_container_cmd_to_bash(ctx, f, poststop, 'deactivate osd') - elif daemon_type == NFSGanesha.daemon_type: - # remove nfs from the rados grace db - nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id) - poststop = nfs_ganesha.get_rados_grace_container('remove') - _write_container_cmd_to_bash(ctx, f, poststop, 'remove daemon from rados grace') elif daemon_type == CephIscsi.daemon_type: # make sure we also stop the tcmu container ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id) @@ -2793,6 +2774,9 @@ def deploy_daemon_units( os.rename(data_dir + '/unit.image.new', data_dir + '/unit.image') + # sysctl + install_sysctl(ctx, fsid, daemon_type) + # systemd install_base_units(ctx, fsid) unit = get_unit_file(ctx, fsid) @@ -2935,6 +2919,36 @@ def update_firewalld(ctx, daemon_type): firewall.apply_rules() +def install_sysctl(ctx: CephadmContext, fsid: str, daemon_type: str) -> None: + """ + Set up sysctl settings + """ + def _write(conf: Path, lines: List[str]) -> None: + lines = [ + '# created by cephadm', + '', + *lines, + '', + ] + with open(conf, 'w') as f: + f.write('\n'.join(lines)) + + conf = Path(ctx.sysctl_dir).joinpath(f'90-ceph-{fsid}-{daemon_type}.conf') + lines: Optional[List] = None + + if daemon_type == 'osd': + lines = OSD.get_sysctl_settings() + elif daemon_type == 'haproxy': + lines = HAproxy.get_sysctl_settings() + elif daemon_type == 'keepalived': + lines = Keepalived.get_sysctl_settings() + + # apply the sysctl settings + if lines: + _write(conf, lines) + call_throws(ctx, ['sysctl', '--system']) + + def install_base_units(ctx, fsid): # type: (CephadmContext, str) -> None """ @@ -3102,13 +3116,13 @@ class CephContainer: 'run', '--rm', '--ipc=host', + # some containers (ahem, haproxy) override this, but we want a fast + # shutdown always (and, more importantly, a successful exit even if we + # fall back to SIGKILL). + '--stop-signal=SIGTERM', ] if isinstance(self.ctx.container_engine, Podman): - # podman adds the container *name* to /etc/hosts (for 127.0.1.1) - # by default, which makes python's socket.getfqdn() return that - # instead of a valid hostname. - cmd_args.append('--no-hosts') if os.path.exists('/etc/ceph/podman-auth.json'): cmd_args.append('--authfile=/etc/ceph/podman-auth.json') @@ -3966,11 +3980,7 @@ def command_bootstrap(ctx): except PermissionError: raise Error(f'Unable to create {dirname} due to permissions failure. Retry with root, or sudo or preallocate the directory.') - if ctx.config and os.path.exists(ctx.config): - with open(ctx.config) as f: - user_conf = f.read() - else: - user_conf = None + (user_conf, _) = get_config_and_keyring(ctx) if not ctx.skip_prepare_host: command_prepare_host(ctx) @@ -4072,7 +4082,7 @@ def command_bootstrap(ctx): # the mgr (e.g. mgr/cephadm/container_image_prometheus) # they don't seem to be stored if there isn't a mgr yet. # Since re-assimilating the same conf settings should be - # idempotent we can just do it aain here. + # idempotent we can just do it again here. with tempfile.NamedTemporaryFile(buffering=0) as tmp: tmp.write(user_conf.encode('utf-8')) cli(['config', 'assimilate-conf', @@ -4329,8 +4339,8 @@ def command_deploy(ctx): ports=daemon_ports) elif daemon_type == NFSGanesha.daemon_type: - if not ctx.reconfig and not redeploy: - daemon_ports.extend(NFSGanesha.port_map.values()) + if not ctx.reconfig and not redeploy and not daemon_ports: + daemon_ports = list(NFSGanesha.port_map.values()) config, keyring = get_config_and_keyring(ctx) # TODO: extract ganesha uid/gid (997, 994) ? @@ -4489,6 +4499,10 @@ def command_shell(ctx): os.path.join(home, f)) mounts[home] = '/root' + for i in ctx.volume: + a, b = i.split(':', 1) + mounts[a] = b + c = CephContainer( ctx, image=ctx.image, @@ -5507,10 +5521,10 @@ def command_rm_cluster(ctx): _zap_osds(ctx) # rm units - call_throws(ctx, ['rm', '-f', ctx.unit_dir + # noqa: W504 - '/ceph-%s@.service' % ctx.fsid]) - call_throws(ctx, ['rm', '-f', ctx.unit_dir + # noqa: W504 - '/ceph-%s.target' % ctx.fsid]) + call_throws(ctx, ['rm', '-f', ctx.unit_dir + + '/ceph-%s@.service' % ctx.fsid]) + call_throws(ctx, ['rm', '-f', ctx.unit_dir + + '/ceph-%s.target' % ctx.fsid]) call_throws(ctx, ['rm', '-rf', ctx.unit_dir + '/ceph-%s.target.wants' % ctx.fsid]) # rm data @@ -5519,12 +5533,21 @@ def command_rm_cluster(ctx): if not ctx.keep_logs: # rm logs call_throws(ctx, ['rm', '-rf', ctx.log_dir + '/' + ctx.fsid]) - call_throws(ctx, ['rm', '-rf', ctx.log_dir + # noqa: W504 - '/*.wants/ceph-%s@*' % ctx.fsid]) + call_throws(ctx, ['rm', '-rf', ctx.log_dir + + '/*.wants/ceph-%s@*' % ctx.fsid]) # rm logrotate config call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/ceph-%s' % ctx.fsid]) + # rm cephadm logrotate config if last cluster on host + if not os.listdir(ctx.data_dir): + call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/cephadm']) + + # rm sysctl settings + sysctl_dir = Path(ctx.sysctl_dir) + for p in sysctl_dir.glob(f'90-ceph-{ctx.fsid}-*.conf'): + p.unlink() + # clean up config, keyring, and pub key files files = ['/etc/ceph/ceph.conf', '/etc/ceph/ceph.pub', '/etc/ceph/ceph.client.admin.keyring'] @@ -6712,16 +6735,6 @@ def command_gather_facts(ctx: CephadmContext): host = HostFacts(ctx) print(host.dump()) -################################## - - -def command_verify_prereqs(ctx: CephadmContext): - if ctx.service_type == 'haproxy' or ctx.service_type == 'keepalived': - out, err, code = call( - ctx, ['sysctl', '-n', 'net.ipv4.ip_nonlocal_bind'] - ) - if out.strip() != '1': - raise Error('net.ipv4.ip_nonlocal_bind not set to 1') ################################## @@ -6803,12 +6816,14 @@ class CephadmDaemonHandler(BaseHTTPRequestHandler): The token is installed at deployment time and must be provided to ensure we only respond to callers who know our token i.e. mgr """ + def wrapper(self, *args, **kwargs): auth = self.headers.get('Authorization', None) if auth != 'Bearer ' + self.server.token: self.send_error(401) return f(self, *args, **kwargs) + return wrapper def _help_page(self): @@ -7483,6 +7498,10 @@ def _get_parser(): '--logrotate-dir', default=LOGROTATE_DIR, help='location of logrotate configuration files') + parser.add_argument( + '--sysctl-dir', + default=SYSCTL_DIR, + help='location of sysctl configuration files') parser.add_argument( '--unit-dir', default=UNIT_DIR, @@ -7662,6 +7681,11 @@ def _get_parser(): action='append', default=[], help='set environment variable') + parser_shell.add_argument( + '--volume', '-v', + action='append', + default=[], + help='set environment variable') parser_shell.add_argument( 'command', nargs=argparse.REMAINDER, help='command (optional)') @@ -8083,15 +8107,6 @@ def _get_parser(): help='Maintenance action - enter maintenance, or exit maintenance') parser_maintenance.set_defaults(func=command_maintenance) - parser_verify_prereqs = subparsers.add_parser( - 'verify-prereqs', - help='verify system prerequisites for a given service are met on this host') - parser_verify_prereqs.set_defaults(func=command_verify_prereqs) - parser_verify_prereqs.add_argument( - '--daemon-type', - required=True, - help='service type of service to whose prereqs will be checked') - return parser @@ -8135,6 +8150,18 @@ def cephadm_init(args: List[str]) -> Optional[CephadmContext]: dictConfig(logging_config) logger = logging.getLogger() + if not os.path.exists(ctx.logrotate_dir + '/cephadm'): + with open(ctx.logrotate_dir + '/cephadm', 'w') as f: + f.write("""# created by cephadm +/var/log/ceph/cephadm.log { + rotate 7 + daily + compress + missingok + notifempty +} +""") + if ctx.verbose: for handler in logger.handlers: if handler.name == 'console':