diff --git a/SOURCES/cephadm b/SOURCES/cephadm index beb9bf0..524c038 100644 --- a/SOURCES/cephadm +++ b/SOURCES/cephadm @@ -43,14 +43,17 @@ from io import StringIO from threading import Thread, RLock from urllib.error import HTTPError from urllib.request import urlopen +from pathlib import Path # Default container images ----------------------------------------------------- -DEFAULT_IMAGE = 'docker.io/ceph/daemon-base:latest-pacific-devel' +DEFAULT_IMAGE = 'docker.io/ceph/ceph:v16' DEFAULT_IMAGE_IS_MASTER = False +DEFAULT_IMAGE_RELEASE = 'pacific' DEFAULT_PROMETHEUS_IMAGE = 'docker.io/prom/prometheus:v2.18.1' DEFAULT_NODE_EXPORTER_IMAGE = 'docker.io/prom/node-exporter:v0.18.1' DEFAULT_GRAFANA_IMAGE = 'docker.io/ceph/ceph-grafana:6.7.4' DEFAULT_ALERT_MANAGER_IMAGE = 'docker.io/prom/alertmanager:v0.20.0' +DEFAULT_REGISTRY = 'docker.io' # normalize unqualified digests to this # ------------------------------------------------------------------------------ LATEST_STABLE_RELEASE = 'pacific' @@ -62,8 +65,8 @@ UNIT_DIR = '/etc/systemd/system' LOG_DIR_MODE = 0o770 DATA_DIR_MODE = 0o700 CONTAINER_INIT = True -CONTAINER_PREFERENCE = ['podman', 'docker'] # prefer podman to docker MIN_PODMAN_VERSION = (2, 0, 2) +CGROUPS_SPLIT_PODMAN_VERSION = (2, 1, 0) CUSTOM_PS1 = r'[ceph: \u@\h \W]\$ ' DEFAULT_TIMEOUT = None # in seconds DEFAULT_RETRY = 15 @@ -115,7 +118,7 @@ class BaseConfig: self.memory_limit: Optional[int] = None self.container_init: bool = CONTAINER_INIT - self.container_path: str = '' + self.container_engine: Optional[ContainerEngine] = None def set_from_args(self, args: argparse.Namespace): argdict: Dict[str, Any] = vars(args) @@ -157,6 +160,40 @@ class CephadmContext: super().__setattr__(name, value) +class ContainerEngine: + def __init__(self): + self.path = find_program(self.EXE) + + @property + def EXE(self) -> str: + raise NotImplementedError() + + +class Podman(ContainerEngine): + EXE = 'podman' + + def __init__(self): + super().__init__() + self._version = None + + @property + def version(self): + if self._version is None: + raise RuntimeError('Please call `get_version` first') + return self._version + + def get_version(self, ctx: CephadmContext): + out, _, _ = call_throws(ctx, [self.path, 'version', '--format', '{{.Client.Version}}']) + self._version = _parse_podman_version(out) + + +class Docker(ContainerEngine): + EXE = 'docker' + + +CONTAINER_PREFERENCE = (Podman, Docker) # prefer podman to docker + + # Log and console output config logging_config = { 'version': 1, @@ -287,7 +324,7 @@ class Monitoring(object): if daemon_type == 'alertmanager': for cmd in ['alertmanager', 'prometheus-alertmanager']: _, err, code = call(ctx, [ - ctx.container_path, 'exec', container_id, cmd, + ctx.container_engine.path, 'exec', container_id, cmd, '--version' ], verbosity=CallVerbosity.DEBUG) if code == 0: @@ -295,7 +332,7 @@ class Monitoring(object): cmd = 'alertmanager' # reset cmd for version extraction else: _, err, code = call(ctx, [ - ctx.container_path, 'exec', container_id, cmd, '--version' + ctx.container_engine.path, 'exec', container_id, cmd, '--version' ], verbosity=CallVerbosity.DEBUG) if code == 0 and \ err.startswith('%s, version ' % cmd): @@ -385,7 +422,7 @@ class NFSGanesha(object): # type: (CephadmContext, str) -> Optional[str] version = None out, err, code = call(ctx, - [ctx.container_path, 'exec', container_id, + [ctx.container_engine.path, 'exec', container_id, NFSGanesha.entrypoint, '-v'], verbosity=CallVerbosity.DEBUG) if code == 0: @@ -547,7 +584,7 @@ class CephIscsi(object): # type: (CephadmContext, str) -> Optional[str] version = None out, err, code = call(ctx, - [ctx.container_path, 'exec', container_id, + [ctx.container_engine.path, 'exec', container_id, '/usr/bin/python3', '-c', "import pkg_resources; print(pkg_resources.require('ceph_iscsi')[0].version)"], verbosity=CallVerbosity.DEBUG) if code == 0: @@ -768,11 +805,19 @@ class Keepalived(object): envs = [ 'KEEPALIVED_AUTOCONF=false', 'KEEPALIVED_CONF=/etc/keepalived/keepalived.conf', - 'KEEPALIVED_CMD="/usr/sbin/keepalived -n -l -f /etc/keepalived/keepalived.conf"', + 'KEEPALIVED_CMD=/usr/sbin/keepalived -n -l -f /etc/keepalived/keepalived.conf', 'KEEPALIVED_DEBUG=false' ] return envs + @staticmethod + def get_prestart(): + return ( + '# keepalived needs IP forwarding and non-local bind\n' + 'sysctl net.ipv4.ip_forward=1\n' + 'sysctl net.ipv4.ip_nonlocal_bind=1\n' + ) + def extract_uid_gid_keepalived(self): # better directory for this? return extract_uid_gid(self.ctx, file_path='/var/lib') @@ -902,6 +947,15 @@ class CustomContainer(object): ################################## +def touch(file_path: str, uid: Optional[int] = None, gid: Optional[int] = None) -> None: + Path(file_path).touch() + if uid and gid: + os.chown(file_path, uid, gid) + + +################################## + + def dict_get(d: Dict, key: str, default: Any = None, require: bool = False) -> Any: """ Helper function to get a key from a dictionary. @@ -1523,18 +1577,8 @@ def try_convert_datetime(s): return None -def get_podman_version(ctx, container_path): - # type: (CephadmContext, str) -> Tuple[int, ...] - if 'podman' not in container_path: - raise ValueError('not using podman') - out, _, _ = call_throws(ctx, [container_path, '--version']) - return _parse_podman_version(out) - - -def _parse_podman_version(out): +def _parse_podman_version(version_str): # type: (str) -> Tuple[int, ...] - _, _, version_str = out.strip().split() - def to_int(val, org_e=None): if not val and org_e: raise org_e @@ -1688,7 +1732,7 @@ def infer_image(func): if not ctx.image: ctx.image = os.environ.get('CEPHADM_IMAGE') if not ctx.image: - ctx.image = get_last_local_ceph_image(ctx, ctx.container_path) + ctx.image = get_last_local_ceph_image(ctx, ctx.container_engine.path) if not ctx.image: ctx.image = _get_default_image(ctx) return func(ctx) @@ -1921,26 +1965,26 @@ def find_program(filename): return name -def find_container_engine(ctx): - # type: (CephadmContext) -> str +def find_container_engine(ctx: CephadmContext): if ctx.docker: - return find_program('docker') + return Docker() else: for i in CONTAINER_PREFERENCE: try: - return find_program(i) + return i() except Exception as e: - logger.debug('Could not locate %s: %s' % (i, e)) - return '' + logger.debug('Could not locate %s: %s' % (i.EXE, e)) + return None def check_container_engine(ctx): # type: (CephadmContext) -> None - engine = os.path.basename(ctx.container_path) if ctx.container_path else None - if engine not in CONTAINER_PREFERENCE: - raise Error('Unable to locate any of %s' % CONTAINER_PREFERENCE) - elif engine == 'podman': - if get_podman_version(ctx, ctx.container_path) < MIN_PODMAN_VERSION: + engine = ctx.container_engine + if not isinstance(engine, CONTAINER_PREFERENCE): + raise Error('Unable to locate any of %s' % [i.EXE for i in CONTAINER_PREFERENCE]) + elif isinstance(engine, Podman): + engine.get_version(ctx) + if engine.version < MIN_PODMAN_VERSION: raise Error('podman version %d.%d.%d or later is required' % MIN_PODMAN_VERSION) @@ -2018,7 +2062,7 @@ def check_units(ctx, units, enabler=None): def is_container_running(ctx: CephadmContext, name: str) -> bool: out, err, ret = call_throws(ctx, [ - ctx.container_path, 'ps', + ctx.container_engine.path, 'ps', '--format', '{{.Names}}']) return name in out @@ -2139,6 +2183,7 @@ def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid, makedirs(os.path.join(data_dir_root, config_dir, 'certs'), uid, gid, 0o755) makedirs(os.path.join(data_dir_root, config_dir, 'provisioning/datasources'), uid, gid, 0o755) makedirs(os.path.join(data_dir_root, 'data'), uid, gid, 0o755) + touch(os.path.join(data_dir_root, 'data', 'grafana.db'), uid, gid) elif daemon_type == 'alertmanager': data_dir_root = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) @@ -2219,14 +2264,20 @@ def get_config_and_keyring(ctx): keyring = d.get('keyring') if 'config' in ctx and ctx.config: - with open(ctx.config, 'r') as f: - config = f.read() + try: + with open(ctx.config, 'r') as f: + config = f.read() + except FileNotFoundError: + raise Error('config file: %s does not exist' % ctx.config) if 'key' in ctx and ctx.key: keyring = '[%s]\n\tkey = %s\n' % (ctx.name, ctx.key) elif 'keyring' in ctx and ctx.keyring: - with open(ctx.keyring, 'r') as f: - keyring = f.read() + try: + with open(ctx.keyring, 'r') as f: + keyring = f.read() + except FileNotFoundError: + raise Error('keyring file: %s does not exist' % ctx.keyring) return config, keyring @@ -2276,19 +2327,20 @@ def get_container_mounts(ctx, fsid, daemon_type, daemon_id, # these do not search for their keyrings in a data directory mounts[data_dir + '/keyring'] = '/etc/ceph/ceph.client.%s.%s.keyring' % (daemon_type, daemon_id) - if daemon_type in ['mon', 'osd']: + if daemon_type in ['mon', 'osd', 'clusterless-ceph-volume']: mounts['/dev'] = '/dev' # FIXME: narrow this down? mounts['/run/udev'] = '/run/udev' - if daemon_type == 'osd': + if daemon_type in ['osd', 'clusterless-ceph-volume']: mounts['/sys'] = '/sys' # for numa.cc, pick_address, cgroups, ... + mounts['/run/lvm'] = '/run/lvm' + mounts['/run/lock/lvm'] = '/run/lock/lvm' + if daemon_type == 'osd': # selinux-policy in the container may not match the host. if HostFacts(ctx).selinux_enabled: selinux_folder = '/var/lib/ceph/%s/selinux' % fsid if not os.path.exists(selinux_folder): os.makedirs(selinux_folder, mode=0o755) mounts[selinux_folder] = '/sys/fs/selinux:ro' - mounts['/run/lvm'] = '/run/lvm' - mounts['/run/lock/lvm'] = '/run/lock/lvm' try: if ctx.shared_ceph_folder: # make easy manager modules/ceph-volume development @@ -2319,6 +2371,7 @@ def get_container_mounts(ctx, fsid, daemon_type, daemon_id, mounts[os.path.join(data_dir, 'etc/grafana/grafana.ini')] = '/etc/grafana/grafana.ini:Z' mounts[os.path.join(data_dir, 'etc/grafana/provisioning/datasources')] = '/etc/grafana/provisioning/datasources:Z' mounts[os.path.join(data_dir, 'etc/grafana/certs')] = '/etc/grafana/certs:Z' + mounts[os.path.join(data_dir, 'data/grafana.db')] = '/var/lib/grafana/grafana.db:Z' elif daemon_type == 'alertmanager': mounts[os.path.join(data_dir, 'etc/alertmanager')] = '/etc/alertmanager:Z' @@ -2395,7 +2448,7 @@ def get_container(ctx: CephadmContext, elif daemon_type == Keepalived.daemon_type: name = '%s.%s' % (daemon_type, daemon_id) envs.extend(Keepalived.get_container_envs()) - container_args.extend(['--cap-add NET_ADMIN']) + container_args.extend(['--cap-add=NET_ADMIN', '--cap-add=NET_RAW']) elif daemon_type == CephIscsi.daemon_type: entrypoint = CephIscsi.entrypoint name = '%s.%s' % (daemon_type, daemon_id) @@ -2425,7 +2478,7 @@ def get_container(ctx: CephadmContext, # if using podman, set -d, --conmon-pidfile & --cidfile flags # so service can have Type=Forking - if 'podman' in ctx.container_path: + if isinstance(ctx.container_engine, Podman): runtime_dir = '/run' container_args.extend([ '-d', '--log-driver', 'journald', @@ -2434,6 +2487,8 @@ def get_container(ctx: CephadmContext, '--cidfile', runtime_dir + '/ceph-%s@%s.%s.service-cid' % (fsid, daemon_type, daemon_id), ]) + if ctx.container_engine.version >= CGROUPS_SPLIT_PODMAN_VERSION: + container_args.append('--cgroups=split') return CephContainer( ctx, @@ -2486,7 +2541,14 @@ def deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid, ports = ports or [] if any([port_in_use(ctx, port) for port in ports]): - raise Error("TCP Port(s) '{}' required for {} already in use".format(','.join(map(str, ports)), daemon_type)) + if daemon_type == 'mgr': + # non-fatal for mgr when we are in mgr_standby_modules=false, but we can't + # tell whether that is the case here. + logger.warning( + f"ceph-mgr TCP port(s) {','.join(map(str, ports))} already in use" + ) + else: + raise Error("TCP Port(s) '{}' required for {} already in use".format(','.join(map(str, ports)), daemon_type)) data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) if reconfig and not os.path.exists(data_dir): @@ -2592,7 +2654,7 @@ def _write_container_cmd_to_bash(ctx, file_obj, container, comment=None, backgro # Sometimes, adding `--rm` to a run_cmd doesn't work. Let's remove the container manually file_obj.write('! ' + ' '.join(container.rm_cmd()) + ' 2> /dev/null\n') # Sometimes, `podman rm` doesn't find the container. Then you'll have to add `--storage` - if 'podman' in ctx.container_path: + if isinstance(ctx.container_engine, Podman): file_obj.write( '! ' + ' '.join([shlex.quote(a) for a in container.rm_cmd(storage=True)]) @@ -2666,6 +2728,8 @@ def deploy_daemon_units( ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id) tcmu_container = ceph_iscsi.get_tcmu_runner_container() _write_container_cmd_to_bash(ctx, f, tcmu_container, 'iscsi tcmu-runnter container', background=True) + elif daemon_type == Keepalived.daemon_type: + f.write(Keepalived.get_prestart()) _write_container_cmd_to_bash(ctx, f, c, '%s.%s' % (daemon_type, str(daemon_id))) @@ -2944,13 +3008,15 @@ def install_base_units(ctx, fsid): def get_unit_file(ctx, fsid): # type: (CephadmContext, str) -> str extra_args = '' - if 'podman' in ctx.container_path: - extra_args = ('ExecStartPre=-/bin/rm -f /%t/%n-pid /%t/%n-cid\n' - 'ExecStopPost=-/bin/rm -f /%t/%n-pid /%t/%n-cid\n' + if isinstance(ctx.container_engine, Podman): + extra_args = ('ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid\n' + 'ExecStopPost=-/bin/rm -f %t/%n-pid %t/%n-cid\n' 'Type=forking\n' - 'PIDFile=/%t/%n-pid\n') + 'PIDFile=%t/%n-pid\n') + if ctx.container_engine.version >= CGROUPS_SPLIT_PODMAN_VERSION: + extra_args += 'Delegate=yes\n' - docker = 'docker' in ctx.container_path + docker = isinstance(ctx.container_engine, Docker) u = """# generated by cephadm [Unit] Description=Ceph %i for {fsid} @@ -2983,7 +3049,7 @@ StartLimitBurst=5 {extra_args} [Install] WantedBy=ceph-{fsid}.target -""".format(container_path=ctx.container_path, +""".format(container_path=ctx.container_engine.path, fsid=fsid, data_dir=ctx.data_dir, extra_args=extra_args, @@ -3032,15 +3098,19 @@ class CephContainer: def run_cmd(self) -> List[str]: cmd_args: List[str] = [ - str(self.ctx.container_path), + str(self.ctx.container_engine.path), 'run', '--rm', '--ipc=host', ] - if 'podman' in self.ctx.container_path and \ - os.path.exists('/etc/ceph/podman-auth.json'): - cmd_args.append('--authfile=/etc/ceph/podman-auth.json') + if isinstance(self.ctx.container_engine, Podman): + # podman adds the container *name* to /etc/hosts (for 127.0.1.1) + # by default, which makes python's socket.getfqdn() return that + # instead of a valid hostname. + cmd_args.append('--no-hosts') + if os.path.exists('/etc/ceph/podman-auth.json'): + cmd_args.append('--authfile=/etc/ceph/podman-auth.json') envs: List[str] = [ '-e', 'CONTAINER_IMAGE=%s' % self.image, @@ -3091,7 +3161,7 @@ class CephContainer: def shell_cmd(self, cmd: List[str]) -> List[str]: cmd_args: List[str] = [ - str(self.ctx.container_path), + str(self.ctx.container_engine.path), 'run', '--rm', '--ipc=host', @@ -3105,6 +3175,8 @@ class CephContainer: if self.host_network: cmd_args.append('--net=host') + if self.ctx.no_hosts: + cmd_args.append('--no-hosts') if self.privileged: cmd_args.extend([ '--privileged', @@ -3132,7 +3204,7 @@ class CephContainer: def exec_cmd(self, cmd): # type: (List[str]) -> List[str] return [ - str(self.ctx.container_path), + str(self.ctx.container_engine.path), 'exec', ] + self.container_args + [ self.cname, @@ -3141,7 +3213,7 @@ class CephContainer: def rm_cmd(self, storage=False): # type: (bool) -> List[str] ret = [ - str(self.ctx.container_path), + str(self.ctx.container_engine.path), 'rm', '-f', ] if storage: @@ -3152,7 +3224,7 @@ class CephContainer: def stop_cmd(self): # type () -> List[str] ret = [ - str(self.ctx.container_path), + str(self.ctx.container_engine.path), 'stop', self.cname, ] return ret @@ -3196,8 +3268,8 @@ def _pull_image(ctx, image): 'Digest did not match, expected', ] - cmd = [ctx.container_path, 'pull', image] - if 'podman' in ctx.container_path and os.path.exists('/etc/ceph/podman-auth.json'): + cmd = [ctx.container_engine.path, 'pull', image] + if isinstance(ctx.container_engine, Podman) and os.path.exists('/etc/ceph/podman-auth.json'): cmd.append('--authfile=/etc/ceph/podman-auth.json') cmd_str = ' '.join(cmd) @@ -3221,7 +3293,7 @@ def _pull_image(ctx, image): def command_inspect_image(ctx): # type: (CephadmContext) -> int out, err, ret = call_throws(ctx, [ - ctx.container_path, 'inspect', + ctx.container_engine.path, 'inspect', '--format', '{{.ID}},{{.RepoDigests}}', ctx.image]) if ret: @@ -3235,6 +3307,20 @@ def command_inspect_image(ctx): return 0 +def normalize_image_digest(digest): + # normal case: + # ceph/ceph -> docker.io/ceph/ceph + # edge cases that shouldn't ever come up: + # ubuntu -> docker.io/ubuntu (ubuntu alias for library/ubuntu) + # no change: + # quay.ceph.io/ceph/ceph -> ceph + # docker.io/ubuntu -> no change + bits = digest.split('/') + if '.' not in bits[0] or len(bits) < 3: + digest = DEFAULT_REGISTRY + '/' + digest + return digest + + def get_image_info_from_inspect(out, image): # type: (str, str) -> Dict[str, Union[str,List[str]]] image_id, digests = out.split(',', 1) @@ -3244,7 +3330,7 @@ def get_image_info_from_inspect(out, image): 'image_id': normalize_container_id(image_id) } # type: Dict[str, Union[str,List[str]]] if digests: - r['repo_digests'] = digests[1:-1].split(' ') + r['repo_digests'] = list(map(normalize_image_digest, digests[1:-1].split(' '))) return r ################################## @@ -3362,7 +3448,10 @@ def prepare_mon_addresses( if not ctx.skip_mon_network: # make sure IP is configured locally, and then figure out the # CIDR network - for net, ips in list_networks(ctx).items(): + for net, ifaces in list_networks(ctx).items(): + ips: List[str] = [] + for iface, ls in ifaces.items(): + ips.extend(ls) if ipaddress.ip_address(unwrap_ipv6(base_ip)) in \ [ipaddress.ip_address(ip) for ip in ips]: mon_network = net @@ -3591,14 +3680,7 @@ def prepare_ssh( cli: Callable, wait_for_mgr_restart: Callable ) -> None: - cli(['config-key', 'set', 'mgr/cephadm/ssh_user', ctx.ssh_user]) - - logger.info('Enabling cephadm module...') - cli(['mgr', 'module', 'enable', 'cephadm']) - wait_for_mgr_restart() - - logger.info('Setting orchestrator backend to cephadm...') - cli(['orch', 'set', 'backend', 'cephadm']) + cli(['cephadm', 'set-user', ctx.ssh_user]) if ctx.ssh_config: logger.info('Using provided ssh config...') @@ -3657,7 +3739,10 @@ def prepare_ssh( host = get_hostname() logger.info('Adding host %s...' % host) try: - cli(['orch', 'host', 'add', host]) + args = ['orch', 'host', 'add', host] + if ctx.mon_ip: + args.append(ctx.mon_ip) + cli(args) except RuntimeError as e: raise Error('Failed to add host <%s>: %s' % (host, e)) @@ -3681,6 +3766,17 @@ def prepare_ssh( cli(['orch', 'apply', t]) +def enable_cephadm_mgr_module( + cli: Callable, wait_for_mgr_restart: Callable +) -> None: + + logger.info('Enabling cephadm module...') + cli(['mgr', 'module', 'enable', 'cephadm']) + wait_for_mgr_restart() + logger.info('Setting orchestrator backend to cephadm...') + cli(['orch', 'set', 'backend', 'cephadm']) + + def prepare_dashboard( ctx: CephadmContext, uid: int, gid: int, @@ -3744,8 +3840,40 @@ def prepare_bootstrap_config( if not cp.has_section('global'): cp.add_section('global') cp.set('global', 'fsid', fsid) - cp.set('global', 'mon host', mon_addr) + cp.set('global', 'mon_host', mon_addr) cp.set('global', 'container_image', image) + + if not cp.has_section('mon'): + cp.add_section('mon') + if ( + not cp.has_option('mon', 'auth_allow_insecure_global_id_reclaim') + and not cp.has_option('mon', 'auth allow insecure global id reclaim') + ): + cp.set('mon', 'auth_allow_insecure_global_id_reclaim', 'false') + + if ctx.single_host_defaults: + logger.info('Adjusting default settings to suit single-host cluster...') + # replicate across osds, not hosts + if ( + not cp.has_option('global', 'osd_crush_choose_leaf_type') + and not cp.has_option('global', 'osd crush choose leaf type') + ): + cp.set('global', 'osd_crush_choose_leaf_type', '0') + # replica 2x + if ( + not cp.has_option('global', 'osd_pool_default_size') + and not cp.has_option('global', 'osd pool default size') + ): + cp.set('global', 'osd_pool_default_size', '2') + # disable mgr standby modules (so we can colocate multiple mgrs on one host) + if not cp.has_section('mgr'): + cp.add_section('mgr') + if ( + not cp.has_option('mgr', 'mgr_standby_modules') + and not cp.has_option('mgr', 'mgr standby modules') + ): + cp.set('mgr', 'mgr_standby_modules', 'false') + cpf = StringIO() cp.write(cpf) config = cpf.getvalue() @@ -3753,9 +3881,6 @@ def prepare_bootstrap_config( if ctx.registry_json or ctx.registry_url: command_registry_login(ctx) - if not ctx.skip_pull: - _pull_image(ctx, image) - return config @@ -3816,8 +3941,6 @@ def finish_bootstrap_config( def command_bootstrap(ctx): # type: (CephadmContext) -> int - host: Optional[str] = None - if not ctx.output_config: ctx.output_config = os.path.join(ctx.output_dir, 'ceph.conf') if not ctx.output_keyring: @@ -3843,6 +3966,12 @@ def command_bootstrap(ctx): except PermissionError: raise Error(f'Unable to create {dirname} due to permissions failure. Retry with root, or sudo or preallocate the directory.') + if ctx.config and os.path.exists(ctx.config): + with open(ctx.config) as f: + user_conf = f.read() + else: + user_conf = None + if not ctx.skip_prepare_host: command_prepare_host(ctx) else: @@ -3865,6 +3994,20 @@ def command_bootstrap(ctx): config = prepare_bootstrap_config(ctx, fsid, addr_arg, ctx.image) + if not ctx.skip_pull: + _pull_image(ctx, ctx.image) + + image_ver = CephContainer(ctx, ctx.image, 'ceph', ['--version']).run().strip() + logger.info(f'Ceph version: {image_ver}') + image_release = image_ver.split()[4] + if ( + not ctx.allow_mismatched_release + and image_release not in [DEFAULT_IMAGE_RELEASE, LATEST_STABLE_RELEASE] + ): + raise Error( + f'Container release {image_release} != cephadm release {DEFAULT_IMAGE_RELEASE}; please use matching version of cephadm (pass --allow-mismatched-release to continue anyway)' + ) + logger.info('Extracting ceph user uid/gid from container image...') (uid, gid) = extract_uid_gid(ctx) @@ -3923,16 +4066,37 @@ def command_bootstrap(ctx): # create mgr create_mgr(ctx, uid, gid, fsid, mgr_id, mgr_key, config, cli) + if user_conf: + # user given config settings were already assimilated earlier + # but if the given settings contained any attributes in + # the mgr (e.g. mgr/cephadm/container_image_prometheus) + # they don't seem to be stored if there isn't a mgr yet. + # Since re-assimilating the same conf settings should be + # idempotent we can just do it aain here. + with tempfile.NamedTemporaryFile(buffering=0) as tmp: + tmp.write(user_conf.encode('utf-8')) + cli(['config', 'assimilate-conf', + '-i', '/var/lib/ceph/user.conf'], + {tmp.name: '/var/lib/ceph/user.conf:z'}) + + def json_loads_retry(cli_func): + for sleep_secs in [1, 4, 4]: + try: + return json.loads(cli_func()) + except json.JSONDecodeError: + logger.debug('Invalid JSON. Retrying in %s seconds...' % sleep_secs) + time.sleep(sleep_secs) + return json.loads(cli_func()) + # wait for mgr to restart (after enabling a module) def wait_for_mgr_restart(): # first get latest mgrmap epoch from the mon. try newer 'mgr # stat' command first, then fall back to 'mgr dump' if # necessary try: - out = cli(['mgr', 'stat']) + j = json_loads_retry(lambda: cli(['mgr', 'stat'])) except Exception: - out = cli(['mgr', 'dump']) - j = json.loads(out) + j = json_loads_retry(lambda: cli(['mgr', 'dump'])) epoch = j['epoch'] # wait for mgr to have it @@ -3949,8 +4113,9 @@ def command_bootstrap(ctx): return False is_available(ctx, 'mgr epoch %d' % epoch, mgr_has_latest_epoch) + enable_cephadm_mgr_module(cli, wait_for_mgr_restart) + # ssh - host = None if not ctx.skip_ssh: prepare_ssh(ctx, cli, wait_for_mgr_restart) @@ -3985,6 +4150,14 @@ def command_bootstrap(ctx): if not ctx.skip_dashboard: prepare_dashboard(ctx, uid, gid, cli, wait_for_mgr_restart) + if ctx.output_config == '/etc/ceph/ceph.conf' and not ctx.skip_admin_label: + logger.info('Enabling client.admin keyring and conf on hosts with "admin" label') + try: + cli(['orch', 'client-keyring', 'set', 'client.admin', 'label:_admin']) + cli(['orch', 'host', 'label', 'add', get_hostname(), '_admin']) + except Exception: + logger.info('Unable to set up "admin" label; assuming older version of Ceph') + if ctx.apply_spec: logger.info('Applying %s to cluster' % ctx.apply_spec) @@ -3993,13 +4166,13 @@ def command_bootstrap(ctx): if 'hostname:' in line: line = line.replace('\n', '') split = line.split(': ') - if split[1] != host: + if split[1] != hostname: logger.info('Adding ssh key to %s' % split[1]) ssh_key = '/etc/ceph/ceph.pub' if ctx.ssh_public_key: ssh_key = ctx.ssh_public_key.name - out, err, code = call_throws(ctx, ['ssh-copy-id', '-f', '-i', ssh_key, '%s@%s' % (ctx.ssh_user, split[1])]) + out, err, code = call_throws(ctx, ['sudo', '-u', ctx.ssh_user, 'ssh-copy-id', '-f', '-i', ssh_key, '-o StrictHostKeyChecking=no', '%s@%s' % (ctx.ssh_user, split[1])]) mounts = {} mounts[pathify(ctx.apply_spec)] = '/tmp/spec.yml:z' @@ -4052,14 +4225,14 @@ def command_registry_login(ctx: CephadmContext): def registry_login(ctx: CephadmContext, url, username, password): logger.info('Logging into custom registry.') try: - container_path = ctx.container_path - cmd = [container_path, 'login', + engine = ctx.container_engine + cmd = [engine.path, 'login', '-u', username, '-p', password, url] - if 'podman' in container_path: + if isinstance(engine, Podman): cmd.append('--authfile=/etc/ceph/podman-auth.json') out, _, _ = call_throws(ctx, cmd) - if 'podman' in container_path: + if isinstance(engine, Podman): os.chmod('/etc/ceph/podman-auth.json', 0o600) except Exception: raise Error('Failed to login to custom registry @ %s as %s with given password' % (ctx.registry_url, ctx.registry_username)) @@ -4237,11 +4410,25 @@ def command_run(ctx): ################################## +def fsid_conf_mismatch(ctx): + # type: (CephadmContext) -> bool + (config, _) = get_config_and_keyring(ctx) + if config: + for c in config.split('\n'): + if 'fsid = ' in c.strip(): + if 'fsid = ' + ctx.fsid != c.strip(): + return True + return False + + @infer_fsid @infer_config @infer_image def command_shell(ctx): # type: (CephadmContext) -> int + if fsid_conf_mismatch(ctx): + raise Error('fsid does not match ceph conf') + if ctx.fsid: make_log_dir(ctx, ctx.fsid) if ctx.name: @@ -4263,7 +4450,7 @@ def command_shell(ctx): if not ctx.keyring and os.path.exists(SHELL_DEFAULT_KEYRING): ctx.keyring = SHELL_DEFAULT_KEYRING - container_args = [] # type: List[str] + container_args: List[str] = ['-i'] mounts = get_container_mounts(ctx, ctx.fsid, daemon_type, daemon_id, no_config=True if ctx.config else False) binds = get_container_binds(ctx, ctx.fsid, daemon_type, daemon_id) @@ -4286,7 +4473,7 @@ def command_shell(ctx): else: command = ['bash'] container_args += [ - '-it', + '-t', '-e', 'LANG=C', '-e', 'PS1=%s' % CUSTOM_PS1, ] @@ -4325,13 +4512,13 @@ def command_enter(ctx): if not ctx.fsid: raise Error('must pass --fsid to specify cluster') (daemon_type, daemon_id) = ctx.name.split('.', 1) - container_args = [] # type: List[str] + container_args = ['-i'] # type: List[str] if ctx.command: command = ctx.command else: command = ['sh'] container_args += [ - '-it', + '-t', '-e', 'LANG=C', '-e', 'PS1=%s' % CUSTOM_PS1, ] @@ -4385,8 +4572,8 @@ def command_ceph_volume(ctx): privileged=True, volume_mounts=mounts, ) - verbosity = CallVerbosity.VERBOSE if ctx.log_output else CallVerbosity.VERBOSE_ON_FAILURE - out, err, code = call_throws(ctx, c.run_cmd(), verbosity=verbosity) + + out, err, code = call_throws(ctx, c.run_cmd()) if not code: print(out) @@ -4434,7 +4621,7 @@ def command_logs(ctx): def list_networks(ctx): - # type: (CephadmContext) -> Dict[str,List[str]] + # type: (CephadmContext) -> Dict[str,Dict[str,List[str]]] # sadly, 18.04's iproute2 4.15.0-2ubun doesn't support the -j flag, # so we'll need to use a regex to parse 'ip' command output. @@ -4457,17 +4644,20 @@ def _list_ipv4_networks(ctx: CephadmContext): def _parse_ipv4_route(out): - r = {} # type: Dict[str,List[str]] - p = re.compile(r'^(\S+) (.*)scope link (.*)src (\S+)') + r = {} # type: Dict[str,Dict[str,List[str]]] + p = re.compile(r'^(\S+) dev (\S+) (.*)scope link (.*)src (\S+)') for line in out.splitlines(): m = p.findall(line) if not m: continue net = m[0][0] - ip = m[0][3] + iface = m[0][1] + ip = m[0][4] if net not in r: - r[net] = [] - r[net].append(ip) + r[net] = {} + if iface not in r[net]: + r[net][iface] = [] + r[net][iface].append(ip) return r @@ -4481,27 +4671,39 @@ def _list_ipv6_networks(ctx: CephadmContext): def _parse_ipv6_route(routes, ips): - r = {} # type: Dict[str,List[str]] + r = {} # type: Dict[str,Dict[str,List[str]]] route_p = re.compile(r'^(\S+) dev (\S+) proto (\S+) metric (\S+) .*pref (\S+)$') ip_p = re.compile(r'^\s+inet6 (\S+)/(.*)scope (.*)$') + iface_p = re.compile(r'^(\d+): (\S+): (.*)$') for line in routes.splitlines(): m = route_p.findall(line) if not m or m[0][0].lower() == 'default': continue net = m[0][0] + if '/' not in net: # only consider networks with a mask + continue + iface = m[0][1] if net not in r: - r[net] = [] + r[net] = {} + if iface not in r[net]: + r[net][iface] = [] + iface = None for line in ips.splitlines(): m = ip_p.findall(line) if not m: + m = iface_p.findall(line) + if m: + # drop @... suffix, if present + iface = m[0][1].split('@')[0] continue ip = m[0][0] # find the network it belongs to net = [n for n in r.keys() if ipaddress.ip_address(ip) in ipaddress.ip_network(n)] if net: - r[net[0]].append(ip) + assert(iface) + r[net[0]][iface].append(ip) return r @@ -4546,7 +4748,7 @@ def list_daemons(ctx, detail=True, legacy_dir=None): # type: (CephadmContext, bool, Optional[str]) -> List[Dict[str, str]] host_version: Optional[str] = None ls = [] - container_path = ctx.container_path + container_path = ctx.container_engine.path data_dir = ctx.data_dir if legacy_dir is not None: @@ -4659,7 +4861,9 @@ def list_daemons(ctx, detail=True, legacy_dir=None): ], verbosity=CallVerbosity.DEBUG) if not code: - image_digests = out.strip()[1:-1].split(' ') + image_digests = list(set(map( + normalize_image_digest, + out.strip()[1:-1].split(' ')))) seen_digests[image_id] = image_digests # identify software version inside the container (if we can) @@ -4710,6 +4914,8 @@ def list_daemons(ctx, detail=True, legacy_dir=None): if not code and \ err.startswith('Keepalived '): version = err.split(' ')[1] + if version[0] == 'v': + version = version[1:] seen_versions[image_id] = version elif daemon_type == CustomContainer.daemon_type: # Because a custom container can contain @@ -5198,6 +5404,68 @@ def command_rm_daemon(ctx): ################################## +def _zap(ctx, what): + mounts = get_container_mounts(ctx, ctx.fsid, 'clusterless-ceph-volume', None) + c = CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/sbin/ceph-volume', + envs=ctx.env, + args=['lvm', 'zap', '--destroy', what], + privileged=True, + volume_mounts=mounts, + ) + logger.info(f'Zapping {what}...') + out, err, code = call_throws(ctx, c.run_cmd()) + + +@infer_image +def _zap_osds(ctx): + # assume fsid lock already held + + # list + mounts = get_container_mounts(ctx, ctx.fsid, 'clusterless-ceph-volume', None) + c = CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/sbin/ceph-volume', + envs=ctx.env, + args=['inventory', '--format', 'json'], + privileged=True, + volume_mounts=mounts, + ) + out, err, code = call_throws(ctx, c.run_cmd()) + if code: + raise Error('failed to list osd inventory') + try: + ls = json.loads(out) + except ValueError as e: + raise Error(f'Invalid JSON in ceph-volume inventory: {e}') + + for i in ls: + matches = [lv.get('cluster_fsid') == ctx.fsid for lv in i.get('lvs', [])] + if any(matches) and all(matches): + _zap(ctx, i.get('path')) + elif any(matches): + lv_names = [lv['name'] for lv in i.get('lvs', [])] + # TODO: we need to map the lv_names back to device paths (the vg + # id isn't part of the output here!) + logger.warning(f'Not zapping LVs (not implemented): {lv_names}') + + +def command_zap_osds(ctx): + if not ctx.force: + raise Error('must pass --force to proceed: ' + 'this command may destroy precious data!') + + lock = FileLock(ctx, ctx.fsid) + lock.acquire() + + _zap_osds(ctx) + +################################## + + def command_rm_cluster(ctx): # type: (CephadmContext) -> None if not ctx.force: @@ -5234,6 +5502,10 @@ def command_rm_cluster(ctx): call(ctx, ['systemctl', 'stop', slice_name], verbosity=CallVerbosity.DEBUG) + # osds? + if ctx.zap_osds: + _zap_osds(ctx) + # rm units call_throws(ctx, ['rm', '-f', ctx.unit_dir + # noqa: W504 '/ceph-%s@.service' % ctx.fsid]) @@ -5243,10 +5515,13 @@ def command_rm_cluster(ctx): ctx.unit_dir + '/ceph-%s.target.wants' % ctx.fsid]) # rm data call_throws(ctx, ['rm', '-rf', ctx.data_dir + '/' + ctx.fsid]) - # rm logs - call_throws(ctx, ['rm', '-rf', ctx.log_dir + '/' + ctx.fsid]) - call_throws(ctx, ['rm', '-rf', ctx.log_dir + # noqa: W504 - '/*.wants/ceph-%s@*' % ctx.fsid]) + + if not ctx.keep_logs: + # rm logs + call_throws(ctx, ['rm', '-rf', ctx.log_dir + '/' + ctx.fsid]) + call_throws(ctx, ['rm', '-rf', ctx.log_dir + # noqa: W504 + '/*.wants/ceph-%s@*' % ctx.fsid]) + # rm logrotate config call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/ceph-%s' % ctx.fsid]) @@ -5263,6 +5538,7 @@ def command_rm_cluster(ctx): if os.path.exists(files[n]): os.remove(files[n]) + ################################## @@ -5283,7 +5559,7 @@ def check_time_sync(ctx, enabler=None): def command_check_host(ctx: CephadmContext) -> None: - container_path = ctx.container_path + container_path = ctx.container_engine.path errors = [] commands = ['systemctl', 'lvcreate'] @@ -5598,6 +5874,7 @@ class YumDnf(Packager): 'centos': ('centos', 'el'), 'rhel': ('centos', 'el'), 'scientific': ('centos', 'el'), + 'rocky': ('centos', 'el'), 'fedora': ('fedora', 'fc'), } @@ -6354,6 +6631,8 @@ class HostFacts(): security['description'] = 'AppArmor: Enabled' try: profiles = read_file(['/sys/kernel/security/apparmor/profiles']) + if len(profiles) == 0: + return {} except OSError: pass else: @@ -6803,7 +7082,6 @@ class CephadmDaemon(): # expects to use self.ctx.command = 'inventory --format=json'.split() self.ctx.fsid = self.fsid - self.ctx.log_output = False ctr = 0 exception_encountered = False @@ -6996,7 +7274,7 @@ class CephadmDaemon(): @property def unit_file(self): - docker = 'docker' in self.ctx.container_path + docker = isinstance(self.ctx.container_engine, Docker) return """#generated by cephadm [Unit] Description=cephadm exporter service for cluster {fsid} @@ -7336,6 +7614,14 @@ def _get_parser(): '--force', action='store_true', help='proceed, even though this may destroy valuable data') + parser_rm_cluster.add_argument( + '--keep-logs', + action='store_true', + help='do not remove log files') + parser_rm_cluster.add_argument( + '--zap-osds', + action='store_true', + help='zap OSD devices for this cluster') parser_run = subparsers.add_parser( 'run', help='run a ceph daemon, in a container, in the foreground') @@ -7379,6 +7665,10 @@ def _get_parser(): parser_shell.add_argument( 'command', nargs=argparse.REMAINDER, help='command (optional)') + parser_shell.add_argument( + '--no-hosts', + action='store_true', + help='dont pass /etc/hosts through to the container') parser_enter = subparsers.add_parser( 'enter', help='run an interactive shell inside a running daemon container') @@ -7410,14 +7700,21 @@ def _get_parser(): '--keyring', '-k', help='ceph.keyring to pass through to the container') parser_ceph_volume.add_argument( - '--log-output', - action='store_true', - default=True, - help='suppress ceph volume output from the log') - parser_ceph_volume.add_argument( 'command', nargs=argparse.REMAINDER, help='command') + parser_zap_osds = subparsers.add_parser( + 'zap-osds', help='zap all OSDs associated with a particular fsid') + parser_zap_osds.set_defaults(func=command_zap_osds) + parser_zap_osds.add_argument( + '--fsid', + required=True, + help='cluster FSID') + parser_zap_osds.add_argument( + '--force', + action='store_true', + help='proceed, even though this may destroy valuable data') + parser_unit = subparsers.add_parser( 'unit', help="operate on the daemon's systemd unit") parser_unit.set_defaults(func=command_unit) @@ -7483,6 +7780,10 @@ def _get_parser(): '--output-pub-ssh-key', help="location to write the cluster's public SSH key") parser_bootstrap.add_argument( + '--skip-admin-label', + action='store_true', + help='do not create admin label for ceph.conf and client.admin keyring distribution') + parser_bootstrap.add_argument( '--skip-ssh', action='store_true', help='skip setup of ssh key on local host') @@ -7561,6 +7862,10 @@ def _get_parser(): action='store_true', help='allow hostname that is fully-qualified (contains ".")') parser_bootstrap.add_argument( + '--allow-mismatched-release', + action='store_true', + help="allow bootstrap of ceph that doesn't match this version of cephadm") + parser_bootstrap.add_argument( '--skip-prepare-host', action='store_true', help='Do not prepare host') @@ -7609,6 +7914,10 @@ def _get_parser(): parser_bootstrap.add_argument( '--cluster-network', help='subnet to use for cluster replication, recovery and heartbeats (in CIDR notation network/mask)') + parser_bootstrap.add_argument( + '--single-host-defaults', + action='store_true', + help='adjust configuration defaults to suit a single-host cluster') parser_deploy = subparsers.add_parser( 'deploy', help='deploy a daemon') @@ -7854,7 +8163,7 @@ def main(): try: # podman or docker? - ctx.container_path = find_container_engine(ctx) + ctx.container_engine = find_container_engine(ctx) if ctx.func not in \ [command_check_host, command_prepare_host, command_add_repo]: check_container_engine(ctx) diff --git a/SPECS/cephadm.spec.in b/SPECS/cephadm.spec.in index 9749c13..dec43bc 100644 --- a/SPECS/cephadm.spec.in +++ b/SPECS/cephadm.spec.in @@ -1,14 +1,14 @@ # Upstream ceph commit upon which this package is based: -# patches_base=3eb70cf622aace689e45749e8a92fce033d3d55c +# patches_base=d3d607667e58de33790b10a32b08f2b29d5aa567 Name: cephadm Epoch: 2 -Version: 16.1.0 -Release: 568%{?dist} +Version: 16.2.4 +Release: 2%{?dist} Summary: Utility to bootstrap Ceph clusters License: LGPL-2.1 URL: https://ceph.io -Source0: https://github.com/ceph/ceph/raw/3eb70cf622aace689e45749e8a92fce033d3d55c/src/cephadm/cephadm +Source0: https://github.com/ceph/ceph/raw/d3d607667e58de33790b10a32b08f2b29d5aa567/src/cephadm/cephadm Source1: COPYING-LGPL2.1 BuildArch: noarch