From a38601968ccd8c8dfdce60c8d66b220eefb344b0 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 28 Mar 2017 18:41:08 +0200 Subject: [PATCH 4/4] Vendor custodia.ipa --- README.custodia.ipa | 137 ++++++++++++++++++++++ custodia/ipa/__init__.py | 1 + custodia/ipa/vault.py | 291 +++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 8 +- tests/test_ipa.py | 195 +++++++++++++++++++++++++++++++ 6 files changed, 638 insertions(+), 2 deletions(-) create mode 100644 README.custodia.ipa create mode 100644 custodia/ipa/__init__.py create mode 100644 custodia/ipa/vault.py create mode 100644 tests/test_ipa.py diff --git a/README.custodia.ipa b/README.custodia.ipa new file mode 100644 index 0000000..a952ef8 --- /dev/null +++ b/README.custodia.ipa @@ -0,0 +1,137 @@ +.. WARNING: AUTO-GENERATED FILE. DO NOT EDIT. + +custodia.ipa — IPA vault plugin for Custodia +============================================ + +**WARNING** *custodia.ipa is a tech preview with a provisional API.* + +custodia.ipa is a storage plugin for +`Custodia `__. It provides integration +with `FreeIPA `__'s +`vault `__ facility. +Secrets are encrypted and stored in +`Dogtag `__'s Key Recovery Agent. + +Requirements +------------ + +Installation +~~~~~~~~~~~~ + +- pip +- setuptools >= 18.0 + +Runtime +~~~~~~~ + +- custodia >= 0.3.1 +- ipalib >= 4.5.0 +- ipaclient >= 4.5.0 +- Python 2.7 (Python 3 support in IPA vault is unstable.) + +custodia.ipa requires an IPA-enrolled host and a Kerberos TGT for +authentication. It is recommended to provide credentials with a keytab +file or GSS-Proxy. + +Testing and development +~~~~~~~~~~~~~~~~~~~~~~~ + +- wheel +- tox + +virtualenv requirements +~~~~~~~~~~~~~~~~~~~~~~~ + +custodia.ipa depends on several binary extensions and shared libraries +for e.g. python-cryptography, python-gssapi, python-ldap, and +python-nss. For installation in a virtual environment, a C compiler and +several development packages are required. + +:: + + $ virtualenv venv + $ venv/bin/pip install --upgrade custodia.ipa + +Fedora +^^^^^^ + +:: + + $ sudo dnf install python2 python-pip python-virtualenv python-devel \ + gcc redhat-rpm-config krb5-workstation krb5-devel libffi-devel \ + nss-devel openldap-devel cyrus-sasl-devel openssl-devel + +Debian / Ubuntu +^^^^^^^^^^^^^^^ + +:: + + $ sudo apt-get update + $ sudo apt-get install -y python2.7 python-pip python-virtualenv python-dev \ + gcc krb5-user libkrb5-dev libffi-dev libnss3-dev libldap2-dev \ + libsasl2-dev libssl-dev + +-------------- + +Example configuration +--------------------- + +Create directories + +:: + + $ sudo mkdir /etc/custodia /var/lib/custodia /var/log/custodia /var/run/custodia + $ sudo chown USER:GROUP /var/lib/custodia /var/log/custodia /var/run/custodia + $ sudo chmod 750 /var/lib/custodia /var/log/custodia + +Create service account and keytab + +:: + + $ kinit admin + $ ipa service-add custodia/client1.ipa.example + $ ipa service-allow-create-keytab custodia/client1.ipa.example --users=admin + $ mkdir -p /etc/custodia + $ ipa-getkeytab -p custodia/client1.ipa.example -k /etc/custodia/custodia.keytab + +Create ``/etc/custodia/custodia.conf`` + +:: + + [DEFAULT] + confdir = /etc/custodia + libdir = /var/lib/custodia + logdir = /var/log/custodia + rundir = /var/run/custodia + + [global] + debug = true + server_socket = ${rundir}/custodia.sock + auditlog = ${logdir}/audit.log + + [store:vault] + handler = IPAVault + keytab = {confdir}/custodia.keytab + ccache = FILE:{rundir}/ccache + + [auth:creds] + handler = SimpleCredsAuth + uid = root + gid = root + + [authz:paths] + handler = SimplePathAuthz + paths = /. /secrets + + [/] + handler = Root + + [/secrets] + handler = Secrets + store = vault + +Run Custodia server + +:: + + $ custodia /etc/custodia/custodia.conf diff --git a/custodia/ipa/__init__.py b/custodia/ipa/__init__.py new file mode 100644 index 0000000..ef1bb9d --- /dev/null +++ b/custodia/ipa/__init__.py @@ -0,0 +1 @@ +# Copyright (C) 2016 Custodia Project Contributors - see LICENSE file diff --git a/custodia/ipa/vault.py b/custodia/ipa/vault.py new file mode 100644 index 0000000..f681c54 --- /dev/null +++ b/custodia/ipa/vault.py @@ -0,0 +1,291 @@ +# Copyright (C) 2016 Custodia Project Contributors - see LICENSE file +"""FreeIPA vault store (PoC) +""" +import os + +import ipalib +from ipalib.errors import DuplicateEntry, NotFound +from ipalib.krb_utils import get_principal + +import six + +from custodia.plugin import CSStore, CSStoreError, CSStoreExists, PluginOption + + +def krb5_unparse_principal_name(name): + """Split a Kerberos principal name into parts + + Returns: + * ('host', hostname, realm) for a host principal + * (servicename, hostname, realm) for a service principal + * (None, username, realm) for a user principal + + :param text name: Kerberos principal name + :return: (service, host, realm) or (None, username, realm) + """ + prefix, realm = name.split(u'@') + if u'/' in prefix: + service, host = prefix.rsplit(u'/', 1) + return service, host, realm + else: + return None, prefix, realm + + +class FreeIPA(object): + """FreeIPA wrapper + + Custodia uses a forking server model. We can bootstrap FreeIPA API in + the main process. Connections must be created in the client process. + """ + def __init__(self, api=None, ipa_context='cli', ipa_confdir=None, + ipa_debug=False): + if api is None: + self._api = ipalib.api + else: + self._api = api + self._ipa_config = dict( + context=ipa_context, + debug=ipa_debug, + log=None, # disable logging to file + ) + if ipa_confdir is not None: + self._ipa_config['confdir'] = ipa_confdir + self._bootstrap() + + @property + def Command(self): + return self._api.Command # pylint: disable=no-member + + @property + def env(self): + return self._api.env # pylint: disable=no-member + + def _bootstrap(self): + if not self._api.isdone('bootstrap'): + # TODO: bandaid for "A PKCS #11 module returned CKR_DEVICE_ERROR" + # https://github.com/avocado-framework/avocado/issues/1112#issuecomment-206999400 + os.environ['NSS_STRICT_NOFORK'] = 'DISABLED' + self._api.bootstrap(**self._ipa_config) + self._api.finalize() + + def __enter__(self): + # pylint: disable=no-member + if not self._api.Backend.rpcclient.isconnected(): + self._api.Backend.rpcclient.connect() + # pylint: enable=no-member + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # pylint: disable=no-member + if self._api.Backend.rpcclient.isconnected(): + self._api.Backend.rpcclient.disconnect() + # pylint: enable=no-member + + +class IPAVault(CSStore): + # vault arguments + principal = PluginOption( + str, None, + "Service principal for service vault (auto-discovered from GSSAPI)" + ) + user = PluginOption( + str, None, + "User name for user vault (auto-discovered from GSSAPI)" + ) + vault_type = PluginOption( + str, None, + "vault type, one of 'user', 'service', 'shared', or " + "auto-discovered from GSSAPI" + ) + + # Kerberos flags + krb5config = PluginOption(str, None, "Kerberos krb5.conf override") + keytab = PluginOption(str, None, "Kerberos keytab for auth") + ccache = PluginOption( + str, None, "Kerberos ccache, e,g. FILE:/path/to/ccache") + + # ipalib.api arguments + ipa_confdir = PluginOption(str, None, "IPA confdir override") + ipa_context = PluginOption(str, "cli", "IPA bootstrap context") + ipa_debug = PluginOption(bool, False, "debug mode for ipalib") + + def __init__(self, config, section=None): + super(IPAVault, self).__init__(config, section) + # configure Kerberos / GSSAPI and acquire principal + gssapi_principal = self._gssapi() + self.logger.info(u"Vault uses Kerberos principal '%s'", + gssapi_principal) + + # bootstrap FreeIPA API + self.ipa = FreeIPA( + ipa_confdir=self.ipa_confdir, + ipa_debug=self.ipa_debug, + ipa_context=self.ipa_context, + ) + # connect + with self.ipa: + self.logger.info("IPA server '%s': %s", + self.ipa.env.server, + self.ipa.Command.ping()[u'summary']) + # retrieve and cache KRA transport cert + response = self.ipa.Command.vaultconfig_show() + servers = response[u'result'][u'kra_server_server'] + self.logger.info("KRA server(s) %s", ', '.join(servers)) + + service, user_host, realm = krb5_unparse_principal_name( + gssapi_principal) + self._init_vault_args(service, user_host, realm) + + def _gssapi(self): + # set client keytab env var for authentication + if self.keytab is not None: + os.environ['KRB5_CLIENT_KTNAME'] = self.keytab + if self.ccache is not None: + os.environ['KRB5CCNAME'] = self.ccache + if self.krb5config is not None: + os.environ['KRB5_CONFIG'] = self.krb5config + try: + return get_principal() + except Exception: + self.logger.error( + "Unable to get principal from GSSAPI. Are you missing a " + "TGT or valid Kerberos keytab?" + ) + raise + + def _init_vault_args(self, service, user_host, realm): + if self.vault_type is None: + self.vault_type = 'user' if service is None else 'service' + self.logger.info("Setting vault type to '%s' from Kerberos", + self.vault_type) + + if self.vault_type == 'shared': + self._vault_args = {'shared': True} + elif self.vault_type == 'user': + if self.user is None: + if service is not None: + msg = "{!r}: User vault requires 'user' parameter" + raise ValueError(msg.format(self)) + else: + self.user = user_host + self.logger.info(u"Setting username '%s' from Kerberos", + self.user) + if six.PY2 and isinstance(self.user, str): + self.user = self.user.decode('utf-8') + self._vault_args = {'username': self.user} + elif self.vault_type == 'service': + if self.principal is None: + if service is None: + msg = "{!r}: Service vault requires 'principal' parameter" + raise ValueError(msg.format(self)) + else: + self.principal = u'/'.join((service, user_host)) + self.logger.info(u"Setting principal '%s' from Kerberos", + self.principal) + if six.PY2 and isinstance(self.principal, str): + self.principal = self.principal.decode('utf-8') + self._vault_args = {'service': self.principal} + else: + msg = '{!r}: Invalid vault type {}' + raise ValueError(msg.format(self, self.vault_type)) + + def _mangle_key(self, key): + if '__' in key: + raise ValueError + key = key.replace('/', '__') + if isinstance(key, bytes): + key = key.decode('utf-8') + return key + + def get(self, key): + key = self._mangle_key(key) + with self.ipa as ipa: + try: + result = ipa.Command.vault_retrieve( + key, **self._vault_args) + except NotFound as e: + self.logger.info(str(e)) + return None + except Exception: + msg = "Failed to retrieve entry {}".format(key) + self.logger.exception(msg) + raise CSStoreError(msg) + else: + return result[u'result'][u'data'] + + def set(self, key, value, replace=False): + key = self._mangle_key(key) + if not isinstance(value, bytes): + value = value.encode('utf-8') + with self.ipa as ipa: + try: + ipa.Command.vault_add( + key, ipavaulttype=u"standard", **self._vault_args) + except DuplicateEntry: + if not replace: + raise CSStoreExists(key) + except Exception: + msg = "Failed to add entry {}".format(key) + self.logger.exception(msg) + raise CSStoreError(msg) + try: + ipa.Command.vault_archive( + key, data=value, **self._vault_args) + except Exception: + msg = "Failed to archive entry {}".format(key) + self.logger.exception(msg) + raise CSStoreError(msg) + + def span(self, key): + raise CSStoreError("span is not implemented") + + def list(self, keyfilter=None): + with self.ipa as ipa: + try: + result = ipa.Command.vault_find( + ipavaulttype=u"standard", **self._vault_args) + except Exception: + msg = "Failed to list entries" + self.logger.exception(msg) + raise CSStoreError(msg) + + names = [] + for entry in result[u'result']: + cn = entry[u'cn'][0] + key = cn.replace('__', '/') + if keyfilter is not None and not key.startswith(keyfilter): + continue + names.append(key.rsplit('/', 1)[-1]) + return names + + def cut(self, key): + key = self._mangle_key(key) + with self.ipa as ipa: + try: + ipa.Command.vault_del(key, **self._vault_args) + except NotFound: + return False + except Exception: + msg = "Failed to delete entry {}".format(key) + self.logger.exception(msg) + raise CSStoreError(msg) + else: + return True + + +if __name__ == '__main__': + from custodia.compat import configparser + + parser = configparser.ConfigParser( + interpolation=configparser.ExtendedInterpolation() + ) + parser.read_string(u""" + [store:ipa_vault] + """) + + v = IPAVault(parser, "store:ipa_vault") + v.set('foo', 'bar', replace=True) + print(v.get('foo')) + print(v.list()) + v.cut('foo') + print(v.list()) diff --git a/setup.py b/setup.py index c8f270d..4bf096c 100755 --- a/setup.py +++ b/setup.py @@ -15,10 +15,14 @@ requirements = [ 'requests' ] +# extra requirements +ipa_requires = ['ipalib >= 4.5.0', 'ipaclient >= 4.5.0'] + # test requirements -test_requires = ['coverage', 'pytest'] +test_requires = ['coverage', 'pytest', 'mock'] + ipa_requires extras_require = { + 'ipa': ipa_requires, 'test': test_requires, 'test_docs': ['docutils', 'markdown'], 'test_pep8': ['flake8', 'flake8-import-order', 'pep8-naming'], @@ -66,6 +70,7 @@ custodia_consumers = [ custodia_stores = [ 'EncryptedOverlay = custodia.store.encgen:EncryptedOverlay', 'EncryptedStore = custodia.store.enclite:EncryptedStore', + 'IPAVault = custodia.ipa.vault:IPAVault', 'SqliteStore = custodia.store.sqlite:SqliteStore', ] @@ -84,6 +89,7 @@ setup( 'custodia', 'custodia.cli', 'custodia.httpd', + 'custodia.ipa', 'custodia.message', 'custodia.server', 'custodia.store', diff --git a/tests/test_ipa.py b/tests/test_ipa.py new file mode 100644 index 0000000..eb4d7fa --- /dev/null +++ b/tests/test_ipa.py @@ -0,0 +1,195 @@ +# Copyright (C) 2017 Custodia project Contributors, for licensee see COPYING + +import os + +import ipalib + +import mock + +import pytest + +from custodia.compat import configparser +from custodia.ipa.vault import FreeIPA, IPAVault, krb5_unparse_principal_name + + +CONFIG = u""" +[store:ipa_service] +vault_type = service +principal = custodia/ipa.example + +[store:ipa_user] +vault_type = user +user = john + +[store:ipa_shared] +vault_type = shared + +[store:ipa_invalid] +vault_type = invalid + +[store:ipa_autodiscover] + +[store:ipa_environ] +krb5config = /path/to/krb5.conf +keytab = /path/to/custodia.keytab +ccache = FILE:/path/to/ccache +""" + +vault_parametrize = pytest.mark.parametrize( + 'plugin,vault_type,vault_args', + [ + ('store:ipa_service', 'service', {'service': 'custodia/ipa.example'}), + ('store:ipa_user', 'user', {'username': 'john'}), + ('store:ipa_shared', 'shared', {'shared': True}), + ] +) + + +class TestCustodiaIPA: + def setup_method(self, method): + self.parser = configparser.ConfigParser( + interpolation=configparser.ExtendedInterpolation(), + ) + self.parser.read_string(CONFIG) + # mocked ipalib.api + self.p_api = mock.patch('ipalib.api', autospec=ipalib.api) + self.m_api = self.p_api.start() + self.m_api.isdone.return_value = False + self.m_api.env = mock.Mock() + self.m_api.env.server = 'server.ipa.example' + self.m_api.Backend = mock.Mock() + self.m_api.Command = mock.Mock() + self.m_api.Command.ping.return_value = { + u'summary': u'IPA server version 4.4.3. API version 2.215', + } + self.m_api.Command.vaultconfig_show.return_value = { + u'result': { + u'kra_server_server': [u'ipa.example'], + } + } + # mocked get_principal + self.p_get_principal = mock.patch('custodia.ipa.vault.get_principal') + self.m_get_principal = self.p_get_principal.start() + self.m_get_principal.return_value = 'custodia/ipa.example@IPA.EXAMPLE' + # mocked environ (empty dict) + self.p_env = mock.patch.dict('os.environ', clear=True) + self.p_env.start() + + def teardown_method(self, method): + self.p_api.stop() + self.p_get_principal.stop() + self.p_env.stop() + + def test_api_init(self): + m_api = self.m_api + freeipa = FreeIPA(api=m_api) + m_api.isdone.assert_called_once_with('bootstrap') + m_api.bootstrap.assert_called_once_with( + context='cli', + debug=False, + log=None, + ) + + m_api.Backend.rpcclient.isconnected.return_value = False + with freeipa: + m_api.Backend.rpcclient.connect.assert_called_once() + m_api.Backend.rpcclient.isconnected.return_value = True + m_api.Backend.rpcclient.disconnect.assert_called_once() + + assert os.environ == dict(NSS_STRICT_NOFORK='DISABLED') + + def test_api_environ(self): + assert os.environ == {} + IPAVault(self.parser, 'store:ipa_environ') + assert os.environ == dict( + NSS_STRICT_NOFORK='DISABLED', + KRB5_CONFIG='/path/to/krb5.conf', + KRB5_CLIENT_KTNAME='/path/to/custodia.keytab', + KRB5CCNAME='FILE:/path/to/ccache', + ) + + def test_invalid_vault_type(self): + pytest.raises(ValueError, IPAVault, self.parser, 'store:ipa_invalid') + + def test_vault_autodiscover_service(self): + self.m_get_principal.return_value = 'custodia/ipa.example@IPA.EXAMPLE' + ipa = IPAVault(self.parser, 'store:ipa_autodiscover') + assert ipa.vault_type == 'service' + assert ipa.principal == 'custodia/ipa.example' + assert ipa.user is None + + def test_vault_autodiscover_user(self): + self.m_get_principal.return_value = 'john@IPA.EXAMPLE' + ipa = IPAVault(self.parser, 'store:ipa_autodiscover') + assert ipa.vault_type == 'user' + assert ipa.principal is None + assert ipa.user == 'john' + + @vault_parametrize + def test_vault_set(self, plugin, vault_type, vault_args): + ipa = IPAVault(self.parser, plugin) + assert ipa.vault_type == vault_type + self.m_api.Command.ping.assert_called_once() + ipa.set('directory/testkey', 'testvalue') + self.m_api.Command.vault_add.assert_called_once_with( + 'directory__testkey', + ipavaulttype=u'standard', + **vault_args + ) + self.m_api.Command.vault_archive.assert_called_once_with( + 'directory__testkey', + data=b'testvalue', + **vault_args + ) + + @vault_parametrize + def test_vault_get(self, plugin, vault_type, vault_args): + ipa = IPAVault(self.parser, plugin) + assert ipa.vault_type == vault_type + self.m_api.Command.vault_retrieve.return_value = { + u'result': { + u'data': b'testvalue', + } + } + assert ipa.get('directory/testkey') == b'testvalue' + self.m_api.Command.vault_retrieve.assert_called_once_with( + 'directory__testkey', + **vault_args + ) + + @vault_parametrize + def test_vault_list(self, plugin, vault_type, vault_args): + ipa = IPAVault(self.parser, plugin) + assert ipa.vault_type == vault_type + self.m_api.Command.vault_find.return_value = { + u'result': [{'cn': [u'directory__testkey']}] + } + assert ipa.list('directory') == ['testkey'] + self.m_api.Command.vault_find.assert_called_once_with( + ipavaulttype=u'standard', + **vault_args + ) + + @vault_parametrize + def test_vault_cut(self, plugin, vault_type, vault_args): + ipa = IPAVault(self.parser, plugin) + assert ipa.vault_type == vault_type + ipa.cut('directory/testkey') + self.m_api.Command.vault_del.assert_called_once_with( + 'directory__testkey', + **vault_args + ) + + +@pytest.mark.parametrize('principal,result', [ + ('john@IPA.EXAMPLE', + (None, 'john', 'IPA.EXAMPLE')), + ('host/host.invalid@IPA.EXAMPLE', + ('host', 'host.invalid', 'IPA.EXAMPLE')), + ('custodia/host.invalid@IPA.EXAMPLE', + ('custodia', 'host.invalid', 'IPA.EXAMPLE')), + ('whatever/custodia/host.invalid@IPA.EXAMPLE', + ('whatever/custodia', 'host.invalid', 'IPA.EXAMPLE')), +]) +def test_unparse(principal, result): + assert krb5_unparse_principal_name(principal) == result -- 2.9.3