From be32fd1d727fc8398dd51fa0fd3f404ef451281a Mon Sep 17 00:00:00 2001
From: Jan Cholasta <jcholast@redhat.com>
Date: Mon, 1 Aug 2016 09:53:39 +0200
Subject: [PATCH] cert: speed up cert-find
Use issuer+serial rather than raw DER blob to identify certificates in
cert-find's intermediate result.
Restructure the code to make it (hopefully) easier to follow.
https://fedorahosted.org/freeipa/ticket/6098
Reviewed-By: Martin Basti <mbasti@redhat.com>
Reviewed-By: Pavel Vomacka <pvomacka@redhat.com>
---
ipaserver/plugins/cert.py | 398 +++++++++++++++++++++++++---------------------
1 file changed, 216 insertions(+), 182 deletions(-)
diff --git a/ipaserver/plugins/cert.py b/ipaserver/plugins/cert.py
index 06041d3083565e8d093b610473d6083111d406d2..47dccf15a4010f2766642aedd2cc16e0a1eb1dd4 100644
--- a/ipaserver/plugins/cert.py
+++ b/ipaserver/plugins/cert.py
@@ -21,6 +21,7 @@
import base64
import binascii
+import collections
import datetime
import os
@@ -295,18 +296,24 @@ class BaseCertObject(Object):
),
)
- def _parse(self, obj):
- cert = x509.load_certificate(obj['certificate'])
- obj['subject'] = DN(unicode(cert.subject))
- obj['issuer'] = DN(unicode(cert.issuer))
- obj['valid_not_before'] = unicode(cert.valid_not_before_str)
- obj['valid_not_after'] = unicode(cert.valid_not_after_str)
- obj['md5_fingerprint'] = unicode(
- nss.data_to_hex(nss.md5_digest(cert.der_data), 64)[0])
- obj['sha1_fingerprint'] = unicode(
- nss.data_to_hex(nss.sha1_digest(cert.der_data), 64)[0])
- obj['serial_number'] = cert.serial_number
- obj['serial_number_hex'] = u'0x%X' % cert.serial_number
+ def _parse(self, obj, full=True):
+ cert = obj.get('certificate')
+ if cert is not None:
+ cert = x509.load_certificate(cert)
+ obj['subject'] = DN(unicode(cert.subject))
+ obj['issuer'] = DN(unicode(cert.issuer))
+ obj['serial_number'] = cert.serial_number
+ if full:
+ obj['valid_not_before'] = unicode(cert.valid_not_before_str)
+ obj['valid_not_after'] = unicode(cert.valid_not_after_str)
+ obj['md5_fingerprint'] = unicode(
+ nss.data_to_hex(nss.md5_digest(cert.der_data), 64)[0])
+ obj['sha1_fingerprint'] = unicode(
+ nss.data_to_hex(nss.sha1_digest(cert.der_data), 64)[0])
+
+ serial_number = obj.get('serial_number')
+ if serial_number is not None:
+ obj['serial_number_hex'] = u'0x%X' % serial_number
class BaseCertMethod(Method):
@@ -691,10 +698,14 @@ class cert(BaseCertObject):
yield self.api.Object[name]
def _fill_owners(self, obj):
+ dns = obj.pop('owner', None)
+ if dns is None:
+ return
+
for owner in self._owners():
container_dn = DN(owner.container_dn, self.api.env.basedn)
name = 'owner_' + owner.name
- for dn in obj['owner']:
+ for dn in dns:
if dn.endswith(container_dn, 1):
value = owner.get_primary_key_from_dn(dn)
obj.setdefault(name, []).append(value)
@@ -776,9 +787,7 @@ class cert_show(Retrieve, CertMethod, VirtualCommand):
result['certificate'] = result['certificate'].replace('\r\n', '')
self.obj._parse(result)
result['revoked'] = ('revocation_reason' in result)
- if 'owner' in result:
- self.obj._fill_owners(result)
- del result['owner']
+ self.obj._fill_owners(result)
if hostname:
# If we have a hostname we want to verify that the subject
@@ -984,36 +993,171 @@ class cert_find(Search, CertMethod):
label=owner.object_name,
)
- def execute(self, criteria=None, all=False, raw=False, pkey_only=False,
- no_members=True, timelimit=None, sizelimit=None, **options):
- ca_options = {'cacn',
- 'revocation_reason',
- 'issuer',
- 'subject',
- 'min_serial_number', 'max_serial_number',
- 'exactly',
- 'validnotafter_from', 'validnotafter_to',
- 'validnotbefore_from', 'validnotbefore_to',
- 'issuedon_from', 'issuedon_to',
- 'revokedon_from', 'revokedon_to'}
- ldap_options = {prefix + owner.name
- for owner in self.obj._owners()
- for prefix in ('', 'no_')}
- has_ca_options = (
- any(name in options for name in ca_options - {'exactly'}) or
- options['exactly'])
- has_ldap_options = any(name in options for name in ldap_options)
- has_cert_option = 'certificate' in options
+ def _get_cert_key(self, cert):
+ nss_cert = x509.load_certificate(cert, x509.DER)
+
+ return (DN(unicode(nss_cert.issuer)), nss_cert.serial_number)
+
+ def _get_cert_obj(self, cert, all, raw, pkey_only):
+ obj = {'certificate': unicode(base64.b64encode(cert))}
+
+ full = not pkey_only and all
+ if not raw:
+ self.obj._parse(obj, full)
+ if not full:
+ del obj['certificate']
+
+ return obj
+
+ def _cert_search(self, all, raw, pkey_only, **options):
+ result = collections.OrderedDict()
+
+ try:
+ cert = options['certificate']
+ except KeyError:
+ return result, False, False
+
+ key = self._get_cert_key(cert)
+
+ result[key] = self._get_cert_obj(cert, all, raw, pkey_only)
+
+ return result, False, True
+
+ def _ca_search(self, all, raw, pkey_only, sizelimit, exactly, **options):
+ ra_options = {}
+ for name in ('revocation_reason',
+ 'issuer',
+ 'subject',
+ 'min_serial_number', 'max_serial_number',
+ 'validnotafter_from', 'validnotafter_to',
+ 'validnotbefore_from', 'validnotbefore_to',
+ 'issuedon_from', 'issuedon_to',
+ 'revokedon_from', 'revokedon_to'):
+ try:
+ value = options[name]
+ except KeyError:
+ continue
+ if isinstance(value, datetime.datetime):
+ value = value.strftime(PKIDATE_FORMAT)
+ elif isinstance(value, DN):
+ value = unicode(value)
+ ra_options[name] = value
+ if sizelimit:
+ ra_options['sizelimit'] = sizelimit
+ if exactly:
+ ra_options['exactly'] = True
+
+ result = collections.OrderedDict()
+ complete = bool(ra_options)
try:
ca_enabled_check()
except errors.NotFound:
- if has_ca_options:
+ if ra_options:
raise
- ca_enabled = False
+ return result, False, complete
+
+ ra = self.api.Backend.ra
+ for ra_obj in ra.find(ra_options):
+ issuer = DN(ra_obj['issuer'])
+ serial_number = ra_obj['serial_number']
+
+ if pkey_only:
+ obj = {'serial_number': serial_number}
+ else:
+ obj = ra_obj
+ obj['issuer'] = issuer
+ obj['subject'] = DN(ra_obj['subject'])
+ del obj['serial_number_hex']
+
+ if all:
+ ra_obj = ra.get_certificate(str(serial_number))
+ if not raw:
+ obj['certificate'] = (
+ ra_obj['certificate'].replace('\r\n', ''))
+ self.obj._parse(obj)
+
+ result[issuer, serial_number] = obj
+
+ return result, False, complete
+
+ def _ldap_search(self, all, raw, pkey_only, no_members, timelimit,
+ sizelimit, **options):
+ ldap = self.api.Backend.ldap2
+
+ filters = []
+ for owner in self.obj._owners():
+ for prefix, rule in (('', ldap.MATCH_ALL),
+ ('no_', ldap.MATCH_NONE)):
+ try:
+ value = options[prefix + owner.name]
+ except KeyError:
+ continue
+
+ filter = ldap.make_filter_from_attr(
+ 'objectclass',
+ owner.object_class,
+ ldap.MATCH_ALL)
+ if filter not in filters:
+ filters.append(filter)
+
+ filter = ldap.make_filter_from_attr(
+ owner.primary_key.name,
+ value,
+ rule)
+ filters.append(filter)
+
+ cert = options.get('certificate')
+ if cert is not None:
+ filter = ldap.make_filter_from_attr('usercertificate', cert)
+ filters.append(filter)
+
+ result = collections.OrderedDict()
+ complete = bool(filters)
+
+ if cert is None:
+ filter = '(usercertificate=*)'
+ filters.append(filter)
+
+ filter = ldap.combine_filters(filters, ldap.MATCH_ALL)
+ try:
+ entries, truncated = ldap.find_entries(
+ base_dn=self.api.env.basedn,
+ filter=filter,
+ attrs_list=['usercertificate'],
+ time_limit=timelimit,
+ size_limit=sizelimit,
+ )
+ except errors.EmptyResult:
+ entries = []
+ truncated = False
else:
- ca_enabled = True
+ truncated = bool(truncated)
+
+ for entry in entries:
+ for attr in ('usercertificate', 'usercertificate;binary'):
+ for cert in entry.get(attr, []):
+ key = self._get_cert_key(cert)
+
+ try:
+ obj = result[key]
+ except KeyError:
+ obj = self._get_cert_obj(cert, all, raw, pkey_only)
+ result[key] = obj
+ if not pkey_only and (all or not no_members):
+ owners = obj.setdefault('owner', [])
+ if entry.dn not in owners:
+ owners.append(entry.dn)
+
+ if not raw:
+ for obj in six.itervalues(result):
+ self.obj._fill_owners(obj)
+
+ return result, truncated, complete
+
+ def execute(self, criteria=None, all=False, raw=False, pkey_only=False,
+ no_members=True, timelimit=None, sizelimit=None, **options):
if 'cacn' in options:
ca_obj = api.Command.ca_show(options['cacn'])['result']
ca_sdn = unicode(ca_obj['ipacasubjectdn'][0])
@@ -1028,153 +1172,43 @@ class cert_find(Search, CertMethod):
if criteria is not None:
return dict(result=[], count=0, truncated=False)
- obj_seq = []
- obj_dict = {}
+ result = collections.OrderedDict()
truncated = False
-
- if has_cert_option:
- cert = options['certificate']
- obj = {'certificate': unicode(base64.b64encode(cert))}
- obj_seq.append(obj)
- obj_dict[cert] = obj
-
- if ca_enabled:
- ra_options = {}
- for name, value in options.items():
- if name not in ca_options:
- continue
- if isinstance(value, datetime.datetime):
- value = value.strftime(PKIDATE_FORMAT)
- elif isinstance(value, DN):
- value = unicode(value)
- ra_options[name] = value
- if sizelimit is not None:
- if sizelimit != 0:
- ra_options['sizelimit'] = sizelimit
- sizelimit = 0
- has_ca_options = True
-
- for ra_obj in self.Backend.ra.find(ra_options):
- obj = {}
- if ((not pkey_only and all) or
- not no_members or
- not has_ca_options or
- has_ldap_options or
- has_cert_option):
- ra_obj.update(
- self.Backend.ra.get_certificate(
- str(ra_obj['serial_number'])))
- cert = base64.b64decode(ra_obj['certificate'])
- try:
- obj = obj_dict[cert]
- except KeyError:
- if has_cert_option:
- continue
- obj = {}
- obj_seq.append(obj)
- obj_dict[cert] = obj
+ complete = False
+
+ for sub_search in (self._cert_search,
+ self._ca_search,
+ self._ldap_search):
+ sub_result, sub_truncated, sub_complete = sub_search(
+ all=all,
+ raw=raw,
+ pkey_only=pkey_only,
+ no_members=no_members,
+ timelimit=timelimit,
+ sizelimit=sizelimit,
+ **options)
+
+ if sub_complete:
+ sizelimit = None
+
+ for key in tuple(result):
+ if key not in sub_result:
+ del result[key]
+
+ for key, sub_obj in six.iteritems(sub_result):
+ try:
+ obj = result[key]
+ except KeyError:
+ if complete:
+ continue
+ result[key] = sub_obj
else:
- obj_seq.append(obj)
- obj.update(ra_obj)
-
- if ((not pkey_only and all) or
- not no_members or
- not has_ca_options or
- has_ldap_options or
- has_cert_option):
- ldap = self.api.Backend.ldap2
+ obj.update(sub_obj)
- filters = []
- if 'certificate' in options:
- cert_filter = ldap.make_filter_from_attr(
- 'usercertificate', options['certificate'])
- else:
- cert_filter = '(usercertificate=*)'
- filters.append(cert_filter)
- for owner in self.obj._owners():
- oc_filter = ldap.make_filter_from_attr(
- 'objectclass', owner.object_class, ldap.MATCH_ALL)
- for prefix, rule in (('', ldap.MATCH_ALL),
- ('no_', ldap.MATCH_NONE)):
- value = options.get(prefix + owner.name)
- if value is None:
- continue
- pkey_filter = ldap.make_filter_from_attr(
- owner.primary_key.name, value, rule)
- filters.append(oc_filter)
- filters.append(pkey_filter)
- filter = ldap.combine_filters(filters, ldap.MATCH_ALL)
+ truncated = truncated or sub_truncated
+ complete = complete or sub_complete
- try:
- entries, truncated = ldap.find_entries(
- base_dn=self.api.env.basedn,
- filter=filter,
- attrs_list=['usercertificate'],
- time_limit=timelimit,
- size_limit=sizelimit,
- )
- except errors.EmptyResult:
- entries, truncated = [], False
- for entry in entries:
- seen = set()
- for attr in ('usercertificate', 'usercertificate;binary'):
- for cert in entry.get(attr, []):
- if cert in seen:
- continue
- seen.add(cert)
- try:
- obj = obj_dict[cert]
- except KeyError:
- if has_ca_options or has_cert_option:
- continue
- obj = {
- 'certificate': unicode(base64.b64encode(cert))}
- obj_seq.append(obj)
- obj_dict[cert] = obj
- obj.setdefault('owner', []).append(entry.dn)
-
- result = []
- for obj in obj_seq:
- if has_ldap_options and 'owner' not in obj:
- continue
- if not pkey_only:
- if not raw:
- if 'certificate' in obj:
- obj['certificate'] = (
- obj['certificate'].replace('\r\n', ''))
- self.obj._parse(obj)
- if not all:
- del obj['certificate']
- del obj['valid_not_before']
- del obj['valid_not_after']
- del obj['md5_fingerprint']
- del obj['sha1_fingerprint']
- if 'subject' in obj:
- obj['subject'] = DN(obj['subject'])
- if 'issuer' in obj:
- obj['issuer'] = DN(obj['issuer'])
- if 'status' in obj:
- obj['revoked'] = (
- obj['status'] in (u'REVOKED', u'REVOKED_EXPIRED'))
- if 'owner' in obj:
- if all or not no_members:
- self.obj._fill_owners(obj)
- del obj['owner']
- else:
- if 'certificate' in obj:
- if not all:
- del obj['certificate']
- if 'owner' in obj:
- if not all and no_members:
- del obj['owner']
- else:
- if 'serial_number' in obj:
- serial_number = obj['serial_number']
- obj.clear()
- obj['serial_number'] = serial_number
- else:
- obj.clear()
- result.append(obj)
+ result = list(six.itervalues(result))
ret = dict(
result=result
--
2.7.4