From 83445df2894426cdaf2f09f8bf2ac4c829922620 Mon Sep 17 00:00:00 2001
From: Christian Heimes <cheimes@redhat.com>
Date: Sun, 8 Jul 2018 11:53:58 +0200
Subject: [PATCH] Auto-retry failed certmonger requests
During parallel replica installation, a request sometimes fails with
CA_REJECTED or CA_UNREACHABLE. The error occur when the master is
either busy or some information haven't been replicated yet. Even
a stuck request can be recovered, e.g. when permission and group
information have been replicated.
A new function request_and_retry_cert() automatically resubmits failing
requests until it times out.
Fixes: https://pagure.io/freeipa/issue/7623
Signed-off-by: Christian Heimes <cheimes@redhat.com>
Reviewed-By: Tibor Dudlak <tdudlak@redhat.com>
---
ipalib/install/certmonger.py | 64 ++++++++++++++++++++++++-------
ipaserver/install/cainstance.py | 4 +-
ipaserver/install/certs.py | 19 ++++++---
ipaserver/install/dsinstance.py | 4 +-
ipaserver/install/httpinstance.py | 5 ++-
ipaserver/install/krbinstance.py | 4 +-
6 files changed, 76 insertions(+), 24 deletions(-)
diff --git a/ipalib/install/certmonger.py b/ipalib/install/certmonger.py
index d2b782ddb0c746a3dfd96d0222bb31c6a960fdff..d915cbf7638c68f3ad5170b2878a706ad39def62 100644
--- a/ipalib/install/certmonger.py
+++ b/ipalib/install/certmonger.py
@@ -302,20 +302,56 @@ def add_subject(request_id, subject):
def request_and_wait_for_cert(
certpath, subject, principal, nickname=None, passwd_fname=None,
dns=None, ca='IPA', profile=None,
- pre_command=None, post_command=None, storage='NSSDB', perms=None):
- """
- Execute certmonger to request a server certificate.
-
- The method also waits for the certificate to be available.
- """
- reqId = request_cert(certpath, subject, principal, nickname,
- passwd_fname, dns, ca, profile,
- pre_command, post_command, storage, perms)
- state = wait_for_request(reqId, api.env.startup_timeout)
- ca_error = get_request_value(reqId, 'ca-error')
- if state != 'MONITORING' or ca_error:
- raise RuntimeError("Certificate issuance failed ({})".format(state))
- return reqId
+ pre_command=None, post_command=None, storage='NSSDB', perms=None,
+ resubmit_timeout=0):
+ """Request certificate, wait and possibly resubmit failing requests
+
+ Submit a cert request to certmonger and wait until the request has
+ finished.
+
+ With timeout, a failed request is resubmitted. During parallel replica
+ installation, a request sometimes fails with CA_REJECTED or
+ CA_UNREACHABLE. The error occurs when the master is either busy or some
+ information haven't been replicated yet. Even a stuck request can be
+ recovered, e.g. when permission and group information have been
+ replicated.
+ """
+ req_id = request_cert(
+ certpath, subject, principal, nickname, passwd_fname, dns, ca,
+ profile, pre_command, post_command, storage, perms
+ )
+
+ deadline = time.time() + resubmit_timeout
+ while True: # until success, timeout, or error
+ state = wait_for_request(req_id, api.env.replication_wait_timeout)
+ ca_error = get_request_value(req_id, 'ca-error')
+ if state == 'MONITORING' and ca_error is None:
+ # we got a winner, exiting
+ root_logger.debug("Cert request %s was successful", req_id)
+ return req_id
+
+ root_logger.debug(
+ "Cert request %s failed: %s (%s)", req_id, state, ca_error
+ )
+ if state not in {'CA_REJECTED', 'CA_UNREACHABLE'}:
+ # probably unrecoverable error
+ root_logger.debug("Giving up on cert request %s", req_id)
+ break
+ elif not resubmit_timeout:
+ # no resubmit
+ break
+ elif time.time() > deadline:
+ root_logger.debug("Request %s reached resubmit dead line", req_id)
+ break
+ else:
+ # sleep and resubmit
+ root_logger.debug("Sleep and resubmit cert request %s", req_id)
+ time.sleep(10)
+ resubmit_request(req_id)
+
+ raise RuntimeError(
+ "Certificate issuance failed ({}: {})".format(state, ca_error)
+ )
def request_cert(
diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py
index 62e9ad7de6f00eabb48f726a3931eb8acf0ba22b..e207911814e3553c5aa5310694170d3575337c55 100644
--- a/ipaserver/install/cainstance.py
+++ b/ipaserver/install/cainstance.py
@@ -863,7 +863,9 @@ class CAInstance(DogtagInstance):
profile='caServerCert',
pre_command='renew_ra_cert_pre',
post_command='renew_ra_cert',
- storage="FILE")
+ storage="FILE",
+ resubmit_timeout=api.env.replication_wait_timeout
+ )
self.__set_ra_cert_perms()
self.requestId = str(reqId)
diff --git a/ipaserver/install/certs.py b/ipaserver/install/certs.py
index de96318db51b03f2515814d574cfebf1b242b6a6..5670d468bb1b168af7ada7b2d8924b8ec9f5d9c1 100644
--- a/ipaserver/install/certs.py
+++ b/ipaserver/install/certs.py
@@ -663,12 +663,19 @@ class CertDB(object):
def export_pem_cert(self, nickname, location):
return self.nssdb.export_pem_cert(nickname, location)
- def request_service_cert(self, nickname, principal, host):
- certmonger.request_and_wait_for_cert(certpath=self.secdir,
- nickname=nickname,
- principal=principal,
- subject=host,
- passwd_fname=self.passwd_fname)
+ def request_service_cert(self, nickname, principal, host,
+ resubmit_timeout=None):
+ if resubmit_timeout is None:
+ resubmit_timeout = api.env.replication_wait_timeout
+ return certmonger.request_and_wait_for_cert(
+ certpath=self.secdir,
+ storage='NSSDB',
+ nickname=nickname,
+ principal=principal,
+ subject=host,
+ passwd_fname=self.passwd_fname,
+ resubmit_timeout=resubmit_timeout
+ )
def is_ipa_issued_cert(self, api, nickname):
"""
diff --git a/ipaserver/install/dsinstance.py b/ipaserver/install/dsinstance.py
index 7adaabd3c1280709150329003130f70233de37f4..8ec5cdd7a4a324a1a40dbe968defcc797db8f054 100644
--- a/ipaserver/install/dsinstance.py
+++ b/ipaserver/install/dsinstance.py
@@ -847,7 +847,9 @@ class DsInstance(service.Service):
ca='IPA',
profile=dogtag.DEFAULT_PROFILE,
dns=[self.fqdn],
- post_command=cmd)
+ post_command=cmd,
+ resubmit_timeout=api.env.replication_wait_timeout
+ )
finally:
if prev_helper is not None:
certmonger.modify_ca_helper('IPA', prev_helper)
diff --git a/ipaserver/install/httpinstance.py b/ipaserver/install/httpinstance.py
index 2df51eaa77d3ee3246027a6bcbc4023dbad61160..2366698bb0d6a7c9cd481ba9d5568d320742f6bb 100644
--- a/ipaserver/install/httpinstance.py
+++ b/ipaserver/install/httpinstance.py
@@ -450,7 +450,10 @@ class HTTPInstance(service.Service):
ca='IPA',
profile=dogtag.DEFAULT_PROFILE,
dns=[self.fqdn],
- post_command='restart_httpd')
+ post_command='restart_httpd',
+ storage='NSSDB',
+ resubmit_timeout=api.env.replication_wait_timeout
+ )
finally:
if prev_helper is not None:
certmonger.modify_ca_helper('IPA', prev_helper)
diff --git a/ipaserver/install/krbinstance.py b/ipaserver/install/krbinstance.py
index 4041d1b5fb3c3cf3db78b6cb282ce5f17793a0e3..62de118b7104fe7d72d2c1fd5577e9a76000c663 100644
--- a/ipaserver/install/krbinstance.py
+++ b/ipaserver/install/krbinstance.py
@@ -436,7 +436,9 @@ class KrbInstance(service.Service):
storage='FILE',
profile=KDC_PROFILE,
post_command='renew_kdc_cert',
- perms=(0o644, 0o600))
+ perms=(0o644, 0o600),
+ resubmit_timeout=api.env.replication_wait_timeout
+ )
except dbus.DBusException as e:
# if the certificate is already tracked, ignore the error
name = e.get_dbus_name()
--
2.17.1