pgreco / rpms / ipa

Forked from forks/areguera/rpms/ipa 4 years ago
Clone
Blob Blame History Raw
From 3e8427f3aa392f961923f5c9d509bab64ce3c9ab Mon Sep 17 00:00:00 2001
From: Christian Heimes <cheimes@redhat.com>
Date: Sun, 8 Jul 2018 11:53:58 +0200
Subject: [PATCH] Auto-retry failed certmonger requests

During parallel replica installation, a request sometimes fails with
CA_REJECTED or CA_UNREACHABLE. The error occur when the master is
either busy or some information haven't been replicated yet. Even
a stuck request can be recovered, e.g. when permission and group
information have been replicated.

A new function request_and_retry_cert() automatically resubmits failing
requests until it times out.

Fixes: https://pagure.io/freeipa/issue/7623
Signed-off-by: Christian Heimes <cheimes@redhat.com>
Reviewed-By: Stanislav Laznicka <slaznick@redhat.com>
---
 ipalib/install/certmonger.py      | 64 ++++++++++++++++++++++++-------
 ipaserver/install/cainstance.py   |  4 +-
 ipaserver/install/certs.py        | 19 ++++++---
 ipaserver/install/dsinstance.py   |  4 +-
 ipaserver/install/httpinstance.py |  5 ++-
 ipaserver/install/krbinstance.py  |  4 +-
 6 files changed, 76 insertions(+), 24 deletions(-)

diff --git a/ipalib/install/certmonger.py b/ipalib/install/certmonger.py
index c07242412affa29eca3312fd27985f65869d3f7a..3e1862192eb0d9245797ebbe8abf2ff69d7e7767 100644
--- a/ipalib/install/certmonger.py
+++ b/ipalib/install/certmonger.py
@@ -305,20 +305,56 @@ def add_subject(request_id, subject):
 def request_and_wait_for_cert(
         certpath, subject, principal, nickname=None, passwd_fname=None,
         dns=None, ca='IPA', profile=None,
-        pre_command=None, post_command=None, storage='NSSDB', perms=None):
-    """
-    Execute certmonger to request a server certificate.
-
-    The method also waits for the certificate to be available.
-    """
-    reqId = request_cert(certpath, subject, principal, nickname,
-                         passwd_fname, dns, ca, profile,
-                         pre_command, post_command, storage, perms)
-    state = wait_for_request(reqId, api.env.startup_timeout)
-    ca_error = get_request_value(reqId, 'ca-error')
-    if state != 'MONITORING' or ca_error:
-        raise RuntimeError("Certificate issuance failed ({})".format(state))
-    return reqId
+        pre_command=None, post_command=None, storage='NSSDB', perms=None,
+        resubmit_timeout=0):
+    """Request certificate, wait and possibly resubmit failing requests
+
+    Submit a cert request to certmonger and wait until the request has
+    finished.
+
+    With timeout, a failed request is resubmitted. During parallel replica
+    installation, a request sometimes fails with CA_REJECTED or
+    CA_UNREACHABLE. The error occurs when the master is either busy or some
+    information haven't been replicated yet. Even a stuck request can be
+    recovered, e.g. when permission and group information have been
+    replicated.
+    """
+    req_id = request_cert(
+        certpath, subject, principal, nickname, passwd_fname, dns, ca,
+        profile, pre_command, post_command, storage, perms
+    )
+
+    deadline = time.time() + resubmit_timeout
+    while True:  # until success, timeout, or error
+        state = wait_for_request(req_id, api.env.replication_wait_timeout)
+        ca_error = get_request_value(req_id, 'ca-error')
+        if state == 'MONITORING' and ca_error is None:
+            # we got a winner, exiting
+            logger.debug("Cert request %s was successful", req_id)
+            return req_id
+
+        logger.debug(
+            "Cert request %s failed: %s (%s)", req_id, state, ca_error
+        )
+        if state not in {'CA_REJECTED', 'CA_UNREACHABLE'}:
+            # probably unrecoverable error
+            logger.debug("Giving up on cert request %s", req_id)
+            break
+        elif not resubmit_timeout:
+            # no resubmit
+            break
+        elif time.time() > deadline:
+            logger.debug("Request %s reached resubmit dead line", req_id)
+            break
+        else:
+            # sleep and resubmit
+            logger.debug("Sleep and resubmit cert request %s", req_id)
+            time.sleep(10)
+            resubmit_request(req_id)
+
+    raise RuntimeError(
+        "Certificate issuance failed ({}: {})".format(state, ca_error)
+    )
 
 
 def request_cert(
diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py
index 8193f3da854b3a20d175de523fbc453f5c5104d8..6dbf69b3e5833f220a4d7d640b66a8fcf824f445 100644
--- a/ipaserver/install/cainstance.py
+++ b/ipaserver/install/cainstance.py
@@ -917,7 +917,9 @@ class CAInstance(DogtagInstance):
                 profile='caServerCert',
                 pre_command='renew_ra_cert_pre',
                 post_command='renew_ra_cert',
-                storage="FILE")
+                storage="FILE",
+                resubmit_timeout=api.env.replication_wait_timeout
+            )
             self.__set_ra_cert_perms()
 
             self.requestId = str(reqId)
diff --git a/ipaserver/install/certs.py b/ipaserver/install/certs.py
index 3f843399f4f964223f52242d610e842a5dc473e8..30b2aa0d3e7b2cafbcc17ad3d04764a342ae8002 100644
--- a/ipaserver/install/certs.py
+++ b/ipaserver/install/certs.py
@@ -600,12 +600,19 @@ class CertDB(object):
     def export_pem_cert(self, nickname, location):
         return self.nssdb.export_pem_cert(nickname, location)
 
-    def request_service_cert(self, nickname, principal, host):
-        certmonger.request_and_wait_for_cert(certpath=self.secdir,
-                                             nickname=nickname,
-                                             principal=principal,
-                                             subject=host,
-                                             passwd_fname=self.passwd_fname)
+    def request_service_cert(self, nickname, principal, host,
+                             resubmit_timeout=None):
+        if resubmit_timeout is None:
+            resubmit_timeout = api.env.replication_wait_timeout
+        return certmonger.request_and_wait_for_cert(
+            certpath=self.secdir,
+            storage='NSSDB',
+            nickname=nickname,
+            principal=principal,
+            subject=host,
+            passwd_fname=self.passwd_fname,
+            resubmit_timeout=resubmit_timeout
+        )
 
     def is_ipa_issued_cert(self, api, nickname):
         """
diff --git a/ipaserver/install/dsinstance.py b/ipaserver/install/dsinstance.py
index eefbde3356e1077d490d09c4ea47d961ce3ce8e6..ac95f8c746477da375de518526dea0d02d51d984 100644
--- a/ipaserver/install/dsinstance.py
+++ b/ipaserver/install/dsinstance.py
@@ -851,7 +851,9 @@ class DsInstance(service.Service):
                     ca='IPA',
                     profile=dogtag.DEFAULT_PROFILE,
                     dns=[self.fqdn],
-                    post_command=cmd)
+                    post_command=cmd,
+                    resubmit_timeout=api.env.replication_wait_timeout
+                )
             finally:
                 if prev_helper is not None:
                     certmonger.modify_ca_helper('IPA', prev_helper)
diff --git a/ipaserver/install/httpinstance.py b/ipaserver/install/httpinstance.py
index 0b7023c2f1b0feb996e0dd0adbefbd49c51da757..3f83248dd89118aeecfbf458c5079dde8b2cb93d 100644
--- a/ipaserver/install/httpinstance.py
+++ b/ipaserver/install/httpinstance.py
@@ -447,7 +447,10 @@ class HTTPInstance(service.Service):
                     ca='IPA',
                     profile=dogtag.DEFAULT_PROFILE,
                     dns=[self.fqdn],
-                    post_command='restart_httpd')
+                    post_command='restart_httpd',
+                    storage='NSSDB',
+                    resubmit_timeout=api.env.replication_wait_timeout
+                )
             finally:
                 if prev_helper is not None:
                     certmonger.modify_ca_helper('IPA', prev_helper)
diff --git a/ipaserver/install/krbinstance.py b/ipaserver/install/krbinstance.py
index 33d66fb94b0a1f7571b22120e5159a0e0ad2e675..09cafb7b84623594fe88083f5b914cee0f050409 100644
--- a/ipaserver/install/krbinstance.py
+++ b/ipaserver/install/krbinstance.py
@@ -445,7 +445,9 @@ class KrbInstance(service.Service):
                 storage='FILE',
                 profile=KDC_PROFILE,
                 post_command='renew_kdc_cert',
-                perms=(0o644, 0o600))
+                perms=(0o644, 0o600),
+                resubmit_timeout=api.env.replication_wait_timeout
+            )
         except dbus.DBusException as e:
             # if the certificate is already tracked, ignore the error
             name = e.get_dbus_name()
-- 
2.17.1