From 83445df2894426cdaf2f09f8bf2ac4c829922620 Mon Sep 17 00:00:00 2001

From: Christian Heimes <cheimes@redhat.com>

Date: Sun, 8 Jul 2018 11:53:58 +0200

Subject: [PATCH] Auto-retry failed certmonger requests

During parallel replica installation, a request sometimes fails with

CA_REJECTED or CA_UNREACHABLE. The error occur when the master is

either busy or some information haven't been replicated yet. Even

a stuck request can be recovered, e.g. when permission and group

information have been replicated.

A new function request_and_retry_cert() automatically resubmits failing

requests until it times out.

Fixes: https://pagure.io/freeipa/issue/7623

Signed-off-by: Christian Heimes <cheimes@redhat.com>

Reviewed-By: Tibor Dudlak <tdudlak@redhat.com>

---

 ipalib/install/certmonger.py      | 64 ++++++++++++++++++++++++-------

 ipaserver/install/cainstance.py   |  4 +-

 ipaserver/install/certs.py        | 19 ++++++---

 ipaserver/install/dsinstance.py   |  4 +-

 ipaserver/install/httpinstance.py |  5 ++-

 ipaserver/install/krbinstance.py  |  4 +-

 6 files changed, 76 insertions(+), 24 deletions(-)

diff --git a/ipalib/install/certmonger.py b/ipalib/install/certmonger.py

index d2b782ddb0c746a3dfd96d0222bb31c6a960fdff..d915cbf7638c68f3ad5170b2878a706ad39def62 100644

--- a/ipalib/install/certmonger.py

+++ b/ipalib/install/certmonger.py

@@ -302,20 +302,56 @@ def add_subject(request_id, subject):

 def request_and_wait_for_cert(

         certpath, subject, principal, nickname=None, passwd_fname=None,

         dns=None, ca='IPA', profile=None,

-        pre_command=None, post_command=None, storage='NSSDB', perms=None):

-    """

-    Execute certmonger to request a server certificate.

-

-    The method also waits for the certificate to be available.

-    """

-    reqId = request_cert(certpath, subject, principal, nickname,

-                         passwd_fname, dns, ca, profile,

-                         pre_command, post_command, storage, perms)

-    state = wait_for_request(reqId, api.env.startup_timeout)

-    ca_error = get_request_value(reqId, 'ca-error')

-    if state != 'MONITORING' or ca_error:

-        raise RuntimeError("Certificate issuance failed ({})".format(state))

-    return reqId

+        pre_command=None, post_command=None, storage='NSSDB', perms=None,

+        resubmit_timeout=0):

+    """Request certificate, wait and possibly resubmit failing requests

+

+    Submit a cert request to certmonger and wait until the request has

+    finished.

+

+    With timeout, a failed request is resubmitted. During parallel replica

+    installation, a request sometimes fails with CA_REJECTED or

+    CA_UNREACHABLE. The error occurs when the master is either busy or some

+    information haven't been replicated yet. Even a stuck request can be

+    recovered, e.g. when permission and group information have been

+    replicated.

+    """

+    req_id = request_cert(

+        certpath, subject, principal, nickname, passwd_fname, dns, ca,

+        profile, pre_command, post_command, storage, perms

+    )

+

+    deadline = time.time() + resubmit_timeout

+    while True:  # until success, timeout, or error

+        state = wait_for_request(req_id, api.env.replication_wait_timeout)

+        ca_error = get_request_value(req_id, 'ca-error')

+        if state == 'MONITORING' and ca_error is None:

+            # we got a winner, exiting

+            root_logger.debug("Cert request %s was successful", req_id)

+            return req_id

+

+        root_logger.debug(

+            "Cert request %s failed: %s (%s)", req_id, state, ca_error

+        )

+        if state not in {'CA_REJECTED', 'CA_UNREACHABLE'}:

+            # probably unrecoverable error

+            root_logger.debug("Giving up on cert request %s", req_id)

+            break

+        elif not resubmit_timeout:

+            # no resubmit

+            break

+        elif time.time() > deadline:

+            root_logger.debug("Request %s reached resubmit dead line", req_id)

+            break

+        else:

+            # sleep and resubmit

+            root_logger.debug("Sleep and resubmit cert request %s", req_id)

+            time.sleep(10)

+            resubmit_request(req_id)

+

+    raise RuntimeError(

+        "Certificate issuance failed ({}: {})".format(state, ca_error)

+    )

 def request_cert(

diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py

index 62e9ad7de6f00eabb48f726a3931eb8acf0ba22b..e207911814e3553c5aa5310694170d3575337c55 100644

--- a/ipaserver/install/cainstance.py

+++ b/ipaserver/install/cainstance.py

@@ -863,7 +863,9 @@ class CAInstance(DogtagInstance):

                 profile='caServerCert',

                 pre_command='renew_ra_cert_pre',

                 post_command='renew_ra_cert',

-                storage="FILE")

+                storage="FILE",

+                resubmit_timeout=api.env.replication_wait_timeout

+            )

             self.__set_ra_cert_perms()

             self.requestId = str(reqId)

diff --git a/ipaserver/install/certs.py b/ipaserver/install/certs.py

index de96318db51b03f2515814d574cfebf1b242b6a6..5670d468bb1b168af7ada7b2d8924b8ec9f5d9c1 100644

--- a/ipaserver/install/certs.py

+++ b/ipaserver/install/certs.py

@@ -663,12 +663,19 @@ class CertDB(object):

     def export_pem_cert(self, nickname, location):

         return self.nssdb.export_pem_cert(nickname, location)

-    def request_service_cert(self, nickname, principal, host):

-        certmonger.request_and_wait_for_cert(certpath=self.secdir,

-                                             nickname=nickname,

-                                             principal=principal,

-                                             subject=host,

-                                             passwd_fname=self.passwd_fname)

+    def request_service_cert(self, nickname, principal, host,

+                             resubmit_timeout=None):

+        if resubmit_timeout is None:

+            resubmit_timeout = api.env.replication_wait_timeout

+        return certmonger.request_and_wait_for_cert(

+            certpath=self.secdir,

+            storage='NSSDB',

+            nickname=nickname,

+            principal=principal,

+            subject=host,

+            passwd_fname=self.passwd_fname,

+            resubmit_timeout=resubmit_timeout

+        )

     def is_ipa_issued_cert(self, api, nickname):

         """

diff --git a/ipaserver/install/dsinstance.py b/ipaserver/install/dsinstance.py

index 7adaabd3c1280709150329003130f70233de37f4..8ec5cdd7a4a324a1a40dbe968defcc797db8f054 100644

--- a/ipaserver/install/dsinstance.py

+++ b/ipaserver/install/dsinstance.py

@@ -847,7 +847,9 @@ class DsInstance(service.Service):

                     ca='IPA',

                     profile=dogtag.DEFAULT_PROFILE,

                     dns=[self.fqdn],

-                    post_command=cmd)

+                    post_command=cmd,

+                    resubmit_timeout=api.env.replication_wait_timeout

+                )

             finally:

                 if prev_helper is not None:

                     certmonger.modify_ca_helper('IPA', prev_helper)

diff --git a/ipaserver/install/httpinstance.py b/ipaserver/install/httpinstance.py

index 2df51eaa77d3ee3246027a6bcbc4023dbad61160..2366698bb0d6a7c9cd481ba9d5568d320742f6bb 100644

--- a/ipaserver/install/httpinstance.py

+++ b/ipaserver/install/httpinstance.py

@@ -450,7 +450,10 @@ class HTTPInstance(service.Service):

                     ca='IPA',

                     profile=dogtag.DEFAULT_PROFILE,

                     dns=[self.fqdn],

-                    post_command='restart_httpd')

+                    post_command='restart_httpd',

+                    storage='NSSDB',

+                    resubmit_timeout=api.env.replication_wait_timeout

+                )

             finally:

                 if prev_helper is not None:

                     certmonger.modify_ca_helper('IPA', prev_helper)

diff --git a/ipaserver/install/krbinstance.py b/ipaserver/install/krbinstance.py

index 4041d1b5fb3c3cf3db78b6cb282ce5f17793a0e3..62de118b7104fe7d72d2c1fd5577e9a76000c663 100644

--- a/ipaserver/install/krbinstance.py

+++ b/ipaserver/install/krbinstance.py

@@ -436,7 +436,9 @@ class KrbInstance(service.Service):

                 storage='FILE',

                 profile=KDC_PROFILE,

                 post_command='renew_kdc_cert',

-                perms=(0o644, 0o600))

+                perms=(0o644, 0o600),

+                resubmit_timeout=api.env.replication_wait_timeout

+            )

         except dbus.DBusException as e:

             # if the certificate is already tracked, ignore the error

             name = e.get_dbus_name()

-- 

2.17.1