From f06a4c36834fae773da8ed429d0a91fbcda8d6aa Mon Sep 17 00:00:00 2001 From: "Endi S. Dewata" Date: Wed, 31 Oct 2018 21:56:14 +0100 Subject: [PATCH 01/26] Updated upgrade framework The upgrade framework has been updated to match PKI 10.6 which no longer requires an upgrade folder for each upgradable version. https://pagure.io/dogtagpki/issue/2686 (cherry picked from commit b4e5de9d618b57868be8b8d9a49d574ea58a7d40) --- base/common/python/pki/upgrade.py | 118 +++++++-------------- base/common/python/pki/util.py | 86 ++++++++++++++- base/common/sbin/pki-upgrade | 3 +- .../python/pki/server/deployment/pkiparser.py | 4 +- base/server/python/pki/server/upgrade.py | 10 ++ base/server/sbin/pki-server-upgrade | 19 +++- 6 files changed, 152 insertions(+), 88 deletions(-) diff --git a/base/common/python/pki/upgrade.py b/base/common/python/pki/upgrade.py index 3106c70..c2d217f 100644 --- a/base/common/python/pki/upgrade.py +++ b/base/common/python/pki/upgrade.py @@ -22,8 +22,8 @@ from __future__ import absolute_import from __future__ import print_function import functools +import logging import os -import re import shutil import traceback @@ -36,70 +36,9 @@ DEFAULT_VERSION = '10.0.0' UPGRADE_DIR = pki.SHARE_DIR + '/upgrade' BACKUP_DIR = pki.LOG_DIR + '/upgrade' SYSTEM_TRACKER = pki.CONF_DIR + '/pki.version' -verbose = False - - -@functools.total_ordering -class Version(object): - - def __init__(self, obj): - - if isinstance(obj, str): - - # parse - - pos = obj.find('-') - - if pos > 0: - self.version = obj[0:pos] - elif pos < 0: - self.version = obj - else: - raise Exception('Invalid version number: ' + obj) - - # parse .. - match = re.match(r'^(\d+)\.(\d+)\.(\d+)$', self.version) - - if match is None: - raise Exception('Invalid version number: ' + self.version) - - self.major = int(match.group(1)) - self.minor = int(match.group(2)) - self.patch = int(match.group(3)) - - elif isinstance(obj, Version): - - self.major = obj.major - self.minor = obj.minor - self.patch = obj.patch - - else: - raise Exception('Unsupported version type: ' + str(type(obj))) - - # release is ignored in comparisons - def __eq__(self, other): - return (self.major == other.major and - self.minor == other.minor and - self.patch == other.patch) - - def __lt__(self, other): - if self.major < other.major: - return True - if self.major == other.major and self.minor < other.minor: - return True - - if (self.major == other.major and - self.minor == other.minor and - self.patch < other.patch): - return True - - return False - - # not hashable - __hash__ = None - - def __repr__(self): - return self.version +logger = logging.getLogger(__name__) +verbose = False class PKIUpgradeTracker(object): @@ -203,9 +142,9 @@ class PKIUpgradeTracker(object): version = self.properties.get(self.version_key) if version: - return Version(version) + return pki.util.Version(version) - return Version(DEFAULT_VERSION) + return pki.util.Version(DEFAULT_VERSION) def set_version(self, version): @@ -479,7 +418,7 @@ class PKIUpgrader(object): if os.path.exists(self.upgrade_dir): for version in os.listdir(self.upgrade_dir): - version = Version(version) + version = pki.util.Version(version) all_versions.append(version) all_versions.sort() @@ -489,25 +428,46 @@ class PKIUpgrader(object): def versions(self): current_version = self.get_current_version() + logger.debug('Current version: %s', current_version) + target_version = self.get_target_version() + logger.debug('Target version: %s', target_version) - current_versions = [] + upgrade_path = [] for version in self.all_versions(): - # skip old versions - if version >= current_version: - current_versions.append(version) + # skip older versions + if version < current_version: + continue + + # skip newer versions + if version > target_version: + continue + + upgrade_path.append(version) - current_versions.sort() + upgrade_path.sort() + + # start from current version + if not upgrade_path or upgrade_path[0] != current_version: + upgrade_path.insert(0, current_version) + + # stop at target version + if not upgrade_path or upgrade_path[-1] != target_version: + upgrade_path.append(target_version) + + logger.debug('Upgrade path:') + for version in upgrade_path: + logger.debug(' - %s', version) versions = [] - for index, version in enumerate(current_versions): + for index, version in enumerate(upgrade_path): # link versions - if index < len(current_versions) - 1: - version.next = current_versions[index + 1] + if index < len(upgrade_path) - 1: + version.next = upgrade_path[index + 1] else: version.next = target_version @@ -587,7 +547,7 @@ class PKIUpgrader(object): return tracker.get_version() def get_target_version(self): - return Version(pki.implementation_version()) + return pki.util.Version(pki.implementation_version()) def is_complete(self): @@ -632,9 +592,6 @@ class PKIUpgrader(object): scriptlet.init() scriptlet.upgrade() - except pki.PKIException: - raise - except Exception as e: # pylint: disable=W0703 print() @@ -699,9 +656,6 @@ class PKIUpgrader(object): try: scriptlet.revert() - except pki.PKIException: - raise - except Exception as e: # pylint: disable=W0703 print() diff --git a/base/common/python/pki/util.py b/base/common/python/pki/util.py index 871c899..65a861f 100644 --- a/base/common/python/pki/util.py +++ b/base/common/python/pki/util.py @@ -24,7 +24,9 @@ Module containing utility functions and classes for the Dogtag python code from __future__ import absolute_import +import functools import os +import re import shutil from shutil import Error try: @@ -32,6 +34,7 @@ try: except ImportError: WindowsError = None +import six import subprocess DEFAULT_PKI_ENV_LIST = [ @@ -124,11 +127,14 @@ def copydirs(source, dest): def chown(path, uid, gid): """ - Change ownership of a folder and its contents. + Change ownership of a file or folder recursively. """ os.chown(path, uid, gid) + if not os.path.isdir(path): + return + for item in os.listdir(path): itempath = os.path.join(path, item) @@ -138,6 +144,25 @@ def chown(path, uid, gid): chown(itempath, uid, gid) +def chmod(path, perms): + """ + Change permissions of a file or folder recursively. + """ + + os.chmod(path, perms) + + if not os.path.isdir(path): + return + + for item in os.listdir(path): + itempath = os.path.join(path, item) + + if os.path.isfile(itempath): + os.chmod(itempath, perms) + elif os.path.isdir(itempath): + chmod(itempath, perms) + + def customize_file(input_file, output_file, params): """ Customize a file with specified parameters. @@ -275,3 +300,62 @@ def read_environment_files(env_file_list=None): if not key.strip() or key == u'_': continue os.environ[key] = value + + +@functools.total_ordering +class Version(object): + + def __init__(self, obj): + + if isinstance(obj, six.string_types): + + # parse ..[] + match = re.match(r'^(\d+)\.(\d+)\.(\d+)', obj) + + if match is None: + raise Exception('Unable to parse version number: %s' % obj) + + self.major = int(match.group(1)) + self.minor = int(match.group(2)) + self.patch = int(match.group(3)) + + elif isinstance(obj, Version): + + self.major = obj.major + self.minor = obj.minor + self.patch = obj.patch + + else: + raise Exception('Unsupported version type: %s' % type(obj)) + + # release is ignored in comparisons + def __eq__(self, other): + return (self.major == other.major and + self.minor == other.minor and + self.patch == other.patch) + + def __ne__(self, other): + return not self.__eq__(other) + + def __lt__(self, other): + if self.major < other.major: + return True + + if self.major == other.major and self.minor < other.minor: + return True + + if (self.major == other.major and + self.minor == other.minor and + self.patch < other.patch): + return True + + return False + + def __gt__(self, other): + return not self.__lt__(other) and not self.__eq__(other) + + # not hashable + __hash__ = None + + def __repr__(self): + return '%d.%d.%d' % (self.major, self.minor, self.patch) diff --git a/base/common/sbin/pki-upgrade b/base/common/sbin/pki-upgrade index 1833de8..b6bf930 100755 --- a/base/common/sbin/pki-upgrade +++ b/base/common/sbin/pki-upgrade @@ -26,6 +26,7 @@ import signal import sys import pki +import pki.util import pki.upgrade # pylint: disable=W0613 @@ -113,7 +114,7 @@ def main(argv): reset_tracker = True elif o == '--set-tracker': - tracker_version = pki.upgrade.Version(a) + tracker_version = pki.util.Version(a) elif o in ('-v', '--verbose'): pki.upgrade.verbose = True diff --git a/base/server/python/pki/server/deployment/pkiparser.py b/base/server/python/pki/server/deployment/pkiparser.py index 2ea7319..8971bb5 100644 --- a/base/server/python/pki/server/deployment/pkiparser.py +++ b/base/server/python/pki/server/deployment/pkiparser.py @@ -40,7 +40,7 @@ from six.moves.urllib.parse import urlparse # pylint: disable=F0401,E0611 # PKI Imports import pki -import pki.upgrade +import pki.util import pki.account import pki.client import pki.system @@ -337,7 +337,7 @@ class PKIConfigParser: default_http_port = '8080' default_https_port = '8443' - application_version = str(pki.upgrade.Version( + application_version = str(pki.util.Version( pki.implementation_version())) self.deployer.main_config = configparser.SafeConfigParser({ diff --git a/base/server/python/pki/server/upgrade.py b/base/server/python/pki/server/upgrade.py index 926c683..e636b8a 100644 --- a/base/server/python/pki/server/upgrade.py +++ b/base/server/python/pki/server/upgrade.py @@ -20,6 +20,7 @@ from __future__ import absolute_import from __future__ import print_function +import logging import os import traceback @@ -35,6 +36,8 @@ BACKUP_DIR = pki.LOG_DIR + '/server/upgrade' INSTANCE_TRACKER = '%s/tomcat.conf' SUBSYSTEM_TRACKER = '%s/CS.cfg' +logger = logging.getLogger(__name__) + class PKIServerUpgradeScriptlet(pki.upgrade.PKIUpgradeScriptlet): @@ -65,8 +68,11 @@ class PKIServerUpgradeScriptlet(pki.upgrade.PKIUpgradeScriptlet): tracker.set_version(self.version.next) def upgrade(self): + for instance in self.upgrader.instances(): + logger.info('Upgrading %s instance', instance.name) + self.upgrade_subsystems(instance) # If upgrading a specific subsystem don't upgrade the instance. @@ -81,6 +87,7 @@ class PKIServerUpgradeScriptlet(pki.upgrade.PKIUpgradeScriptlet): try: if verbose: print('Upgrading ' + str(instance) + ' instance.') + self.upgrade_instance(instance) self.update_server_tracker(instance) @@ -106,8 +113,11 @@ class PKIServerUpgradeScriptlet(pki.upgrade.PKIUpgradeScriptlet): 'Upgrade failed in %s: %s' % (instance, e), e, instance) def upgrade_subsystems(self, instance): + for subsystem in self.upgrader.subsystems(instance): + logger.info('Upgrading %s subsystem', subsystem.name) + if not self.can_upgrade_server(instance, subsystem): if verbose: print('Skipping ' + str(subsystem) + ' subsystem.') diff --git a/base/server/sbin/pki-server-upgrade b/base/server/sbin/pki-server-upgrade index 73e0e4a..932f1c5 100755 --- a/base/server/sbin/pki-server-upgrade +++ b/base/server/sbin/pki-server-upgrade @@ -22,6 +22,7 @@ from __future__ import absolute_import from __future__ import print_function import getopt +import logging import signal import sys @@ -29,6 +30,8 @@ import pki import pki.upgrade import pki.server.upgrade +logger = logging.getLogger('pki.server.cli.upgrade') + # pylint: disable=W0613 def interrupt_handler(event, frame): @@ -71,13 +74,15 @@ def main(argv): signal.signal(signal.SIGINT, interrupt_handler) + logging.basicConfig(format='%(levelname)s: %(message)s') + try: opts, _ = getopt.getopt(argv[1:], 'hi:s:t:vX', [ 'instance=', 'subsystem=', 'instance-type=', 'scriptlet-version=', 'scriptlet-index=', 'silent', 'status', 'revert', 'remove-tracker', 'reset-tracker', 'set-tracker=', - 'verbose', 'help']) + 'verbose', 'debug', 'help']) except getopt.GetoptError as e: print('ERROR: ' + str(e)) @@ -132,10 +137,14 @@ def main(argv): reset_tracker = True elif o == '--set-tracker': - tracker_version = pki.upgrade.Version(a) + tracker_version = pki.util.Version(a) elif o in ('-v', '--verbose'): pki.upgrade.verbose = True + logging.getLogger().setLevel(logging.INFO) + + elif o == '--debug': + logging.getLogger().setLevel(logging.DEBUG) elif o in ('-h', '--help'): usage() @@ -171,21 +180,27 @@ def main(argv): silent=silent) if status: + logger.info('Getting PKI server upgrade status') upgrader.status() elif revert: + logger.info('Reverting PKI server last upgrade') upgrader.revert() elif remove_tracker: + logger.info('Removing PKI server upgrade tracker') upgrader.remove_tracker() elif reset_tracker: + logger.info('Resetting PKI server upgrade tracker') upgrader.reset_tracker() elif tracker_version is not None: + logger.info('Setting PKI server upgrade tracker') upgrader.set_tracker(tracker_version) else: + logger.info('Upgrading PKI server') upgrader.upgrade() except pki.PKIException as e: -- 1.8.3.1 From a7e4a037ed99dfc44de67dd4396627d452c34355 Mon Sep 17 00:00:00 2001 From: "Endi S. Dewata" Date: Wed, 31 Oct 2018 22:57:17 +0100 Subject: [PATCH 02/26] Removed empty upgrade folders The empty upgrade folders have been removed since they are no longer necessary for upgrades. https://pagure.io/dogtagpki/issue/2686 (cherry picked from commit 42f14ef88deb25336563a23c67fb2ad3a3a8aa3c) --- base/common/upgrade/10.0.0/.gitignore | 4 ---- base/common/upgrade/10.0.2/.gitignore | 4 ---- base/common/upgrade/10.0.4/.gitignore | 4 ---- base/common/upgrade/10.0.5/.gitignore | 4 ---- base/common/upgrade/10.0.6/.gitignore | 4 ---- base/common/upgrade/10.1.0/.gitignore | 4 ---- base/common/upgrade/10.1.1/.gitignore | 4 ---- base/common/upgrade/10.1.2/.gitignore | 4 ---- base/common/upgrade/10.1.99/.gitignore | 4 ---- base/common/upgrade/10.2.0/.gitignore | 4 ---- base/common/upgrade/10.2.1/.gitignore | 4 ---- base/common/upgrade/10.2.2/.gitignore | 4 ---- base/common/upgrade/10.2.3/.gitignore | 4 ---- base/common/upgrade/10.2.4/.gitignore | 0 base/common/upgrade/10.2.5/.gitignore | 4 ---- base/common/upgrade/10.2.6/.gitignore | 4 ---- base/common/upgrade/10.3.0/.gitignore | 4 ---- base/common/upgrade/10.3.1/.gitignore | 4 ---- base/common/upgrade/10.3.2/.gitignore | 4 ---- base/common/upgrade/10.3.3/.gitignore | 4 ---- base/common/upgrade/10.3.4/.gitignore | 4 ---- base/common/upgrade/10.3.5/.gitignore | 4 ---- base/common/upgrade/10.4.0/.gitignore | 4 ---- base/common/upgrade/10.4.1/.gitignore | 4 ---- base/common/upgrade/10.4.2/.gitignore | 4 ---- base/common/upgrade/10.4.3/.gitignore | 4 ---- base/common/upgrade/10.4.4/.gitignore | 4 ---- base/common/upgrade/10.4.5/.gitignore | 4 ---- base/common/upgrade/10.4.6/.gitignore | 4 ---- base/server/upgrade/10.0.0/.gitignore | 4 ---- base/server/upgrade/10.0.2/.gitignore | 4 ---- base/server/upgrade/10.0.3/.gitignore | 4 ---- base/server/upgrade/10.0.4/.gitignore | 4 ---- base/server/upgrade/10.0.6/.gitignore | 4 ---- base/server/upgrade/10.1.0/.gitignore | 4 ---- base/server/upgrade/10.1.2/.gitignore | 4 ---- base/server/upgrade/10.2.0/.gitignore | 4 ---- base/server/upgrade/10.3.1/.gitignore | 4 ---- base/server/upgrade/10.3.2/.gitignore | 4 ---- base/server/upgrade/10.3.4/.gitignore | 4 ---- base/server/upgrade/10.4.1/.gitignore | 4 ---- base/server/upgrade/10.4.3/.gitignore | 4 ---- base/server/upgrade/10.4.4/.gitignore | 4 ---- base/server/upgrade/10.4.5/.gitignore | 4 ---- base/server/upgrade/10.5.1/.gitignore | 4 ---- 45 files changed, 176 deletions(-) delete mode 100644 base/common/upgrade/10.0.0/.gitignore delete mode 100644 base/common/upgrade/10.0.2/.gitignore delete mode 100644 base/common/upgrade/10.0.4/.gitignore delete mode 100644 base/common/upgrade/10.0.5/.gitignore delete mode 100644 base/common/upgrade/10.0.6/.gitignore delete mode 100644 base/common/upgrade/10.1.0/.gitignore delete mode 100644 base/common/upgrade/10.1.1/.gitignore delete mode 100644 base/common/upgrade/10.1.2/.gitignore delete mode 100644 base/common/upgrade/10.1.99/.gitignore delete mode 100644 base/common/upgrade/10.2.0/.gitignore delete mode 100644 base/common/upgrade/10.2.1/.gitignore delete mode 100644 base/common/upgrade/10.2.2/.gitignore delete mode 100644 base/common/upgrade/10.2.3/.gitignore delete mode 100644 base/common/upgrade/10.2.4/.gitignore delete mode 100644 base/common/upgrade/10.2.5/.gitignore delete mode 100644 base/common/upgrade/10.2.6/.gitignore delete mode 100644 base/common/upgrade/10.3.0/.gitignore delete mode 100644 base/common/upgrade/10.3.1/.gitignore delete mode 100644 base/common/upgrade/10.3.2/.gitignore delete mode 100644 base/common/upgrade/10.3.3/.gitignore delete mode 100644 base/common/upgrade/10.3.4/.gitignore delete mode 100644 base/common/upgrade/10.3.5/.gitignore delete mode 100644 base/common/upgrade/10.4.0/.gitignore delete mode 100644 base/common/upgrade/10.4.1/.gitignore delete mode 100644 base/common/upgrade/10.4.2/.gitignore delete mode 100644 base/common/upgrade/10.4.3/.gitignore delete mode 100644 base/common/upgrade/10.4.4/.gitignore delete mode 100644 base/common/upgrade/10.4.5/.gitignore delete mode 100644 base/common/upgrade/10.4.6/.gitignore delete mode 100644 base/server/upgrade/10.0.0/.gitignore delete mode 100644 base/server/upgrade/10.0.2/.gitignore delete mode 100644 base/server/upgrade/10.0.3/.gitignore delete mode 100644 base/server/upgrade/10.0.4/.gitignore delete mode 100644 base/server/upgrade/10.0.6/.gitignore delete mode 100644 base/server/upgrade/10.1.0/.gitignore delete mode 100644 base/server/upgrade/10.1.2/.gitignore delete mode 100644 base/server/upgrade/10.2.0/.gitignore delete mode 100644 base/server/upgrade/10.3.1/.gitignore delete mode 100644 base/server/upgrade/10.3.2/.gitignore delete mode 100644 base/server/upgrade/10.3.4/.gitignore delete mode 100644 base/server/upgrade/10.4.1/.gitignore delete mode 100644 base/server/upgrade/10.4.3/.gitignore delete mode 100644 base/server/upgrade/10.4.4/.gitignore delete mode 100644 base/server/upgrade/10.4.5/.gitignore delete mode 100644 base/server/upgrade/10.5.1/.gitignore diff --git a/base/common/upgrade/10.0.0/.gitignore b/base/common/upgrade/10.0.0/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.0.0/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.0.2/.gitignore b/base/common/upgrade/10.0.2/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.0.2/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.0.4/.gitignore b/base/common/upgrade/10.0.4/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.0.4/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.0.5/.gitignore b/base/common/upgrade/10.0.5/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.0.5/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.0.6/.gitignore b/base/common/upgrade/10.0.6/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.0.6/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.1.0/.gitignore b/base/common/upgrade/10.1.0/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.1.0/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.1.1/.gitignore b/base/common/upgrade/10.1.1/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.1.1/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.1.2/.gitignore b/base/common/upgrade/10.1.2/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.1.2/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.1.99/.gitignore b/base/common/upgrade/10.1.99/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.1.99/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.2.0/.gitignore b/base/common/upgrade/10.2.0/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.2.0/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.2.1/.gitignore b/base/common/upgrade/10.2.1/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.2.1/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.2.2/.gitignore b/base/common/upgrade/10.2.2/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.2.2/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.2.3/.gitignore b/base/common/upgrade/10.2.3/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.2.3/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.2.4/.gitignore b/base/common/upgrade/10.2.4/.gitignore deleted file mode 100644 index e69de29..0000000 diff --git a/base/common/upgrade/10.2.5/.gitignore b/base/common/upgrade/10.2.5/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.2.5/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.2.6/.gitignore b/base/common/upgrade/10.2.6/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.2.6/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.3.0/.gitignore b/base/common/upgrade/10.3.0/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.3.0/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.3.1/.gitignore b/base/common/upgrade/10.3.1/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.3.1/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.3.2/.gitignore b/base/common/upgrade/10.3.2/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.3.2/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.3.3/.gitignore b/base/common/upgrade/10.3.3/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.3.3/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.3.4/.gitignore b/base/common/upgrade/10.3.4/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.3.4/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.3.5/.gitignore b/base/common/upgrade/10.3.5/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.3.5/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.4.0/.gitignore b/base/common/upgrade/10.4.0/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.4.0/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.4.1/.gitignore b/base/common/upgrade/10.4.1/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.4.1/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.4.2/.gitignore b/base/common/upgrade/10.4.2/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.4.2/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.4.3/.gitignore b/base/common/upgrade/10.4.3/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.4.3/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.4.4/.gitignore b/base/common/upgrade/10.4.4/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.4.4/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.4.5/.gitignore b/base/common/upgrade/10.4.5/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.4.5/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/common/upgrade/10.4.6/.gitignore b/base/common/upgrade/10.4.6/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/common/upgrade/10.4.6/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.0.0/.gitignore b/base/server/upgrade/10.0.0/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.0.0/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.0.2/.gitignore b/base/server/upgrade/10.0.2/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.0.2/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.0.3/.gitignore b/base/server/upgrade/10.0.3/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.0.3/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.0.4/.gitignore b/base/server/upgrade/10.0.4/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.0.4/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.0.6/.gitignore b/base/server/upgrade/10.0.6/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.0.6/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.1.0/.gitignore b/base/server/upgrade/10.1.0/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.1.0/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.1.2/.gitignore b/base/server/upgrade/10.1.2/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.1.2/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.2.0/.gitignore b/base/server/upgrade/10.2.0/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.2.0/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.3.1/.gitignore b/base/server/upgrade/10.3.1/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.3.1/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.3.2/.gitignore b/base/server/upgrade/10.3.2/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.3.2/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.3.4/.gitignore b/base/server/upgrade/10.3.4/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.3.4/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.4.1/.gitignore b/base/server/upgrade/10.4.1/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.4.1/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.4.3/.gitignore b/base/server/upgrade/10.4.3/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.4.3/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.4.4/.gitignore b/base/server/upgrade/10.4.4/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.4.4/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.4.5/.gitignore b/base/server/upgrade/10.4.5/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.4.5/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/base/server/upgrade/10.5.1/.gitignore b/base/server/upgrade/10.5.1/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/base/server/upgrade/10.5.1/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore -- 1.8.3.1 From 8bdcb3dcb6d304604dc68e44917847b71724cde5 Mon Sep 17 00:00:00 2001 From: "Endi S. Dewata" Date: Thu, 1 Nov 2018 04:34:50 +0100 Subject: [PATCH 03/26] Updated pki-server -audit-event-find The pki-server -audit-event-find has been modified to support searching all events, enabled events, and disabled events. https://pagure.io/dogtagpki/issue/2686 (cherry picked from commit 1d7b48538cc6ede7780489cc22bc631caffebe04) --- base/server/python/pki/server/__init__.py | 95 ++++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 7 deletions(-) diff --git a/base/server/python/pki/server/__init__.py b/base/server/python/pki/server/__init__.py index b5180f0..ace98f3 100644 --- a/base/server/python/pki/server/__init__.py +++ b/base/server/python/pki/server/__init__.py @@ -428,24 +428,105 @@ class PKISubsystem(object): def find_audit_events(self, enabled=None): - if not enabled: - raise Exception('This operation is not yet supported. Specify --enabled True.') - events = [] - names = self.config['log.instance.SignedAudit.events'].split(',') - names = list(map(str.strip, names)) - names.sort() + # get enabled events + enabled_event_names = self.get_enabled_audit_events() + + if enabled is None: + # get all events + names = self.get_audit_events() + + elif enabled: # enabled == True + # get enabled events + names = enabled_event_names + + else: # enabled == False + # get all events + all_event_names = self.get_audit_events() + + # get disabled events by subtracting enabled events from all events + names = sorted(set(all_event_names) - set(enabled_event_names)) + # get event properties for name in names: event = {} event['name'] = name - event['enabled'] = True + event['enabled'] = name in enabled_event_names event['filter'] = self.config.get('log.instance.SignedAudit.filters.%s' % name) events.append(event) return events + def get_audit_events(self): + + # get the full list of audit events from LogMessages.properties + + properties = {} + tmpdir = tempfile.mkdtemp() + + try: + # export LogMessages.properties from cmsbundle.jar + cmsbundle_jar = \ + '/usr/share/pki/%s/webapps/%s/WEB-INF/lib/pki-cmsbundle.jar' \ + % (self.name, self.name) + + cmd = [ + 'jar', + 'xf', + cmsbundle_jar, + 'LogMessages.properties' + ] + + logger.debug('Command: %s', ' '.join(cmd)) + + subprocess.check_output( + cmd, + cwd=tmpdir, + stderr=subprocess.STDOUT) + + # load LogMessages.properties + log_messages_properties = os.path.join(tmpdir, 'LogMessages.properties') + pki.util.load_properties(log_messages_properties, properties) + + finally: + shutil.rmtree(tmpdir) + + # get audit events + events = set() + name_pattern = re.compile(r'LOGGING_SIGNED_AUDIT_') + value_pattern = re.compile(r':') + + for name in properties: + + name_match = name_pattern.match(name) + if not name_match: + continue + + value = properties[name] + + value_match = value_pattern.match(value) + if not value_match: + continue + + event = value_match.group(1) + events.add(event) + + return sorted(events) + + def get_enabled_audit_events(self): + + # parse enabled audit events + value = self.config['log.instance.SignedAudit.events'] + event_list = value.replace(' ', '').split(',') + + # remove duplicates + events = set() + for event in event_list: + events.add(event) + + return sorted(events) + def get_audit_log_dir(self): current_file_path = self.config['log.instance.SignedAudit.fileName'] -- 1.8.3.1 From bcc43b903a67a88c254240840e885407e7c51f3c Mon Sep 17 00:00:00 2001 From: "Endi S. Dewata" Date: Wed, 7 Nov 2018 16:53:57 +0100 Subject: [PATCH 04/26] Updated pki.util.load_properties() The pki.util.load_properties() has been modified to support multi-line property value. https://pagure.io/dogtagpki/issue/2686 (cherry picked from commit 618c5aec2cf1f16bcf30e676d3ed1f84722a32e3) --- base/common/python/pki/util.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/base/common/python/pki/util.py b/base/common/python/pki/util.py index 65a861f..a5d220f 100644 --- a/base/common/python/pki/util.py +++ b/base/common/python/pki/util.py @@ -180,22 +180,42 @@ def load_properties(filename, properties): with open(filename) as f: lines = f.read().splitlines() + name = None + multi_line = False for index, line in enumerate(lines): - line = line.strip() + if multi_line: + # append line to previous property - if not line or line.startswith('#'): - continue + value = properties[name] + value = value + line - parts = line.split('=', 1) + else: + # parse line for new property + + line = line.lstrip() + if not line or line.startswith('#'): + continue + + parts = line.split('=', 1) + if len(parts) < 2: + raise Exception('Missing delimiter in %s line %d' % + (filename, index + 1)) - if len(parts) < 2: - raise Exception('Missing delimiter in %s line %d' % - (filename, index + 1)) + name = parts[0].rstrip() + value = parts[1].lstrip() + + # check if the value is multi-line + if value.endswith('\\'): + value = value[:-1] + multi_line = True + + else: + value = value.rstrip() + multi_line = False - name = parts[0].strip() - value = parts[1].strip() + # store value in properties properties[name] = value -- 1.8.3.1 From 68427be67b3b5cf1c55b2ffe5eefd37f45dd8cab Mon Sep 17 00:00:00 2001 From: "Endi S. Dewata" Date: Fri, 9 Nov 2018 16:34:14 +0100 Subject: [PATCH 05/26] Added audit event management tools The pki-server -audit-* commands have been backported to PKI 10.5. https://pagure.io/dogtagpki/issue/2686 (cherry picked from commit adc316972072789b12ab2c2feb391bbdb01768d5) --- base/server/python/pki/server/__init__.py | 83 +++- base/server/python/pki/server/cli/audit.py | 587 ++++++++++++++++++++++++++++- 2 files changed, 662 insertions(+), 8 deletions(-) diff --git a/base/server/python/pki/server/__init__.py b/base/server/python/pki/server/__init__.py index ace98f3..6cbda2f 100644 --- a/base/server/python/pki/server/__init__.py +++ b/base/server/python/pki/server/__init__.py @@ -426,7 +426,65 @@ class PKISubsystem(object): pki.util.customize_file(input_file, output_file, params) - def find_audit_events(self, enabled=None): + def enable_audit_event(self, event_name): + + if not event_name: + raise ValueError("Please specify the Event name") + + names = self.get_audit_events() + if event_name not in names: + raise PKIServerException('Invalid audit event: %s' % event_name) + + value = self.config['log.instance.SignedAudit.events'] + events = set(value.replace(' ', '').split(',')) + + if event_name in events: + return False + + events.add(event_name) + event_list = ','.join(sorted(events)) + self.config['log.instance.SignedAudit.events'] = event_list + + return True + + def update_audit_event_filter(self, event_name, event_filter): + + if not event_name: + raise ValueError("Please specify the Event name") + + names = self.get_audit_events() + if event_name not in names: + raise PKIServerException('Invalid audit event: %s' % event_name) + + name = 'log.instance.SignedAudit.filters.%s' % event_name + + if event_filter: + self.config[name] = event_filter + else: + self.config.pop(name, None) + + def disable_audit_event(self, event_name): + + if not event_name: + raise ValueError("Please specify the Event name") + + names = self.get_audit_events() + if event_name not in names: + raise PKIServerException('Invalid audit event: %s' % event_name) + + value = self.config['log.instance.SignedAudit.events'] + events = set(value.replace(' ', '').split(',')) + + if event_name not in events: + return False + + events.remove(event_name) + event_list = ','.join(sorted(events)) + self.config['log.instance.SignedAudit.events'] = event_list + + return True + + def find_audit_event_configs(self, enabled=None): events = [] @@ -458,6 +516,22 @@ class PKISubsystem(object): return events + def get_audit_event_config(self, name): + + names = self.get_audit_events() + + if name not in names: + raise PKIServerException('Invalid audit event: %s' % name) + + enabled_event_names = self.get_enabled_audit_events() + + event = {} + event['name'] = name + event['enabled'] = name in enabled_event_names + event['filter'] = self.config.get('log.instance.SignedAudit.filters.%s' % name) + + return event + def get_audit_events(self): # get the full list of audit events from LogMessages.properties @@ -518,12 +592,7 @@ class PKISubsystem(object): # parse enabled audit events value = self.config['log.instance.SignedAudit.events'] - event_list = value.replace(' ', '').split(',') - - # remove duplicates - events = set() - for event in event_list: - events.add(event) + events = set(value.replace(' ', '').split(',')) return sorted(events) diff --git a/base/server/python/pki/server/cli/audit.py b/base/server/python/pki/server/cli/audit.py index bbbdd10..44fd86a 100644 --- a/base/server/python/pki/server/cli/audit.py +++ b/base/server/python/pki/server/cli/audit.py @@ -20,6 +20,7 @@ from __future__ import absolute_import from __future__ import print_function + import getopt import os import shutil @@ -37,10 +38,271 @@ class AuditCLI(pki.cli.CLI): 'audit', 'Audit management commands') self.parent = parent + self.add_module(AuditConfigShowCLI(self)) + self.add_module(AuditConfigModifyCLI(self)) self.add_module(AuditEventFindCLI(self)) + self.add_module(AuditEventShowCLI(self)) + self.add_module(AuditEventEnableCLI(self)) + self.add_module(AuditEventDisableCLI(self)) + self.add_module(AuditEventUpdateCLI(self)) self.add_module(AuditFileFindCLI(self)) self.add_module(AuditFileVerifyCLI(self)) + @staticmethod + def print_audit_config(subsystem): + + name = 'log.instance.SignedAudit.%s' + + enabled = subsystem.config[name % 'enable'].lower() == 'true' + + fileName = subsystem.config[name % 'fileName'] + bufferSize = subsystem.config[name % 'bufferSize'] + flushInterval = subsystem.config[name % 'flushInterval'] + + maxFileSize = subsystem.config[name % 'maxFileSize'] + rolloverInterval = subsystem.config[name % 'rolloverInterval'] + expirationTime = subsystem.config[name % 'expirationTime'] + + logSigning = subsystem.config[name % 'logSigning'].lower() == 'true' + signedAuditCertNickname = subsystem.config[name % 'signedAuditCertNickname'] + + print(' Enabled: %s' % enabled) + + print(' Log File: %s' % fileName) + print(' Buffer Size (bytes): %s' % bufferSize) + print(' Flush Interval (seconds): %s' % flushInterval) + + print(' Max File Size (bytes): %s' % maxFileSize) + print(' Rollover Interval (seconds): %s' % rolloverInterval) + print(' Expiration Time (seconds): %s' % expirationTime) + + print(' Log Signing: %s' % logSigning) + print(' Signing Certificate: %s' % signedAuditCertNickname) + + @staticmethod + def print_audit_event_config(event): + print(' Event Name: %s' % event.get('name')) + print(' Enabled: %s' % event.get('enabled')) + print(' Filter: %s' % event.get('filter')) + + +class AuditConfigShowCLI(pki.cli.CLI): + + def __init__(self, parent): + super(AuditConfigShowCLI, self).__init__( + 'config-show', 'Display audit configuration') + self.parent = parent + + def print_help(self): + print('Usage: pki-server %s-audit-config-show [OPTIONS]' % self.parent.parent.name) + print() + print(' -i, --instance Instance ID (default: pki-tomcat).') + print(' --help Show help message.') + print() + + def execute(self, argv): + try: + opts, _ = getopt.gnu_getopt(argv, 'i:v', [ + 'instance=', + 'verbose', 'help']) + + except getopt.GetoptError as e: + print('ERROR: ' + str(e)) + self.print_help() + sys.exit(1) + + instance_name = 'pki-tomcat' + + for o, a in opts: + if o in ('-i', '--instance'): + instance_name = a + + elif o == '--help': + self.print_help() + sys.exit() + + else: + print('ERROR: unknown option ' + o) + self.print_help() + sys.exit(1) + + instance = pki.server.PKIInstance(instance_name) + if not instance.is_valid(): + print('ERROR: Invalid instance %s.' % instance_name) + sys.exit(1) + + instance.load() + + subsystem_name = self.parent.parent.name + subsystem = instance.get_subsystem(subsystem_name) + + if not subsystem: + print('ERROR: No %s subsystem in instance %s.' + % (subsystem_name.upper(), instance_name)) + sys.exit(1) + + AuditCLI.print_audit_config(subsystem) + + +class AuditConfigModifyCLI(pki.cli.CLI): + + def __init__(self, parent): + super(AuditConfigModifyCLI, self).__init__( + 'config-mod', 'Modify audit configuration') + self.parent = parent + + def print_help(self): + print('Usage: pki-server %s-audit-config-mod [OPTIONS]' % self.parent.parent.name) + print() + print(' -i, --instance Instance ID (default: pki-tomcat).') + print(' --enabled Enable/disable audit logging.') + print(' --logFile Set log file.') + print(' --bufferSize Set buffer size (bytes).') + print(' --flushInterval Set flush interval (seconds).') + print(' --maxFileSize Set maximum file size (bytes).') + print(' --rolloverInterval Set rollover interval (seconds).') + print(' --expirationTime