From 89c8f062e459a1c5083db2400b28610b85f8aa49 Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Aug 12 2021 04:22:41 +0000 Subject: import sos-4.1-5.el8 --- diff --git a/SOURCES/sos-bz1665947-rhui-plugin.patch b/SOURCES/sos-bz1665947-rhui-plugin.patch new file mode 100644 index 0000000..e884396 --- /dev/null +++ b/SOURCES/sos-bz1665947-rhui-plugin.patch @@ -0,0 +1,387 @@ +From 94b9b90c818eb18f0ca8d78fe063dc5b0677c885 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Tue, 22 Jun 2021 12:58:03 +0200 +Subject: [PATCH] [rhui] add plugin to RHUI + +Add a new/revoked plugin for RHUI (newly based on python3 and pulp-3). + +Edditionally, collect /etc/pki/pulp certificates except for RSA keys. + +Resolves: #2590 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/pulpcore.py | 7 ++++- + sos/report/plugins/rhui.py | 49 ++++++++++++++++++++++++++++++++++ + 2 files changed, 55 insertions(+), 1 deletion(-) + create mode 100644 sos/report/plugins/rhui.py + +diff --git a/sos/report/plugins/pulpcore.py b/sos/report/plugins/pulpcore.py +index ccaac3185..77ceacb92 100644 +--- a/sos/report/plugins/pulpcore.py ++++ b/sos/report/plugins/pulpcore.py +@@ -77,7 +77,12 @@ def separate_value(line, sep=':'): + def setup(self): + self.parse_settings_config() + +- self.add_copy_spec("/etc/pulp/settings.py") ++ self.add_copy_spec([ ++ "/etc/pulp/settings.py", ++ "/etc/pki/pulp/*" ++ ]) ++ # skip collecting certificate keys ++ self.add_forbidden_path("/etc/pki/pulp/*.key") + + self.add_cmd_output("rq info -u redis://localhost:6379/8", + env={"LC_ALL": "en_US.UTF-8"}, +diff --git a/sos/report/plugins/rhui.py b/sos/report/plugins/rhui.py +new file mode 100644 +index 000000000..7acd3f49e +--- /dev/null ++++ b/sos/report/plugins/rhui.py +@@ -0,0 +1,49 @@ ++# Copyright (C) 2021 Red Hat, Inc., Pavel Moravec ++ ++# This file is part of the sos project: https://github.com/sosreport/sos ++# ++# This copyrighted material is made available to anyone wishing to use, ++# modify, copy, or redistribute it subject to the terms and conditions of ++# version 2 of the GNU General Public License. ++# ++# See the LICENSE file in the source distribution for further information. ++ ++from sos.report.plugins import Plugin, RedHatPlugin ++ ++ ++class Rhui(Plugin, RedHatPlugin): ++ ++ short_desc = 'Red Hat Update Infrastructure' ++ ++ plugin_name = "rhui" ++ commands = ("rhui-manager",) ++ files = ("/etc/ansible/facts.d/rhui_auth.fact", "/usr/lib/rhui/cds.py") ++ ++ def setup(self): ++ self.add_copy_spec([ ++ "/etc/rhui/rhui-tools.conf", ++ "/etc/rhui/registered_subscriptions.conf", ++ "/etc/pki/rhui/*", ++ "/var/log/rhui-subscription-sync.log", ++ "/var/cache/rhui/*", ++ "/root/.rhui/*", ++ ]) ++ # skip collecting certificate keys ++ self.add_forbidden_path("/etc/pki/rhui/*.key") ++ ++ self.add_cmd_output([ ++ "rhui-manager status", ++ "rhui-manager cert info", ++ "ls -lR /var/lib/rhui/remote_share", ++ ]) ++ ++ def postproc(self): ++ # obfuscate admin_pw and secret_key values ++ for prop in ["admin_pw", "secret_key"]: ++ self.do_path_regex_sub( ++ "/etc/ansible/facts.d/rhui_auth.fact", ++ r"(%s\s*=\s*)(.*)" % prop, ++ r"\1********") ++ ++ ++# vim: set et ts=4 sw=4 : +From bd15dc764c9d4554d8e8f08163228d65ca099985 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Thu, 24 Jun 2021 17:53:27 +0200 +Subject: [PATCH 1/4] [plugins] Allow add_forbidden_path to apply glob + recursively + +Add option to apply glob.glob to forbidden path recursively. + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/__init__.py | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py +index 06923300..6fd1a3b2 100644 +--- a/sos/report/plugins/__init__.py ++++ b/sos/report/plugins/__init__.py +@@ -1187,12 +1187,14 @@ class Plugin(object): + 'symlink': "no" + }) + +- def add_forbidden_path(self, forbidden): ++ def add_forbidden_path(self, forbidden, recursive=False): + """Specify a path, or list of paths, to not copy, even if it's part of + an ``add_copy_spec()`` call + + :param forbidden: A filepath to forbid collection from + :type forbidden: ``str`` or a ``list`` of strings ++ ++ :param recursive: Should forbidden glob be applied recursively + """ + if isinstance(forbidden, str): + forbidden = [forbidden] +@@ -1202,7 +1204,7 @@ class Plugin(object): + + for forbid in forbidden: + self._log_info("adding forbidden path '%s'" % forbid) +- for path in glob.glob(forbid): ++ for path in glob.glob(forbid, recursive=recursive): + self.forbidden_paths.append(path) + + def get_all_options(self): +-- +2.31.1 + + +From b695201baeb629a6543445d98dbb04f357670621 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Thu, 24 Jun 2021 17:57:48 +0200 +Subject: [PATCH 2/4] [pulpcore] improve settings.py parsing + +- deal with /etc/pulp/settings.py as a one-line string +- parse dbname from it as well +- dont collect any *.key file from whole /etc/pki/pulp dir + +Related: #2593 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/pulpcore.py | 23 +++++++++++++++-------- + 1 file changed, 15 insertions(+), 8 deletions(-) + +diff --git a/sos/report/plugins/pulpcore.py b/sos/report/plugins/pulpcore.py +index 77ceacb9..be526035 100644 +--- a/sos/report/plugins/pulpcore.py ++++ b/sos/report/plugins/pulpcore.py +@@ -28,9 +28,10 @@ class PulpCore(Plugin, IndependentPlugin): + databases_scope = False + self.dbhost = "localhost" + self.dbport = 5432 ++ self.dbname = "pulpcore" + self.dbpasswd = "" + # TODO: read also redis config (we dont expect much customisations) +- # TODO: read also db user (pulp) and database name (pulpcore) ++ # TODO: read also db user (pulp) + self.staticroot = "/var/lib/pulp/assets" + self.uploaddir = "/var/lib/pulp/media/upload" + +@@ -44,7 +45,10 @@ class PulpCore(Plugin, IndependentPlugin): + return val + + try: +- for line in open("/etc/pulp/settings.py").read().splitlines(): ++ # split the lines to "one option per line" format ++ for line in open("/etc/pulp/settings.py").read() \ ++ .replace(',', ',\n').replace('{', '{\n') \ ++ .replace('}', '\n}').splitlines(): + # skip empty lines and lines with comments + if not line or line[0] == '#': + continue +@@ -53,11 +57,14 @@ class PulpCore(Plugin, IndependentPlugin): + continue + # example HOST line to parse: + # 'HOST': 'localhost', +- if databases_scope and match(r"\s+'HOST'\s*:\s+\S+", line): ++ pattern = r"\s*['|\"]%s['|\"]\s*:\s*\S+" ++ if databases_scope and match(pattern % 'HOST', line): + self.dbhost = separate_value(line) +- if databases_scope and match(r"\s+'PORT'\s*:\s+\S+", line): ++ if databases_scope and match(pattern % 'PORT', line): + self.dbport = separate_value(line) +- if databases_scope and match(r"\s+'PASSWORD'\s*:\s+\S+", line): ++ if databases_scope and match(pattern % 'NAME', line): ++ self.dbname = separate_value(line) ++ if databases_scope and match(pattern % 'PASSWORD', line): + self.dbpasswd = separate_value(line) + # if line contains closing '}' database_scope end + if databases_scope and '}' in line: +@@ -82,7 +89,7 @@ class PulpCore(Plugin, IndependentPlugin): + "/etc/pki/pulp/*" + ]) + # skip collecting certificate keys +- self.add_forbidden_path("/etc/pki/pulp/*.key") ++ self.add_forbidden_path("/etc/pki/pulp/**/*.key", recursive=True) + + self.add_cmd_output("rq info -u redis://localhost:6379/8", + env={"LC_ALL": "en_US.UTF-8"}, +@@ -104,8 +111,8 @@ class PulpCore(Plugin, IndependentPlugin): + _query = "select * from %s where pulp_last_updated > NOW() - " \ + "interval '%s days' order by pulp_last_updated" % \ + (table, task_days) +- _cmd = "psql -h %s -p %s -U pulp -d pulpcore -c %s" % \ +- (self.dbhost, self.dbport, quote(_query)) ++ _cmd = "psql -h %s -p %s -U pulp -d %s -c %s" % \ ++ (self.dbhost, self.dbport, self.dbname, quote(_query)) + self.add_cmd_output(_cmd, env=self.env, suggest_filename=table) + + def postproc(self): +-- +2.31.1 + + +From 0286034da44bce43ab368dfc6815da7d74d60719 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Thu, 24 Jun 2021 17:59:36 +0200 +Subject: [PATCH 3/4] [rhui] call rhui-* commands with proper env and timeout + +rhui-manager commands timeout when not being logged in, which +should be reacted by adding proper cmd timeout. + +Adding the env.variable ensures potentially unaswered "RHUI Username:" +is also printed/colected. + +Further, prevent collecting any *.key file from the whole /etc/pki/rhui +dir. + +Related: #2593 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/rhui.py | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/sos/report/plugins/rhui.py b/sos/report/plugins/rhui.py +index 7acd3f49..5a152427 100644 +--- a/sos/report/plugins/rhui.py ++++ b/sos/report/plugins/rhui.py +@@ -29,13 +29,16 @@ class Rhui(Plugin, RedHatPlugin): + "/root/.rhui/*", + ]) + # skip collecting certificate keys +- self.add_forbidden_path("/etc/pki/rhui/*.key") ++ self.add_forbidden_path("/etc/pki/rhui/**/*.key", recursive=True) + ++ # call rhui-manager commands with 1m timeout and ++ # with an env. variable ensuring that "RHUI Username:" ++ # even unanswered prompt gets collected + self.add_cmd_output([ + "rhui-manager status", + "rhui-manager cert info", + "ls -lR /var/lib/rhui/remote_share", +- ]) ++ ], timeout=60, env={'PYTHONUNBUFFERED': '1'}) + + def postproc(self): + # obfuscate admin_pw and secret_key values +-- +2.31.1 + + +From a656bd239ab86dfd8973f733ae2c0fbd0c57d416 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Thu, 24 Jun 2021 18:01:14 +0200 +Subject: [PATCH 4/4] [rhui] fix broken obfuscation + +- /etc/ansible/facts.d/rhui_*.fact must be collected by +rhui plugin to let some file to be obfuscated there +- obfuscate also cookies values that can grant login access + +Resolves: #2593 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/ansible.py | 3 +++ + sos/report/plugins/rhui.py | 7 +++++++ + 2 files changed, 10 insertions(+) + +diff --git a/sos/report/plugins/ansible.py b/sos/report/plugins/ansible.py +index 3e5d3d37..5991b786 100644 +--- a/sos/report/plugins/ansible.py ++++ b/sos/report/plugins/ansible.py +@@ -29,4 +29,7 @@ class Ansible(Plugin, RedHatPlugin, UbuntuPlugin): + "ansible --version" + ]) + ++ # let rhui plugin collects the RHUI specific files ++ self.add_forbidden_path("/etc/ansible/facts.d/rhui_*.fact") ++ + # vim: set et ts=4 sw=4 : +diff --git a/sos/report/plugins/rhui.py b/sos/report/plugins/rhui.py +index 5a152427..1d479f85 100644 +--- a/sos/report/plugins/rhui.py ++++ b/sos/report/plugins/rhui.py +@@ -27,6 +27,7 @@ class Rhui(Plugin, RedHatPlugin): + "/var/log/rhui-subscription-sync.log", + "/var/cache/rhui/*", + "/root/.rhui/*", ++ "/etc/ansible/facts.d/rhui_*.fact", + ]) + # skip collecting certificate keys + self.add_forbidden_path("/etc/pki/rhui/**/*.key", recursive=True) +@@ -47,6 +48,12 @@ class Rhui(Plugin, RedHatPlugin): + "/etc/ansible/facts.d/rhui_auth.fact", + r"(%s\s*=\s*)(.*)" % prop, + r"\1********") ++ # obfuscate twoo cookies for login session ++ for cookie in ["csrftoken", "sessionid"]: ++ self.do_path_regex_sub( ++ r"/root/\.rhui/.*/cookies.txt", ++ r"(%s\s+)(\S+)" % cookie, ++ r"\1********") + + + # vim: set et ts=4 sw=4 : +-- +2.31.1 + +From 4e5bebffca9936bcdf4d38aad9989970a15dd72b Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Tue, 3 Aug 2021 21:54:33 +0200 +Subject: [PATCH] [rhui] Update the plugin on several places + +- obfuscate "rhui_manager_password: xxx" in /root/.rhui/answers.yaml* +- no need to collect or obfuscate anything from /etc/ansible/facts.d +- newly detect the plugin via /etc/rhui/rhui-tools.conf file or rhui-manager + command (only) + +Resolves: #2637 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/rhui.py | 14 ++++++-------- + 1 file changed, 6 insertions(+), 8 deletions(-) + +diff --git a/sos/report/plugins/rhui.py b/sos/report/plugins/rhui.py +index 1d479f85..52065fb4 100644 +--- a/sos/report/plugins/rhui.py ++++ b/sos/report/plugins/rhui.py +@@ -16,8 +16,8 @@ class Rhui(Plugin, RedHatPlugin): + short_desc = 'Red Hat Update Infrastructure' + + plugin_name = "rhui" +- commands = ("rhui-manager",) +- files = ("/etc/ansible/facts.d/rhui_auth.fact", "/usr/lib/rhui/cds.py") ++ commands = ("rhui-manager", ) ++ files = ("/etc/rhui/rhui-tools.conf", ) + + def setup(self): + self.add_copy_spec([ +@@ -27,7 +27,6 @@ class Rhui(Plugin, RedHatPlugin): + "/var/log/rhui-subscription-sync.log", + "/var/cache/rhui/*", + "/root/.rhui/*", +- "/etc/ansible/facts.d/rhui_*.fact", + ]) + # skip collecting certificate keys + self.add_forbidden_path("/etc/pki/rhui/**/*.key", recursive=True) +@@ -42,11 +41,10 @@ class Rhui(Plugin, RedHatPlugin): + ], timeout=60, env={'PYTHONUNBUFFERED': '1'}) + + def postproc(self): +- # obfuscate admin_pw and secret_key values +- for prop in ["admin_pw", "secret_key"]: +- self.do_path_regex_sub( +- "/etc/ansible/facts.d/rhui_auth.fact", +- r"(%s\s*=\s*)(.*)" % prop, ++ # hide rhui_manager_password value in (also rotated) answers file ++ self.do_path_regex_sub( ++ r"/root/\.rhui/answers.yaml.*", ++ r"(\s*rhui_manager_password\s*:)\s*(\S+)", + r"\1********") + # obfuscate twoo cookies for login session + for cookie in ["csrftoken", "sessionid"]: +-- +2.31.1 + diff --git a/SOURCES/sos-bz1923938-sos-log-effective-options.patch b/SOURCES/sos-bz1923938-sos-log-effective-options.patch index 434a107..120df02 100644 --- a/SOURCES/sos-bz1923938-sos-log-effective-options.patch +++ b/SOURCES/sos-bz1923938-sos-log-effective-options.patch @@ -112,3 +112,173 @@ index 1eda55d6..a014a022 100644 -- 2.31.1 +From c7d3644c0c64e9e5439806250592a55c8e2de26f Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Thu, 1 Jul 2021 08:11:15 +0200 +Subject: [PATCH] [report,collect] unify --map-file arguments + +Unify --map[-file] argument among report/collect/clean. + +Resolves: #2602 + +Signed-off-by: Pavel Moravec +--- + sos/cleaner/__init__.py | 2 +- + sos/collector/__init__.py | 2 +- + sos/report/__init__.py | 2 +- + 3 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py +index 7414b55e0..4c9837826 100644 +--- a/sos/cleaner/__init__.py ++++ b/sos/cleaner/__init__.py +@@ -192,7 +192,7 @@ def add_parser_options(cls, parser): + 'file for obfuscation')) + clean_grp.add_argument('--no-update', dest='no_update', default=False, + action='store_true', +- help='Do not update the --map file with new ' ++ help='Do not update the --map-file with new ' + 'mappings from this run') + clean_grp.add_argument('--keep-binary-files', default=False, + action='store_true', +diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py +index 7b8cfcf72..6d96d6923 100644 +--- a/sos/collector/__init__.py ++++ b/sos/collector/__init__.py +@@ -427,7 +427,7 @@ def add_parser_options(cls, parser): + cleaner_grp.add_argument('--no-update', action='store_true', + default=False, dest='no_update', + help='Do not update the default cleaner map') +- cleaner_grp.add_argument('--map', dest='map_file', ++ cleaner_grp.add_argument('--map-file', dest='map_file', + default='/etc/sos/cleaner/default_mapping', + help=('Provide a previously generated mapping' + ' file for obfuscation')) +diff --git a/sos/report/__init__.py b/sos/report/__init__.py +index 7ad2d24a4..411c4eb03 100644 +--- a/sos/report/__init__.py ++++ b/sos/report/__init__.py +@@ -341,7 +341,7 @@ def add_parser_options(cls, parser): + cleaner_grp.add_argument('--no-update', action='store_true', + default=False, dest='no_update', + help='Do not update the default cleaner map') +- cleaner_grp.add_argument('--map', dest='map_file', ++ cleaner_grp.add_argument('--map-file', dest='map_file', + default='/etc/sos/cleaner/default_mapping', + help=('Provide a previously generated mapping' + ' file for obfuscation')) +From fd75745e7a5a6c5def8e6d23190227872b9912c3 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 11 Aug 2021 10:48:41 -0400 +Subject: [PATCH] [sosnode] Fix passing of plugin options when using + `--only-plugins` + +Fixes the handling of plugin options passed by `sos collect` to each +node by first aligning the SoSOption name to those of `report` +(`plugopts`), and second re-arranges the handling of plugin options and +preset options passed by the user when also using `--only-plugins` so +that the former are preserved and passed only with the `--only-plugins` +option value. + +Resolves: #2641 + +Signed-off-by: Jake Hunsaker +--- + sos/collector/__init__.py | 5 +++-- + sos/collector/sosnode.py | 34 +++++++++++++++++----------------- + 2 files changed, 20 insertions(+), 19 deletions(-) + +diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py +index 57ef074e..70b7a69e 100644 +--- a/sos/collector/__init__.py ++++ b/sos/collector/__init__.py +@@ -84,7 +84,7 @@ class SoSCollector(SoSComponent): + 'only_plugins': [], + 'password': False, + 'password_per_node': False, +- 'plugin_options': [], ++ 'plugopts': [], + 'plugin_timeout': None, + 'cmd_timeout': None, + 'preset': '', +@@ -273,7 +273,8 @@ class SoSCollector(SoSComponent): + help="chroot executed commands to SYSROOT") + sos_grp.add_argument('-e', '--enable-plugins', action="extend", + help='Enable specific plugins for sosreport') +- sos_grp.add_argument('-k', '--plugin-option', action="extend", ++ sos_grp.add_argument('-k', '--plugin-option', '--plugopts', ++ action="extend", dest='plugopts', + help='Plugin option as plugname.option=value') + sos_grp.add_argument('--log-size', default=0, type=int, + help='Limit the size of individual logs (in MiB)') +diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py +index 426edcba..5d05c297 100644 +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -667,10 +667,10 @@ class SosNode(): + + if self.cluster.sos_plugin_options: + for opt in self.cluster.sos_plugin_options: +- if not any(opt in o for o in self.plugin_options): ++ if not any(opt in o for o in self.plugopts): + option = '%s=%s' % (opt, + self.cluster.sos_plugin_options[opt]) +- self.plugin_options.append(option) ++ self.plugopts.append(option) + + # set master-only options + if self.cluster.check_node_is_master(self): +@@ -688,7 +688,7 @@ class SosNode(): + self.only_plugins = list(self.opts.only_plugins) + self.skip_plugins = list(self.opts.skip_plugins) + self.enable_plugins = list(self.opts.enable_plugins) +- self.plugin_options = list(self.opts.plugin_options) ++ self.plugopts = list(self.opts.plugopts) + self.preset = list(self.opts.preset) + + def finalize_sos_cmd(self): +@@ -754,6 +754,20 @@ class SosNode(): + os.path.join(self.host.sos_bin_path, self.sos_bin) + ) + ++ if self.plugopts: ++ opts = [o for o in self.plugopts ++ if self._plugin_exists(o.split('.')[0]) ++ and self._plugin_option_exists(o.split('=')[0])] ++ if opts: ++ sos_opts.append('-k %s' % quote(','.join(o for o in opts))) ++ ++ if self.preset: ++ if self._preset_exists(self.preset): ++ sos_opts.append('--preset=%s' % quote(self.preset)) ++ else: ++ self.log_debug('Requested to enable preset %s but preset does ' ++ 'not exist on node' % self.preset) ++ + if self.only_plugins: + plugs = [o for o in self.only_plugins if self._plugin_exists(o)] + if len(plugs) != len(self.only_plugins): +@@ -792,20 +806,6 @@ class SosNode(): + if enable: + sos_opts.append('--enable-plugins=%s' % quote(enable)) + +- if self.plugin_options: +- opts = [o for o in self.plugin_options +- if self._plugin_exists(o.split('.')[0]) +- and self._plugin_option_exists(o.split('=')[0])] +- if opts: +- sos_opts.append('-k %s' % quote(','.join(o for o in opts))) +- +- if self.preset: +- if self._preset_exists(self.preset): +- sos_opts.append('--preset=%s' % quote(self.preset)) +- else: +- self.log_debug('Requested to enable preset %s but preset does ' +- 'not exist on node' % self.preset) +- + self.sos_cmd = "%s %s" % (sos_cmd, ' '.join(sos_opts)) + self.log_info('Final sos command set to %s' % self.sos_cmd) + self.manifest.add_field('final_sos_command', self.sos_cmd) +-- +2.31.1 + diff --git a/SOURCES/sos-bz1985037-cleaner-AD-users-obfuscation.patch b/SOURCES/sos-bz1985037-cleaner-AD-users-obfuscation.patch new file mode 100644 index 0000000..2e5835a --- /dev/null +++ b/SOURCES/sos-bz1985037-cleaner-AD-users-obfuscation.patch @@ -0,0 +1,142 @@ +From 7e471676fe41dab155a939c60446cc7b7dab773b Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Tue, 20 Jul 2021 11:09:29 -0400 +Subject: [PATCH] [username parser] Load usernames from `last` for LDAP users + +AD/LDAP users are not reported into `lastlog` generally, however they +are reported in `last`. Conversely, `last` does not report local users +who have not logged in but still exist. + +In order to obfuscate both kinds of users, we need to look at both +sources. + +For this, first allow parsers to specify multiple prep files. Second, +update the username parser to search through all `lastlog` collections +as well as the `last` collection. + +Also includes a small update to the username parser's prep loading logic +to ensure we are iterating over each username discovered only once. + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/__init__.py | 38 ++++++++++++++------------ + sos/cleaner/parsers/__init__.py | 2 +- + sos/cleaner/parsers/username_parser.py | 24 +++++++++++++--- + 3 files changed, 42 insertions(+), 22 deletions(-) + +diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py +index ca5f93e5..6aadfe79 100644 +--- a/sos/cleaner/__init__.py ++++ b/sos/cleaner/__init__.py +@@ -518,23 +518,27 @@ third party. + for _parser in self.parsers: + if not _parser.prep_map_file: + continue +- _arc_path = os.path.join(_arc_name, _parser.prep_map_file) +- try: +- if is_dir: +- _pfile = open(_arc_path, 'r') +- content = _pfile.read() +- else: +- _pfile = archive.extractfile(_arc_path) +- content = _pfile.read().decode('utf-8') +- _pfile.close() +- if isinstance(_parser, SoSUsernameParser): +- _parser.load_usernames_into_map(content) +- for line in content.splitlines(): +- if isinstance(_parser, SoSHostnameParser): +- _parser.load_hostname_into_map(line) +- self.obfuscate_line(line) +- except Exception as err: +- self.log_debug("Could not prep %s: %s" % (_arc_path, err)) ++ if isinstance(_parser.prep_map_file, str): ++ _parser.prep_map_file = [_parser.prep_map_file] ++ for parse_file in _parser.prep_map_file: ++ _arc_path = os.path.join(_arc_name, parse_file) ++ try: ++ if is_dir: ++ _pfile = open(_arc_path, 'r') ++ content = _pfile.read() ++ else: ++ _pfile = archive.extractfile(_arc_path) ++ content = _pfile.read().decode('utf-8') ++ _pfile.close() ++ if isinstance(_parser, SoSUsernameParser): ++ _parser.load_usernames_into_map(content) ++ for line in content.splitlines(): ++ if isinstance(_parser, SoSHostnameParser): ++ _parser.load_hostname_into_map(line) ++ self.obfuscate_line(line) ++ except Exception as err: ++ self.log_debug("Could not prep %s: %s" ++ % (_arc_path, err)) + + def obfuscate_report(self, report): + """Individually handle each archive or directory we've discovered by +diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py +index 3076db39..af6e375e 100644 +--- a/sos/cleaner/parsers/__init__.py ++++ b/sos/cleaner/parsers/__init__.py +@@ -50,7 +50,7 @@ class SoSCleanerParser(): + skip_line_patterns = [] + skip_files = [] + map_file_key = 'unset' +- prep_map_file = 'unset' ++ prep_map_file = [] + + def __init__(self, conf_file=None): + # attempt to load previous run data into the mapping for the parser +diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py +index 96ce5f0c..b142e371 100644 +--- a/sos/cleaner/parsers/username_parser.py ++++ b/sos/cleaner/parsers/username_parser.py +@@ -25,13 +25,24 @@ class SoSUsernameParser(SoSCleanerParser + + name = 'Username Parser' + map_file_key = 'username_map' +- prep_map_file = 'sos_commands/login/lastlog_-u_1000-60000' ++ prep_map_file = [ ++ 'sos_commands/login/lastlog_-u_1000-60000', ++ 'sos_commands/login/lastlog_-u_60001-65536', ++ 'sos_commands/login/lastlog_-u_65537-4294967295', ++ # AD users will be reported here, but favor the lastlog files since ++ # those will include local users who have not logged in ++ 'sos_commands/login/last' ++ ] + regex_patterns = [] + skip_list = [ + 'core', + 'nobody', + 'nfsnobody', +- 'root' ++ 'shutdown', ++ 'reboot', ++ 'root', ++ 'ubuntu', ++ 'wtmp' + ] + + def __init__(self, conf_file=None, opt_names=None): +@@ -44,11 +54,17 @@ class SoSUsernameParser(SoSCleanerParser): + """Since we don't get the list of usernames from a straight regex for + this parser, we need to override the initial parser prepping here. + """ ++ users = set() + for line in content.splitlines()[1:]: +- user = line.split()[0] ++ try: ++ user = line.split()[0] ++ except Exception: ++ continue + if user in self.skip_list: + continue +- self.mapping.get(user) ++ users.add(user) ++ for each in users: ++ self.mapping.get(each) + + def parse_line(self, line): + count = 0 +-- +2.31.1 + diff --git a/SPECS/sos.spec b/SPECS/sos.spec index 1f5c778..30e6657 100644 --- a/SPECS/sos.spec +++ b/SPECS/sos.spec @@ -5,7 +5,7 @@ Summary: A set of tools to gather troubleshooting information from a system Name: sos Version: 4.1 -Release: 4%{?dist} +Release: 5%{?dist} Group: Applications/System Source0: https://github.com/sosreport/sos/archive/%{version}/sos-%{version}.tar.gz Source1: sos-audit-%{auditversion}.tgz @@ -40,6 +40,8 @@ Patch17: sos-bz1973675-ocp-cluster-cleaner.patch Patch18: sos-bz1923938-sos-log-effective-options.patch Patch19: sos-bz1985986-potential-issues-static-analyse.patch Patch20: sos-bz1959598-conversions-and-upgrades.patch +Patch21: sos-bz1665947-rhui-plugin.patch +Patch22: sos-bz1985037-cleaner-AD-users-obfuscation.patch %description @@ -71,6 +73,8 @@ support technicians and developers. %patch18 -p1 %patch19 -p1 %patch20 -p1 +%patch21 -p1 +%patch22 -p1 %build %py3_build @@ -137,6 +141,14 @@ of the system. Currently storage and filesystem commands are audited. %ghost /etc/audit/rules.d/40-sos-storage.rules %changelog +* Wed Aug 11 2021 Pavel Moravec = 4.1-5 +- [report,collect] unify --map-file arguments + Resolves: bz1923938 +- [rhui] add new plugin for RHUI 4 + Resolves: bz1665947 +- [username parser] Load usernames from `last` for LDAP users + Resolves: bz1985037 + * Mon Jul 26 2021 Pavel Moravec = 4.1-4 - [options] allow variant option names in config file Resolves: bz1923938