From ba407dc816dbcb49526039afc8852f409d678bab Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Nov 09 2021 10:04:23 +0000 Subject: import sos-4.1-5.el8 --- diff --git a/.gitignore b/.gitignore index 7b4b434..0f2b270 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -SOURCES/sos-4.0.tar.gz +SOURCES/sos-4.1.tar.gz SOURCES/sos-audit-0.3.tgz diff --git a/.sos.metadata b/.sos.metadata index bd28d9a..950a640 100644 --- a/.sos.metadata +++ b/.sos.metadata @@ -1,2 +1,2 @@ -f4850f7d3a4cd3e52f58bbc408e8d5e17df04741 SOURCES/sos-4.0.tar.gz +7d4d03af232e2357e3359ad564a59f4c3654eac0 SOURCES/sos-4.1.tar.gz 9d478b9f0085da9178af103078bbf2fd77b0175a SOURCES/sos-audit-0.3.tgz diff --git a/README.debrand b/README.debrand deleted file mode 100644 index 01c46d2..0000000 --- a/README.debrand +++ /dev/null @@ -1,2 +0,0 @@ -Warning: This package was configured for automatic debranding, but the changes -failed to apply. diff --git a/SOURCES/sos-bz1665947-rhui-plugin.patch b/SOURCES/sos-bz1665947-rhui-plugin.patch new file mode 100644 index 0000000..e884396 --- /dev/null +++ b/SOURCES/sos-bz1665947-rhui-plugin.patch @@ -0,0 +1,387 @@ +From 94b9b90c818eb18f0ca8d78fe063dc5b0677c885 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Tue, 22 Jun 2021 12:58:03 +0200 +Subject: [PATCH] [rhui] add plugin to RHUI + +Add a new/revoked plugin for RHUI (newly based on python3 and pulp-3). + +Edditionally, collect /etc/pki/pulp certificates except for RSA keys. + +Resolves: #2590 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/pulpcore.py | 7 ++++- + sos/report/plugins/rhui.py | 49 ++++++++++++++++++++++++++++++++++ + 2 files changed, 55 insertions(+), 1 deletion(-) + create mode 100644 sos/report/plugins/rhui.py + +diff --git a/sos/report/plugins/pulpcore.py b/sos/report/plugins/pulpcore.py +index ccaac3185..77ceacb92 100644 +--- a/sos/report/plugins/pulpcore.py ++++ b/sos/report/plugins/pulpcore.py +@@ -77,7 +77,12 @@ def separate_value(line, sep=':'): + def setup(self): + self.parse_settings_config() + +- self.add_copy_spec("/etc/pulp/settings.py") ++ self.add_copy_spec([ ++ "/etc/pulp/settings.py", ++ "/etc/pki/pulp/*" ++ ]) ++ # skip collecting certificate keys ++ self.add_forbidden_path("/etc/pki/pulp/*.key") + + self.add_cmd_output("rq info -u redis://localhost:6379/8", + env={"LC_ALL": "en_US.UTF-8"}, +diff --git a/sos/report/plugins/rhui.py b/sos/report/plugins/rhui.py +new file mode 100644 +index 000000000..7acd3f49e +--- /dev/null ++++ b/sos/report/plugins/rhui.py +@@ -0,0 +1,49 @@ ++# Copyright (C) 2021 Red Hat, Inc., Pavel Moravec ++ ++# This file is part of the sos project: https://github.com/sosreport/sos ++# ++# This copyrighted material is made available to anyone wishing to use, ++# modify, copy, or redistribute it subject to the terms and conditions of ++# version 2 of the GNU General Public License. ++# ++# See the LICENSE file in the source distribution for further information. ++ ++from sos.report.plugins import Plugin, RedHatPlugin ++ ++ ++class Rhui(Plugin, RedHatPlugin): ++ ++ short_desc = 'Red Hat Update Infrastructure' ++ ++ plugin_name = "rhui" ++ commands = ("rhui-manager",) ++ files = ("/etc/ansible/facts.d/rhui_auth.fact", "/usr/lib/rhui/cds.py") ++ ++ def setup(self): ++ self.add_copy_spec([ ++ "/etc/rhui/rhui-tools.conf", ++ "/etc/rhui/registered_subscriptions.conf", ++ "/etc/pki/rhui/*", ++ "/var/log/rhui-subscription-sync.log", ++ "/var/cache/rhui/*", ++ "/root/.rhui/*", ++ ]) ++ # skip collecting certificate keys ++ self.add_forbidden_path("/etc/pki/rhui/*.key") ++ ++ self.add_cmd_output([ ++ "rhui-manager status", ++ "rhui-manager cert info", ++ "ls -lR /var/lib/rhui/remote_share", ++ ]) ++ ++ def postproc(self): ++ # obfuscate admin_pw and secret_key values ++ for prop in ["admin_pw", "secret_key"]: ++ self.do_path_regex_sub( ++ "/etc/ansible/facts.d/rhui_auth.fact", ++ r"(%s\s*=\s*)(.*)" % prop, ++ r"\1********") ++ ++ ++# vim: set et ts=4 sw=4 : +From bd15dc764c9d4554d8e8f08163228d65ca099985 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Thu, 24 Jun 2021 17:53:27 +0200 +Subject: [PATCH 1/4] [plugins] Allow add_forbidden_path to apply glob + recursively + +Add option to apply glob.glob to forbidden path recursively. + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/__init__.py | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py +index 06923300..6fd1a3b2 100644 +--- a/sos/report/plugins/__init__.py ++++ b/sos/report/plugins/__init__.py +@@ -1187,12 +1187,14 @@ class Plugin(object): + 'symlink': "no" + }) + +- def add_forbidden_path(self, forbidden): ++ def add_forbidden_path(self, forbidden, recursive=False): + """Specify a path, or list of paths, to not copy, even if it's part of + an ``add_copy_spec()`` call + + :param forbidden: A filepath to forbid collection from + :type forbidden: ``str`` or a ``list`` of strings ++ ++ :param recursive: Should forbidden glob be applied recursively + """ + if isinstance(forbidden, str): + forbidden = [forbidden] +@@ -1202,7 +1204,7 @@ class Plugin(object): + + for forbid in forbidden: + self._log_info("adding forbidden path '%s'" % forbid) +- for path in glob.glob(forbid): ++ for path in glob.glob(forbid, recursive=recursive): + self.forbidden_paths.append(path) + + def get_all_options(self): +-- +2.31.1 + + +From b695201baeb629a6543445d98dbb04f357670621 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Thu, 24 Jun 2021 17:57:48 +0200 +Subject: [PATCH 2/4] [pulpcore] improve settings.py parsing + +- deal with /etc/pulp/settings.py as a one-line string +- parse dbname from it as well +- dont collect any *.key file from whole /etc/pki/pulp dir + +Related: #2593 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/pulpcore.py | 23 +++++++++++++++-------- + 1 file changed, 15 insertions(+), 8 deletions(-) + +diff --git a/sos/report/plugins/pulpcore.py b/sos/report/plugins/pulpcore.py +index 77ceacb9..be526035 100644 +--- a/sos/report/plugins/pulpcore.py ++++ b/sos/report/plugins/pulpcore.py +@@ -28,9 +28,10 @@ class PulpCore(Plugin, IndependentPlugin): + databases_scope = False + self.dbhost = "localhost" + self.dbport = 5432 ++ self.dbname = "pulpcore" + self.dbpasswd = "" + # TODO: read also redis config (we dont expect much customisations) +- # TODO: read also db user (pulp) and database name (pulpcore) ++ # TODO: read also db user (pulp) + self.staticroot = "/var/lib/pulp/assets" + self.uploaddir = "/var/lib/pulp/media/upload" + +@@ -44,7 +45,10 @@ class PulpCore(Plugin, IndependentPlugin): + return val + + try: +- for line in open("/etc/pulp/settings.py").read().splitlines(): ++ # split the lines to "one option per line" format ++ for line in open("/etc/pulp/settings.py").read() \ ++ .replace(',', ',\n').replace('{', '{\n') \ ++ .replace('}', '\n}').splitlines(): + # skip empty lines and lines with comments + if not line or line[0] == '#': + continue +@@ -53,11 +57,14 @@ class PulpCore(Plugin, IndependentPlugin): + continue + # example HOST line to parse: + # 'HOST': 'localhost', +- if databases_scope and match(r"\s+'HOST'\s*:\s+\S+", line): ++ pattern = r"\s*['|\"]%s['|\"]\s*:\s*\S+" ++ if databases_scope and match(pattern % 'HOST', line): + self.dbhost = separate_value(line) +- if databases_scope and match(r"\s+'PORT'\s*:\s+\S+", line): ++ if databases_scope and match(pattern % 'PORT', line): + self.dbport = separate_value(line) +- if databases_scope and match(r"\s+'PASSWORD'\s*:\s+\S+", line): ++ if databases_scope and match(pattern % 'NAME', line): ++ self.dbname = separate_value(line) ++ if databases_scope and match(pattern % 'PASSWORD', line): + self.dbpasswd = separate_value(line) + # if line contains closing '}' database_scope end + if databases_scope and '}' in line: +@@ -82,7 +89,7 @@ class PulpCore(Plugin, IndependentPlugin): + "/etc/pki/pulp/*" + ]) + # skip collecting certificate keys +- self.add_forbidden_path("/etc/pki/pulp/*.key") ++ self.add_forbidden_path("/etc/pki/pulp/**/*.key", recursive=True) + + self.add_cmd_output("rq info -u redis://localhost:6379/8", + env={"LC_ALL": "en_US.UTF-8"}, +@@ -104,8 +111,8 @@ class PulpCore(Plugin, IndependentPlugin): + _query = "select * from %s where pulp_last_updated > NOW() - " \ + "interval '%s days' order by pulp_last_updated" % \ + (table, task_days) +- _cmd = "psql -h %s -p %s -U pulp -d pulpcore -c %s" % \ +- (self.dbhost, self.dbport, quote(_query)) ++ _cmd = "psql -h %s -p %s -U pulp -d %s -c %s" % \ ++ (self.dbhost, self.dbport, self.dbname, quote(_query)) + self.add_cmd_output(_cmd, env=self.env, suggest_filename=table) + + def postproc(self): +-- +2.31.1 + + +From 0286034da44bce43ab368dfc6815da7d74d60719 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Thu, 24 Jun 2021 17:59:36 +0200 +Subject: [PATCH 3/4] [rhui] call rhui-* commands with proper env and timeout + +rhui-manager commands timeout when not being logged in, which +should be reacted by adding proper cmd timeout. + +Adding the env.variable ensures potentially unaswered "RHUI Username:" +is also printed/colected. + +Further, prevent collecting any *.key file from the whole /etc/pki/rhui +dir. + +Related: #2593 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/rhui.py | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/sos/report/plugins/rhui.py b/sos/report/plugins/rhui.py +index 7acd3f49..5a152427 100644 +--- a/sos/report/plugins/rhui.py ++++ b/sos/report/plugins/rhui.py +@@ -29,13 +29,16 @@ class Rhui(Plugin, RedHatPlugin): + "/root/.rhui/*", + ]) + # skip collecting certificate keys +- self.add_forbidden_path("/etc/pki/rhui/*.key") ++ self.add_forbidden_path("/etc/pki/rhui/**/*.key", recursive=True) + ++ # call rhui-manager commands with 1m timeout and ++ # with an env. variable ensuring that "RHUI Username:" ++ # even unanswered prompt gets collected + self.add_cmd_output([ + "rhui-manager status", + "rhui-manager cert info", + "ls -lR /var/lib/rhui/remote_share", +- ]) ++ ], timeout=60, env={'PYTHONUNBUFFERED': '1'}) + + def postproc(self): + # obfuscate admin_pw and secret_key values +-- +2.31.1 + + +From a656bd239ab86dfd8973f733ae2c0fbd0c57d416 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Thu, 24 Jun 2021 18:01:14 +0200 +Subject: [PATCH 4/4] [rhui] fix broken obfuscation + +- /etc/ansible/facts.d/rhui_*.fact must be collected by +rhui plugin to let some file to be obfuscated there +- obfuscate also cookies values that can grant login access + +Resolves: #2593 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/ansible.py | 3 +++ + sos/report/plugins/rhui.py | 7 +++++++ + 2 files changed, 10 insertions(+) + +diff --git a/sos/report/plugins/ansible.py b/sos/report/plugins/ansible.py +index 3e5d3d37..5991b786 100644 +--- a/sos/report/plugins/ansible.py ++++ b/sos/report/plugins/ansible.py +@@ -29,4 +29,7 @@ class Ansible(Plugin, RedHatPlugin, UbuntuPlugin): + "ansible --version" + ]) + ++ # let rhui plugin collects the RHUI specific files ++ self.add_forbidden_path("/etc/ansible/facts.d/rhui_*.fact") ++ + # vim: set et ts=4 sw=4 : +diff --git a/sos/report/plugins/rhui.py b/sos/report/plugins/rhui.py +index 5a152427..1d479f85 100644 +--- a/sos/report/plugins/rhui.py ++++ b/sos/report/plugins/rhui.py +@@ -27,6 +27,7 @@ class Rhui(Plugin, RedHatPlugin): + "/var/log/rhui-subscription-sync.log", + "/var/cache/rhui/*", + "/root/.rhui/*", ++ "/etc/ansible/facts.d/rhui_*.fact", + ]) + # skip collecting certificate keys + self.add_forbidden_path("/etc/pki/rhui/**/*.key", recursive=True) +@@ -47,6 +48,12 @@ class Rhui(Plugin, RedHatPlugin): + "/etc/ansible/facts.d/rhui_auth.fact", + r"(%s\s*=\s*)(.*)" % prop, + r"\1********") ++ # obfuscate twoo cookies for login session ++ for cookie in ["csrftoken", "sessionid"]: ++ self.do_path_regex_sub( ++ r"/root/\.rhui/.*/cookies.txt", ++ r"(%s\s+)(\S+)" % cookie, ++ r"\1********") + + + # vim: set et ts=4 sw=4 : +-- +2.31.1 + +From 4e5bebffca9936bcdf4d38aad9989970a15dd72b Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Tue, 3 Aug 2021 21:54:33 +0200 +Subject: [PATCH] [rhui] Update the plugin on several places + +- obfuscate "rhui_manager_password: xxx" in /root/.rhui/answers.yaml* +- no need to collect or obfuscate anything from /etc/ansible/facts.d +- newly detect the plugin via /etc/rhui/rhui-tools.conf file or rhui-manager + command (only) + +Resolves: #2637 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/rhui.py | 14 ++++++-------- + 1 file changed, 6 insertions(+), 8 deletions(-) + +diff --git a/sos/report/plugins/rhui.py b/sos/report/plugins/rhui.py +index 1d479f85..52065fb4 100644 +--- a/sos/report/plugins/rhui.py ++++ b/sos/report/plugins/rhui.py +@@ -16,8 +16,8 @@ class Rhui(Plugin, RedHatPlugin): + short_desc = 'Red Hat Update Infrastructure' + + plugin_name = "rhui" +- commands = ("rhui-manager",) +- files = ("/etc/ansible/facts.d/rhui_auth.fact", "/usr/lib/rhui/cds.py") ++ commands = ("rhui-manager", ) ++ files = ("/etc/rhui/rhui-tools.conf", ) + + def setup(self): + self.add_copy_spec([ +@@ -27,7 +27,6 @@ class Rhui(Plugin, RedHatPlugin): + "/var/log/rhui-subscription-sync.log", + "/var/cache/rhui/*", + "/root/.rhui/*", +- "/etc/ansible/facts.d/rhui_*.fact", + ]) + # skip collecting certificate keys + self.add_forbidden_path("/etc/pki/rhui/**/*.key", recursive=True) +@@ -42,11 +41,10 @@ class Rhui(Plugin, RedHatPlugin): + ], timeout=60, env={'PYTHONUNBUFFERED': '1'}) + + def postproc(self): +- # obfuscate admin_pw and secret_key values +- for prop in ["admin_pw", "secret_key"]: +- self.do_path_regex_sub( +- "/etc/ansible/facts.d/rhui_auth.fact", +- r"(%s\s*=\s*)(.*)" % prop, ++ # hide rhui_manager_password value in (also rotated) answers file ++ self.do_path_regex_sub( ++ r"/root/\.rhui/answers.yaml.*", ++ r"(\s*rhui_manager_password\s*:)\s*(\S+)", + r"\1********") + # obfuscate twoo cookies for login session + for cookie in ["csrftoken", "sessionid"]: +-- +2.31.1 + diff --git a/SOURCES/sos-bz1827801-streamlined-sanitize_item.patch b/SOURCES/sos-bz1827801-streamlined-sanitize_item.patch deleted file mode 100644 index ffae766..0000000 --- a/SOURCES/sos-bz1827801-streamlined-sanitize_item.patch +++ /dev/null @@ -1,37 +0,0 @@ -From 086c1c5ca52b0ed8b810ad5a293a574ba990e635 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 13 Oct 2020 20:14:35 +0200 -Subject: [PATCH] [cleaner] more streamlined sanitize_item method - -Remove a duplicate call in both IF branches. - -Resolves: #2272 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/mappings/ip_map.py | 3 +-- - 1 file changed, 1 insertion(+), 2 deletions(-) - -diff --git a/sos/cleaner/mappings/ip_map.py b/sos/cleaner/mappings/ip_map.py -index 45fd9739..e6dffd60 100644 ---- a/sos/cleaner/mappings/ip_map.py -+++ b/sos/cleaner/mappings/ip_map.py -@@ -121,13 +121,12 @@ class SoSIPMap(SoSMap): - # network and if it has, replace the default /32 netmask that - # ipaddress applies to no CIDR-notated addresses - self.set_ip_cidr_from_existing_subnet(addr) -- return self.sanitize_ipaddr(addr) - else: - # we have a CIDR notation, so generate an obfuscated network - # address and then generate an IP address within that network's - # range - self.sanitize_network(network) -- return self.sanitize_ipaddr(addr) -+ return self.sanitize_ipaddr(addr) - - def sanitize_network(self, network): - """Obfuscate the network address provided, and if there are host bits --- -2.26.2 - diff --git a/SOURCES/sos-bz1848095-collect-rhev-pki.patch b/SOURCES/sos-bz1848095-collect-rhev-pki.patch deleted file mode 100644 index 959f58c..0000000 --- a/SOURCES/sos-bz1848095-collect-rhev-pki.patch +++ /dev/null @@ -1,42 +0,0 @@ -From bbd28011f8bb710d64283bd6d3ec68c0fb5430b4 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Fri, 6 Nov 2020 21:28:37 +0100 -Subject: [PATCH] [ovirt] collect /etc/pki/ovirt-engine/.truststore - -.truststore contains useful public CAs but a_c_s skips collecting -that hidden file. - -Closes: #2296 -Resolves: #2297 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/ovirt.py | 7 +++++-- - 1 file changed, 5 insertions(+), 2 deletions(-) - -diff --git a/sos/report/plugins/ovirt.py b/sos/report/plugins/ovirt.py -index 4c112cac..127c971c 100644 ---- a/sos/report/plugins/ovirt.py -+++ b/sos/report/plugins/ovirt.py -@@ -137,12 +137,15 @@ class Ovirt(Plugin, RedHatPlugin): - "/var/lib/ovirt-engine-reports/jboss_runtime/config" - ]) - -- # Copying host certs. -+ # Copying host certs; extra copy the hidden .truststore file - self.add_forbidden_path([ - "/etc/pki/ovirt-engine/keys", - "/etc/pki/ovirt-engine/private" - ]) -- self.add_copy_spec("/etc/pki/ovirt-engine/") -+ self.add_copy_spec([ -+ "/etc/pki/ovirt-engine/", -+ "/etc/pki/ovirt-engine/.truststore", -+ ]) - - def postproc(self): - """ --- -2.26.2 - diff --git a/SOURCES/sos-bz1874295-osp-ironic-inspector-configs.patch b/SOURCES/sos-bz1874295-osp-ironic-inspector-configs.patch deleted file mode 100644 index b9be279..0000000 --- a/SOURCES/sos-bz1874295-osp-ironic-inspector-configs.patch +++ /dev/null @@ -1,179 +0,0 @@ -From 4e46e0c8db3e2ecea7279ae7a781ae2e22a81b69 Mon Sep 17 00:00:00 2001 -From: David Vallee Delisle -Date: Mon, 31 Aug 2020 18:58:42 -0400 -Subject: [PATCH] [openstack_ironic] Missing ironic-inspector configs - -We're missing the ironic-inspector configurations, probably because they -were in the RedHatPlugin class, at the bottom of the file and they were -probably missed when updating this plugin. Moving them at the top with -the other `add_copy_spec` will help tracking them in case something -change again. - -Revamping also the way we grab logs to check if we're in a container -first. - -Resolves: #2223 - -Signed-off-by: David Vallee Delisle -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/openstack_ironic.py | 121 +++++++++++++++++-------- - 1 file changed, 82 insertions(+), 39 deletions(-) - -diff --git a/sos/report/plugins/openstack_ironic.py b/sos/report/plugins/openstack_ironic.py -index 314d2a58..57060a27 100644 ---- a/sos/report/plugins/openstack_ironic.py -+++ b/sos/report/plugins/openstack_ironic.py -@@ -20,41 +20,95 @@ class OpenStackIronic(Plugin): - profiles = ('openstack', 'openstack_undercloud') - - var_puppet_gen = "/var/lib/config-data/puppet-generated/ironic" -+ ins_puppet_gen = var_puppet_gen + "_inspector" - - def setup(self): -- self.conf_list = [ -- "/etc/ironic/*", -- self.var_puppet_gen + "/etc/ironic/*", -- self.var_puppet_gen + "_api/etc/ironic/*" -- ] -- self.add_copy_spec([ -- "/etc/ironic/", -- self.var_puppet_gen + "/etc/xinetd.conf", -- self.var_puppet_gen + "/etc/xinetd.d/", -- self.var_puppet_gen + "/etc/ironic/", -- self.var_puppet_gen + "/etc/httpd/conf/", -- self.var_puppet_gen + "/etc/httpd/conf.d/", -- self.var_puppet_gen + "/etc/httpd/conf.modules.d/*.conf", -- self.var_puppet_gen + "/etc/my.cnf.d/tripleo.cnf", -- self.var_puppet_gen + "_api/etc/ironic/", -- self.var_puppet_gen + "_api/etc/httpd/conf/", -- self.var_puppet_gen + "_api/etc/httpd/conf.d/", -- self.var_puppet_gen + "_api/etc/httpd/conf.modules.d/*.conf", -- self.var_puppet_gen + "_api/etc/my.cnf.d/tripleo.cnf" -- ]) -- -- if self.get_option("all_logs"): -+ -+ in_container = self.container_exists('.*ironic_api') -+ -+ if in_container: -+ self.conf_list = [ -+ self.var_puppet_gen + "/etc/ironic/*", -+ self.var_puppet_gen + "/etc/ironic-inspector/*", -+ self.var_puppet_gen + "_api/etc/ironic/*", -+ self.ins_puppet_gen + "/etc/ironic-inspector/*", -+ self.ins_puppet_gen + "/var/lib/httpboot/inspector.ipxe" -+ ] - self.add_copy_spec([ -- "/var/log/ironic/", -+ "/var/lib/ironic-inspector/", -+ "/var/log/containers/ironic-inspector/ramdisk/", -+ self.var_puppet_gen + "/etc/xinetd.conf", -+ self.var_puppet_gen + "/etc/xinetd.d/", -+ self.var_puppet_gen + "/etc/ironic/", -+ self.var_puppet_gen + "/etc/ironic-inspector/", -+ self.var_puppet_gen + "/etc/httpd/conf/", -+ self.var_puppet_gen + "/etc/httpd/conf.d/", -+ self.var_puppet_gen + "/etc/httpd/conf.modules.d/*.conf", -+ self.var_puppet_gen + "/etc/my.cnf.d/tripleo.cnf", -+ self.var_puppet_gen + "_api/etc/ironic/", -+ self.var_puppet_gen + "_api/etc/httpd/conf/", -+ self.var_puppet_gen + "_api/etc/httpd/conf.d/", -+ self.var_puppet_gen + "_api/etc/httpd/conf.modules.d/*.conf", -+ self.var_puppet_gen + "_api/etc/my.cnf.d/tripleo.cnf", -+ self.ins_puppet_gen + "/etc/ironic-inspector/*", -+ self.ins_puppet_gen + "/var/lib/httpboot/inspector.ipxe" - ]) -+ -+ if self.get_option("all_logs"): -+ self.add_copy_spec([ -+ "/var/log/containers/ironic/", -+ "/var/log/containers/ironic-inspector/" -+ ]) -+ else: -+ self.add_copy_spec([ -+ "/var/log/containers/ironic/*.log", -+ "/var/log/containers/ironic-inspector/*.log", -+ ]) -+ -+ for path in ['/var/lib/ironic', '/httpboot', '/tftpboot', -+ self.ins_puppet_gen + '/var/lib/httpboot/', -+ self.ins_puppet_gen + '/var/lib/tftpboot/']: -+ self.add_cmd_output('ls -laRt %s' % path) -+ self.add_cmd_output('ls -laRt %s' % -+ (self.var_puppet_gen + path)) -+ -+ # Let's get the packages from the containers, always helpful when -+ # troubleshooting. -+ for container_name in ['ironic_inspector_dnsmasq', -+ 'ironic_inspector', 'ironic_pxe_http', -+ 'ironic_pxe_tftp', 'ironic_neutron_agent', -+ 'ironic_conductor', 'ironic_api']: -+ if self.container_exists('.*' + container_name): -+ self.add_cmd_output(self.fmt_container_cmd(container_name, -+ 'rpm -qa')) -+ - else: -+ self.conf_list = [ -+ "/etc/ironic/*", -+ "/etc/ironic-inspector/*", -+ ] - self.add_copy_spec([ -- "/var/log/ironic/*.log", -+ "/etc/ironic/", -+ "/etc/ironic-inspector/", -+ "/var/lib/ironic-inspector/", -+ "/var/log/ironic-inspector/ramdisk/", -+ "/etc/my.cnf.d/tripleo.cnf", -+ "/var/lib/httpboot/inspector.ipxe" - ]) - -- for path in ['/var/lib/ironic', '/httpboot', '/tftpboot']: -- self.add_cmd_output('ls -laRt %s' % path) -- self.add_cmd_output('ls -laRt %s' % (self.var_puppet_gen + path)) -+ if self.get_option("all_logs"): -+ self.add_copy_spec([ -+ "/var/log/ironic/", -+ "/var/log/ironic-inspector/", -+ ]) -+ else: -+ self.add_copy_spec([ -+ "/var/log/ironic/*.log", -+ "/var/log/ironic-inspector/*.log", -+ ]) -+ -+ for path in ['/var/lib/ironic', '/httpboot', '/tftpboot']: -+ self.add_cmd_output('ls -laRt %s' % path) - - vars_all = [p in os.environ for p in [ - 'OS_USERNAME', 'OS_PASSWORD']] -@@ -136,6 +190,7 @@ class RedHatIronic(OpenStackIronic, RedHatPlugin): - def setup(self): - super(RedHatIronic, self).setup() - -+ # ironic-discoverd was renamed to ironic-inspector in Liberty - # is the optional ironic-discoverd service installed? - if any([self.is_installed(p) for p in self.discoverd_packages]): - self.conf_list.append('/etc/ironic-discoverd/*') -@@ -146,18 +201,6 @@ class RedHatIronic(OpenStackIronic, RedHatPlugin): - self.add_journal(units="openstack-ironic-discoverd") - self.add_journal(units="openstack-ironic-discoverd-dnsmasq") - -- # ironic-discoverd was renamed to ironic-inspector in Liberty -- self.conf_list.append('/etc/ironic-inspector/*') -- self.conf_list.append(self.var_puppet_gen + '/etc/ironic-inspector/*') -- self.add_copy_spec('/etc/ironic-inspector/') -- self.add_copy_spec(self.var_puppet_gen + '/etc/ironic-inspector/') -- self.add_copy_spec('/var/lib/ironic-inspector/') -- if self.get_option("all_logs"): -- self.add_copy_spec('/var/log/ironic-inspector/') -- else: -- self.add_copy_spec('/var/log/ironic-inspector/*.log') -- self.add_copy_spec('/var/log/ironic-inspector/ramdisk/') -- - self.add_journal(units="openstack-ironic-inspector-dnsmasq") - - if self.osc_available: --- -2.26.2 - diff --git a/SOURCES/sos-bz1880372-power-logs.patch b/SOURCES/sos-bz1880372-power-logs.patch deleted file mode 100644 index 9c556ae..0000000 --- a/SOURCES/sos-bz1880372-power-logs.patch +++ /dev/null @@ -1,228 +0,0 @@ -From b8da3e3ed94075fa5ccf74a61ce64812b904d0c5 Mon Sep 17 00:00:00 2001 -From: Mamatha Inamdar -Date: Tue, 25 Aug 2020 14:16:52 +0530 -Subject: [PATCH] [powerpc]Add support to collect hardware component logs - -This patch updates powerpc plugin to collect Hardware and -firmware information. - -In this patch we are reading Hardware and firmware version -details through lsvpd, lscfg and lsmcode commands - -Related: #2213 - -Signed-off-by: Mamatha Inamdar -Reported-by: Borislav Stoymirski -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/powerpc.py | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/sos/report/plugins/powerpc.py b/sos/report/plugins/powerpc.py -index 6cd8dd14..9f38bd7d 100644 ---- a/sos/report/plugins/powerpc.py -+++ b/sos/report/plugins/powerpc.py -@@ -50,7 +50,10 @@ class PowerPC(Plugin, RedHatPlugin, UbuntuPlugin, DebianPlugin): - "ppc64_cpu --run-mode", - "ppc64_cpu --frequency", - "ppc64_cpu --dscr", -- "diag_encl -v" -+ "diag_encl -v", -+ "lsvpd -D", -+ "lsmcode -A", -+ "lscfg -v" - ]) - - if ispSeries: --- -2.26.2 - -From 3d25bbfdadf6c5f33dba7522536f744da1940794 Mon Sep 17 00:00:00 2001 -From: Mamatha Inamdar -Date: Tue, 25 Aug 2020 14:21:21 +0530 -Subject: [PATCH] [iprconfig]Add support to collect RAID adapter logs - -This patch updates iprconfig plugin to collect IBM Power -RAID adapter device driver information. - -Related: #2213 - -Signed-off-by: Mamatha Inamdar -Reported-by: Borislav Stoymirski -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/iprconfig.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/sos/report/plugins/iprconfig.py b/sos/report/plugins/iprconfig.py -index 08503a78..f7511a6c 100644 ---- a/sos/report/plugins/iprconfig.py -+++ b/sos/report/plugins/iprconfig.py -@@ -32,6 +32,7 @@ class IprConfig(Plugin, RedHatPlugin, UbuntuPlugin, DebianPlugin): - "iprconfig -c show-af-disks", - "iprconfig -c show-all-af-disks", - "iprconfig -c show-slots", -+ "iprconfig -c dump" - ]) - - show_ioas = self.collect_cmd_output("iprconfig -c show-ioas") --- -2.26.2 - -From effdb3b84ab80fa68d41af1438bfae465c571127 Mon Sep 17 00:00:00 2001 -From: Mamatha Inamdar -Date: Tue, 25 Aug 2020 14:30:23 +0530 -Subject: [PATCH] [kernel]Add support to collect network debugging logs - -This patch is to collect hybrid network debugging messages - -Related: #2213 - -Signed-off-by: Mamatha Inamdar -Reported-by: Luciano Chavez -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/kernel.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/sos/report/plugins/kernel.py b/sos/report/plugins/kernel.py -index febe2ad0..27e0e4d0 100644 ---- a/sos/report/plugins/kernel.py -+++ b/sos/report/plugins/kernel.py -@@ -106,6 +106,7 @@ class Kernel(Plugin, IndependentPlugin): - "/proc/misc", - "/var/log/dmesg", - "/sys/fs/pstore", -+ "/var/log/hcnmgr", - clocksource_path + "available_clocksource", - clocksource_path + "current_clocksource" - ]) --- -2.26.2 - -From b3fd83f0cc92b89e7adf8d66c446f3cf5ab1388b Mon Sep 17 00:00:00 2001 -From: Mamatha Inamdar -Date: Mon, 31 Aug 2020 10:56:15 +0530 -Subject: [PATCH] [mvcli]Add support to collect mvCLI PCI adapter infomation - -This patch is to add new plugin mvcli to collect -SATA drives connected to system backplane adapter information. - -infor -o vd ----> If the virtual disk was successfully - created, show a new RAID virtual disk -info -o pd -----> To show all physical disks and IDs: -info -o hba -----> To show all host bus adapters (HBAs): -smart -p 0 -----> To check for errors on a disk: - -Related: #2213 - -Signed-off-by: Mamatha Inamdar -Reported-by: Borislav Stoymirski -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/mvcli.py | 35 +++++++++++++++++++++++++++++++++++ - 1 file changed, 35 insertions(+) - create mode 100644 sos/report/plugins/mvcli.py - -diff --git a/sos/report/plugins/mvcli.py b/sos/report/plugins/mvcli.py -new file mode 100644 -index 00000000..ce7bf77b ---- /dev/null -+++ b/sos/report/plugins/mvcli.py -@@ -0,0 +1,35 @@ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+ -+# This sosreport plugin is meant for sas adapters. -+# This plugin logs inforamtion on each adapter it finds. -+ -+from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin -+ -+ -+class mvCLI(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin): -+ -+ short_desc = 'mvCLI Integrated RAID adapter information' -+ -+ plugin_name = "mvcli" -+ commands = ("/opt/marvell/bin/mvcli",) -+ -+ def setup(self): -+ -+ # get list of adapters -+ subcmds = [ -+ 'info -o vd', -+ 'info -o pd', -+ 'info -o hba', -+ 'smart -p 0', -+ ] -+ -+ self.add_cmd_output(["/opt/marvell/bin/mvcli %s" % s for s in subcmds]) -+ -+# vim: et ts=4 sw=4 --- -2.26.2 - -From 48ac730fbf4b168604079b18675867c5ed6dc1ae Mon Sep 17 00:00:00 2001 -From: Mamatha Inamdar -Date: Mon, 31 Aug 2020 11:54:52 +0530 -Subject: [PATCH] [arcconf]Add support to collect arcconf adapter infomation - -This patch is to add new arcconf plugin to collect -SATA drives connected to system backplane adapter information. - -arcconf getconfig 1 ----> To list the logical drives and - device configurations - -Closes: #2213 - -Signed-off-by: Mamatha Inamdar -Reported-by: Borislav Stoymirski -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/arcconf.py | 28 ++++++++++++++++++++++++++++ - 1 file changed, 28 insertions(+) - create mode 100644 sos/report/plugins/arcconf.py - -diff --git a/sos/report/plugins/arcconf.py b/sos/report/plugins/arcconf.py -new file mode 100644 -index 00000000..64d6bb1e ---- /dev/null -+++ b/sos/report/plugins/arcconf.py -@@ -0,0 +1,28 @@ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+ -+# This sosreport plugin is meant for sas adapters. -+# This plugin logs inforamtion on each adapter it finds. -+ -+from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin -+ -+ -+class arcconf(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin): -+ -+ short_desc = 'arcconf Integrated RAID adapter information' -+ -+ plugin_name = "arcconf" -+ commands = ("arcconf",) -+ -+ def setup(self): -+ -+ # get list of adapters -+ self.add_cmd_output("arcconf getconfig 1") -+ -+# vim: et ts=4 sw=4 --- -2.26.2 - diff --git a/SOURCES/sos-bz1881118-crio-conf-d.patch b/SOURCES/sos-bz1881118-crio-conf-d.patch deleted file mode 100644 index d39b353..0000000 --- a/SOURCES/sos-bz1881118-crio-conf-d.patch +++ /dev/null @@ -1,70 +0,0 @@ -From 019f7c49768f27ef15f39d80db8a03b2aaa453ee Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 21 Sep 2020 17:33:25 +0200 -Subject: [PATCH] [crio] collect /etc/crio/crio.conf.d/ - -Crio configs can be newly in the dir also. - -Resolves: #2240 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/crio.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/sos/report/plugins/crio.py b/sos/report/plugins/crio.py -index dacc0745..e8b566c3 100644 ---- a/sos/report/plugins/crio.py -+++ b/sos/report/plugins/crio.py -@@ -31,6 +31,7 @@ class CRIO(Plugin, RedHatPlugin, UbuntuPlugin): - "/etc/crictl.yaml", - "/etc/crio/crio.conf", - "/etc/crio/seccomp.json", -+ "/etc/crio/crio.conf.d/", - "/etc/systemd/system/cri-o.service", - "/etc/sysconfig/crio-*" - ]) --- -2.26.2 - -From 7f72a36144b3e235159556689b5129b7453294e3 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 15 Dec 2020 14:19:34 +0100 -Subject: [PATCH] [component] Use sysroot from Policy when opts doesn't specify - it - -Until --sysroot option is specified, Archive (sub)classes should -be called with sysroot determined from Policy. - -Resolves: #2346 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/component.py | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/sos/component.py b/sos/component.py -index 69d3b755..7774c05a 100644 ---- a/sos/component.py -+++ b/sos/component.py -@@ -246,13 +246,13 @@ class SoSComponent(): - auto_archive = self.policy.get_preferred_archive() - self.archive = auto_archive(archive_name, self.tmpdir, - self.policy, self.opts.threads, -- enc_opts, self.opts.sysroot, -+ enc_opts, self.sysroot, - self.manifest) - - else: - self.archive = TarFileArchive(archive_name, self.tmpdir, - self.policy, self.opts.threads, -- enc_opts, self.opts.sysroot, -+ enc_opts, self.sysroot, - self.manifest) - - self.archive.set_debug(True if self.opts.debug else False) --- -2.26.2 - diff --git a/SOURCES/sos-bz1882368-upload-functionality-issues.patch b/SOURCES/sos-bz1882368-upload-functionality-issues.patch deleted file mode 100644 index 470dce7..0000000 --- a/SOURCES/sos-bz1882368-upload-functionality-issues.patch +++ /dev/null @@ -1,211 +0,0 @@ -From a3b493a8accc338158faa53b9e221067323b75f5 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 24 Sep 2020 10:06:17 -0400 -Subject: [PATCH] [redhat] Ease upload url determination logic - -The logic for determining if an archive should be uploaded to the -Customer Portal was too strict, ease it to now properly only block on a -missing case number since username and passwords may now be provided via -env vars. - -Signed-off-by: Jake Hunsaker ---- - sos/policies/__init__.py | 6 ++++-- - sos/policies/redhat.py | 8 ++++++-- - 2 files changed, 10 insertions(+), 4 deletions(-) - -diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py -index 9a1aac07..215739bd 100644 ---- a/sos/policies/__init__.py -+++ b/sos/policies/__init__.py -@@ -1427,8 +1427,8 @@ class LinuxPolicy(Policy): - """Should be overridden by policies to determine if a password needs to - be provided for upload or not - """ -- if ((not self.upload_password and not self._upload_password) and -- self.upload_user): -+ if not self.get_upload_password() and (self.get_upload_user() != -+ self._upload_user): - msg = ( - "Please provide the upload password for %s: " - % self.upload_user -@@ -1472,7 +1473,8 @@ class LinuxPolicy(Policy): - Print a more human-friendly string than vendor URLs - """ - self.upload_archive = archive -- self.upload_url = self.get_upload_url() -+ if not self.upload_url: -+ self.upload_url = self.get_upload_url() - if not self.upload_url: - raise Exception("No upload destination provided by policy or by " - "--upload-url") -diff --git a/sos/policies/redhat.py b/sos/policies/redhat.py -index 34b421f3..f2f847a5 100644 ---- a/sos/policies/redhat.py -+++ b/sos/policies/redhat.py -@@ -320,12 +320,16 @@ support representative. - "Enter your Red Hat Customer Portal username (empty to use " - "public dropbox): ") - ) -+ if not self.upload_user: -+ self.upload_url = RH_FTP_HOST -+ self.upload_user = self._upload_user - - def get_upload_url(self): -+ if self.upload_url: -+ return self.upload_url - if self.commons['cmdlineopts'].upload_url: - return self.commons['cmdlineopts'].upload_url -- if (not self.case_id or not self.upload_user or not -- self.upload_password): -+ if not self.case_id: - # Cannot use the RHCP. Use anonymous dropbox - self.upload_user = self._upload_user - self.upload_directory = self._upload_directory --- -2.26.2 - -From 11cc6f478a9b41ce81b5b74faab5ca42930262ee Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 24 Sep 2020 10:17:25 -0400 -Subject: [PATCH] [policy] Use user-provided FTP directory if specified - -Fixes an issue whereby we ignore a user-provided FTP directory. - -Signed-off-by: Jake Hunsaker ---- - sos/policies/__init__.py | 2 +- - sos/policies/redhat.py | 3 ++- - 2 files changed, 3 insertions(+), 2 deletions(-) - -diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py -index 215739bd..32f271d9 100644 ---- a/sos/policies/__init__.py -+++ b/sos/policies/__init__.py -@@ -1677,7 +1677,7 @@ class LinuxPolicy(Policy): - password = self.get_upload_password() - - if not directory: -- directory = self._upload_directory -+ directory = self.upload_directory or self._upload_directory - - try: - session = ftplib.FTP(url, user, password) -diff --git a/sos/policies/redhat.py b/sos/policies/redhat.py -index f2f847a5..d079406f 100644 ---- a/sos/policies/redhat.py -+++ b/sos/policies/redhat.py -@@ -332,7 +332,8 @@ support representative. - if not self.case_id: - # Cannot use the RHCP. Use anonymous dropbox - self.upload_user = self._upload_user -- self.upload_directory = self._upload_directory -+ if self.upload_directory is None: -+ self.upload_directory = self._upload_directory - self.upload_password = None - return RH_FTP_HOST - else: --- -2.26.2 - -From caa9a2f2a511689080d019ffab61a4de5787d8be Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 24 Sep 2020 10:25:00 -0400 -Subject: [PATCH] [policy] Handle additional failure conditions for FTP uploads - -Adds a timeout and a timeout handler for FTP connections, rather than -letting the connection attempt continue indefinitely. - -Second, adds exception handling for an edge case where the connection to -the FTP server fails, but does not generate an exception from the ftplib -module. - -Additionally, correct the type-ing of the error numbers being checked so -that we actually match them. - -Resolves: #2245 - -Signed-off-by: Jake Hunsaker ---- - sos/policies/__init__.py | 11 ++++++++--- - 1 file changed, 8 insertions(+), 3 deletions(-) - -diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py -index 32f271d9..826d022e 100644 ---- a/sos/policies/__init__.py -+++ b/sos/policies/__init__.py -@@ -1680,15 +1680,20 @@ class LinuxPolicy(Policy): - directory = self.upload_directory or self._upload_directory - - try: -- session = ftplib.FTP(url, user, password) -+ session = ftplib.FTP(url, user, password, timeout=15) -+ if not session: -+ raise Exception("connection failed, did you set a user and " -+ "password?") - session.cwd(directory) -+ except socket.timeout: -+ raise Exception("timeout hit while connecting to %s" % url) - except socket.gaierror: - raise Exception("unable to connect to %s" % url) - except ftplib.error_perm as err: - errno = str(err).split()[0] -- if errno == 503: -+ if errno == '503': - raise Exception("could not login as '%s'" % user) -- if errno == 550: -+ if errno == '550': - raise Exception("could not set upload directory to %s" - % directory) - --- -2.26.2 - -From 21720a0f8c9cf6739e26470b2280e005f0f3e3f1 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Thu, 15 Oct 2020 13:45:37 +0200 -Subject: [PATCH] [policy] Use FTP server when user isnt set in batch mode - -Caling "sos report --upload --case-id=123 --batch" should fallback -to uploading to FTP server as the upload user is unknown and can't -be prompted in batch mode. - -Resolves: #2276 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/policies/redhat.py | 12 ++++++++++-- - 1 file changed, 10 insertions(+), 2 deletions(-) - -diff --git a/sos/policies/redhat.py b/sos/policies/redhat.py -index d079406f..3a65b9fa 100644 ---- a/sos/policies/redhat.py -+++ b/sos/policies/redhat.py -@@ -324,13 +324,21 @@ support representative. - self.upload_url = RH_FTP_HOST - self.upload_user = self._upload_user - -+ def _upload_user_set(self): -+ user = self.get_upload_user() -+ return user and (user != 'anonymous') -+ - def get_upload_url(self): - if self.upload_url: - return self.upload_url - if self.commons['cmdlineopts'].upload_url: - return self.commons['cmdlineopts'].upload_url -- if not self.case_id: -- # Cannot use the RHCP. Use anonymous dropbox -+ # anonymous FTP server should be used as fallback when either: -+ # - case id is not set, or -+ # - upload user isn't set AND batch mode prevents to prompt for it -+ if (not self.case_id) or \ -+ ((not self._upload_user_set()) and -+ self.commons['cmdlineopts'].batch): - self.upload_user = self._upload_user - if self.upload_directory is None: - self.upload_directory = self._upload_directory --- -2.26.2 - diff --git a/SOURCES/sos-bz1886711-enhance-tc-hw-offload.patch b/SOURCES/sos-bz1886711-enhance-tc-hw-offload.patch new file mode 100644 index 0000000..ed1088b --- /dev/null +++ b/SOURCES/sos-bz1886711-enhance-tc-hw-offload.patch @@ -0,0 +1,32 @@ +From bbb7f8bf522960a8ca7625f539e9e5d109abb704 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Wed, 19 May 2021 08:31:45 +0200 +Subject: [PATCH] [networking] collect also tc filter show ingress + +Both "tc -s filter show dev %eth [|ingress]" commands required as +they provide different output. + +Resolves: #2550 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/networking.py | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/sos/report/plugins/networking.py b/sos/report/plugins/networking.py +index acfa027f..35646268 100644 +--- a/sos/report/plugins/networking.py ++++ b/sos/report/plugins/networking.py +@@ -156,7 +156,8 @@ class Networking(Plugin): + "ethtool --phy-statistics " + eth, + "ethtool --show-priv-flags " + eth, + "ethtool --show-eee " + eth, +- "tc -s filter show dev " + eth ++ "tc -s filter show dev " + eth, ++ "tc -s filter show dev " + eth + " ingress", + ], tags=eth) + + # skip EEPROM collection by default, as it might hang or +-- +2.26.3 + diff --git a/SOURCES/sos-bz1886782-exclude-panfs.patch b/SOURCES/sos-bz1886782-exclude-panfs.patch deleted file mode 100644 index 357ca2c..0000000 --- a/SOURCES/sos-bz1886782-exclude-panfs.patch +++ /dev/null @@ -1,36 +0,0 @@ -From 6a4e3fb718a5c3249425dc4ae167b977abdb7f2e Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Renaud=20M=C3=A9trich?= -Date: Thu, 8 Oct 2020 11:51:13 +0200 -Subject: [PATCH] [filesys] never collect content of /proc/fs/panfs -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -panfs (from Panasas company) provides statistics under /proc/fs/panfs -which makes sosreports become several hundreds of GBs. This path must -hence be blacklisted. - -Resolves: #2262 - -Signed-off-by: Renaud Métrich -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/filesys.py | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/sos/report/plugins/filesys.py b/sos/report/plugins/filesys.py -index 3baec3ce..57f608d0 100644 ---- a/sos/report/plugins/filesys.py -+++ b/sos/report/plugins/filesys.py -@@ -43,6 +43,8 @@ class Filesys(Plugin, DebianPlugin, UbuntuPlugin, CosPlugin): - "lslocks" - ]) - -+ self.add_forbidden_path('/proc/fs/panfs') -+ - if self.get_option('lsof'): - self.add_cmd_output("lsof -b +M -n -l -P", root_symlink="lsof") - --- -2.26.2 - diff --git a/SOURCES/sos-bz1887390-kdump-logfiles.patch b/SOURCES/sos-bz1887390-kdump-logfiles.patch deleted file mode 100644 index ee95a56..0000000 --- a/SOURCES/sos-bz1887390-kdump-logfiles.patch +++ /dev/null @@ -1,49 +0,0 @@ -From be347440d5f8d650791ff044970c5e65ee8ec2a3 Mon Sep 17 00:00:00 2001 -From: Jose Castillo -Date: Mon, 12 Oct 2020 13:47:47 +0100 -Subject: [PATCH] [kdump] Collect new kdump logfiles - -Two new logfiles are available in kdump: - -/var/log/kdump.log -/var/crash/*/kexec-kdump.log - -The path for the second logfile mentioned above is the -default one, but this patch deals with a change in -default directory the same way that we do with the -file vmcore-dmesg.txt. - -Resolves: RHBZ#1817042 and RHBZ#1887390. -Resolves: #2270 - -Signed-off-by: Jose Castillo -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/kdump.py | 4 +++- - 1 file changed, 3 insertions(+), 1 deletion(-) - -diff --git a/sos/report/plugins/kdump.py b/sos/report/plugins/kdump.py -index 41d08b5b..4eccb3ff 100644 ---- a/sos/report/plugins/kdump.py -+++ b/sos/report/plugins/kdump.py -@@ -71,7 +71,8 @@ class RedHatKDump(KDump, RedHatPlugin): - self.add_copy_spec([ - "/etc/kdump.conf", - "/etc/udev/rules.d/*kexec.rules", -- "/var/crash/*/vmcore-dmesg.txt" -+ "/var/crash/*/vmcore-dmesg.txt", -+ "/var/log/kdump.log" - ]) - try: - path = self.read_kdump_conffile() -@@ -80,6 +81,7 @@ class RedHatKDump(KDump, RedHatPlugin): - path = "/var/crash" - - self.add_copy_spec("{}/*/vmcore-dmesg.txt".format(path)) -+ self.add_copy_spec("{}/*/kexec-kdump.log".format(path)) - - - class DebianKDump(KDump, DebianPlugin, UbuntuPlugin): --- -2.26.2 - diff --git a/SOURCES/sos-bz1887402-kexec-logs.patch b/SOURCES/sos-bz1887402-kexec-logs.patch deleted file mode 100644 index 670659f..0000000 --- a/SOURCES/sos-bz1887402-kexec-logs.patch +++ /dev/null @@ -1,40 +0,0 @@ -From a0cb4efb473a553fa034aaa8980635897adf1894 Mon Sep 17 00:00:00 2001 -From: Jose Castillo -Date: Tue, 26 Jan 2021 16:20:44 +0100 -Subject: [PATCH] [kdump] Gather the file kexec-dmesg.log - -Moved the file name from kexec-kdump.log to -the right one, kexec-dmesg.log and -added it to the list of files to gather via -add_copy_spec as per #1546. - -Resolves: RHBZ#1817042 -Resolves: #2386 - -Signed-off-by: Jose Castillo -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/kdump.py | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/sos/report/plugins/kdump.py b/sos/report/plugins/kdump.py -index 4eccb3ffe..6bcb7f74d 100644 ---- a/sos/report/plugins/kdump.py -+++ b/sos/report/plugins/kdump.py -@@ -72,6 +72,7 @@ def setup(self): - "/etc/kdump.conf", - "/etc/udev/rules.d/*kexec.rules", - "/var/crash/*/vmcore-dmesg.txt", -+ "/var/crash/*/kexec-dmesg.log", - "/var/log/kdump.log" - ]) - try: -@@ -81,7 +82,7 @@ def setup(self): - path = "/var/crash" - - self.add_copy_spec("{}/*/vmcore-dmesg.txt".format(path)) -- self.add_copy_spec("{}/*/kexec-kdump.log".format(path)) -+ self.add_copy_spec("{}/*/kexec-dmesg.log".format(path)) - - - class DebianKDump(KDump, DebianPlugin, UbuntuPlugin): diff --git a/SOURCES/sos-bz1888012-stratis-new-feature-output.patch b/SOURCES/sos-bz1888012-stratis-new-feature-output.patch deleted file mode 100644 index f41110c..0000000 --- a/SOURCES/sos-bz1888012-stratis-new-feature-output.patch +++ /dev/null @@ -1,36 +0,0 @@ -From 22b19739d94f0a40fb8dfd3236e63991a0c027b9 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Wed, 14 Oct 2020 08:33:28 +0200 -Subject: [PATCH] [stratis] Collect key list and report engine - -Required for troubleshooting Stratis-engine v. 2.1.0. - -Resolves: #2274 -Closes: #2273 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/stratis.py | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/sos/report/plugins/stratis.py b/sos/report/plugins/stratis.py -index b6071d32..a41c9476 100644 ---- a/sos/report/plugins/stratis.py -+++ b/sos/report/plugins/stratis.py -@@ -24,8 +24,11 @@ class Stratis(Plugin, RedHatPlugin): - 'pool list', - 'filesystem list', - 'blockdev list', -+ 'key list', - 'daemon redundancy', -- 'daemon version' -+ 'daemon version', -+ 'report engine_state_report', -+ '--version', - ] - - self.add_cmd_output(["stratis %s" % subcmd for subcmd in subcmds]) --- -2.26.2 - diff --git a/SOURCES/sos-bz1891562-tmp-dir-relative-path.patch b/SOURCES/sos-bz1891562-tmp-dir-relative-path.patch deleted file mode 100644 index d887f79..0000000 --- a/SOURCES/sos-bz1891562-tmp-dir-relative-path.patch +++ /dev/null @@ -1,34 +0,0 @@ -From c07bdbc94269603d2b910ccafa289512478160aa Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Mon, 26 Oct 2020 14:11:56 -0400 -Subject: [PATCH] [sos] Fix use of relative paths for --tmp-dir - -Fixes an issue where the use of relative paths for `--tmp-dir` causes a -failure in the building of the final archive. Previously, a relative -path would cause the tarball to be produced in a nested directory under -the temp directory of the same name, which would in turn cause -compression and all further operations for the archive to fail. - -Fix this by converting relative paths in the option to the absolute path -internally. - -Resolves: RHBZ#1891562 - -Signed-off-by: Jake Hunsaker ---- - sos/component.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/component.py b/sos/component.py -index 0aef770c6..b44fdf829 100644 ---- a/sos/component.py -+++ b/sos/component.py -@@ -138,7 +138,7 @@ def get_tmpdir_default(self): - use a standardized env var to redirect to the host's filesystem instead - """ - if self.opts.tmp_dir: -- return self.opts.tmp_dir -+ return os.path.abspath(self.opts.tmp_dir) - - tmpdir = '/var/tmp' - diff --git a/SOURCES/sos-bz1895316-collector--cluster-type.patch b/SOURCES/sos-bz1895316-collector--cluster-type.patch deleted file mode 100644 index 03d8646..0000000 --- a/SOURCES/sos-bz1895316-collector--cluster-type.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 1c6efee74557f433dfc5b67fb8ab76b0d9e6f988 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 8 Dec 2020 20:02:37 +0100 -Subject: [PATCH] [collector] allow overriding plain --cluster-type - -In few user scenarios, it is useful to force sos collect to override -cluster type, but let it generate list of nodes by itself. For that, -it is sufficient to set the self.cluster_type accordingly. - -Resolves: #2331 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/collector/__init__.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index 1abb08ae..bd84acaf 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -764,6 +764,7 @@ class SoSCollector(SoSComponent): - self.cluster = self.clusters['jbon'] - else: - self.cluster = self.clusters[self.opts.cluster_type] -+ self.cluster_type = self.opts.cluster_type - self.cluster.master = self.master - - else: --- -2.26.2 - diff --git a/SOURCES/sos-bz1904045-preset-ignores-verbosity.patch b/SOURCES/sos-bz1904045-preset-ignores-verbosity.patch deleted file mode 100644 index 608403d..0000000 --- a/SOURCES/sos-bz1904045-preset-ignores-verbosity.patch +++ /dev/null @@ -1,53 +0,0 @@ -From 169898b47b26734a8cdcb748157f7314f7f8821b Mon Sep 17 00:00:00 2001 -From: Erik Bernoth -Date: Tue, 10 Nov 2020 18:32:40 +0100 -Subject: [PATCH] [component] Add log verbosity from presets - -Closes: #2289 - -The main problem this tries to solve was that preset verbosity was -ignored in logging. - -With a simple test this could be reproduced: -sudo sh -c "source /path/to/repo/sosreport/venv/bin/activate; \ - cd /tmp/foo; sos report --preset sostestpreset; cd -" - -The bug is that without a change of code there are no messages from the -plugin `host` (no lines of output start wiht "[plugin:host]"). - -The problem is that the logging is set in the inherited __init__() method -from Component, but the presets are only handled afterwards in the -Report's __init__(). - -Since it is good to have logging configured from the beginning, the -only option is to reconfigure it after the preset config is known. - -The simplest method is to reinitialize the logging, although maybe not -the most efficient. - -Signed-off-by: Erik Bernoth -Signed-off-by: Jake Hunsaker ---- - sos/report/__init__.py | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index 2e97010b..c6b0c21c 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -153,6 +153,12 @@ class SoSReport(SoSComponent): - self.opts.merge(self.preset.opts) - # re-apply any cmdline overrides to the preset - self.opts = self.apply_options_from_cmdline(self.opts) -+ if hasattr(self.preset.opts, 'verbosity') and \ -+ self.preset.opts.verbosity > 0: -+ print('\nWARNING: It is not recommended to set verbosity via the ' -+ 'preset as it might have\nunforseen consequences for your ' -+ 'report logs.\n') -+ self._setup_logging() - - self._set_directories() - --- -2.26.2 - diff --git a/SOURCES/sos-bz1905657-empty-file-stops-zero-sizelimit.patch b/SOURCES/sos-bz1905657-empty-file-stops-zero-sizelimit.patch deleted file mode 100644 index 13c05ed..0000000 --- a/SOURCES/sos-bz1905657-empty-file-stops-zero-sizelimit.patch +++ /dev/null @@ -1,56 +0,0 @@ -From c2ddb50fbbb045daffa6fe5cf489fe47aeef4590 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 9 Dec 2020 10:21:32 -0500 -Subject: [PATCH] [options] Fix --log-size=0 being ignored and unreported - otherwise - -The `--log-size` option was being silently ignored, due to a too-loose -conditional in `Component.apply_options_from_cmdline()` which was -inadvertently filtering out the option when a user set it to 0. Note -that this did not affect `sos.conf` settings for this same value. - -Similarly, reporting the effective options after preset and cmdline -merging was skipping log-size when it was set to 0, since we normally -want to filter out null-value options (which imply they were not -invoked). Adding an explicit check for `log-size` here is the easiest -route forward to allow the reporting we expect. - -Closes: #2334 -Resolves: #2335 - -Signed-off-by: Jake Hunsaker ---- - sos/component.py | 2 +- - sos/options.py | 3 +++ - 2 files changed, 4 insertions(+), 1 deletion(-) - -diff --git a/sos/component.py b/sos/component.py -index 00f27f5e..69d3b755 100644 ---- a/sos/component.py -+++ b/sos/component.py -@@ -192,7 +192,7 @@ class SoSComponent(): - for opt, val in codict.items(): - if opt not in cmdopts.arg_defaults.keys(): - continue -- if val and val != opts.arg_defaults[opt]: -+ if val is not None and val != opts.arg_defaults[opt]: - setattr(opts, opt, val) - - return opts -diff --git a/sos/options.py b/sos/options.py -index ba3db130..b82a7d36 100644 ---- a/sos/options.py -+++ b/sos/options.py -@@ -282,6 +282,9 @@ class SoSOptions(): - """ - if name in ("add_preset", "del_preset", "desc", "note"): - return False -+ # Exception list for options that still need to be reported when 0 -+ if name in ['log_size', 'plugin_timeout'] and value == 0: -+ return True - return has_value(name, value) - - def argify(name, value): --- -2.26.2 - diff --git a/SOURCES/sos-bz1906598-collect-broken-symlinks.patch b/SOURCES/sos-bz1906598-collect-broken-symlinks.patch deleted file mode 100644 index 2b324b7..0000000 --- a/SOURCES/sos-bz1906598-collect-broken-symlinks.patch +++ /dev/null @@ -1,56 +0,0 @@ -From 15e54577289a29e72c636f8987859e91c3a55a7c Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Thu, 10 Dec 2020 20:23:03 +0100 -Subject: [PATCH] [report] collect broken symlinks - -Information about broken symlink destination is useful information -that sos report should collect. Currently it stops doing so as -stat-ing the symlink to determine filesize fails. - -Closes: #2333 -Resolves: #2338 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/__init__.py | 16 ++++++++++------ - 1 file changed, 10 insertions(+), 6 deletions(-) - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index 510e116e..1527caea 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -1449,11 +1449,16 @@ class Plugin(object): - continue - - try: -- filestat = os.stat(_file) -+ file_size = os.stat(_file)[stat.ST_SIZE] - except OSError: -- self._log_info("failed to stat '%s'" % _file) -- continue -- current_size += filestat[stat.ST_SIZE] -+ # if _file is a broken symlink, we should collect it, -+ # otherwise skip it -+ if os.path.islink(_file): -+ file_size = 0 -+ else: -+ self._log_info("failed to stat '%s', skipping" % _file) -+ continue -+ current_size += file_size - - if sizelimit and current_size > sizelimit: - limit_reached = True -@@ -1467,8 +1472,7 @@ class Plugin(object): - strfile = ( - file_name.replace(os.path.sep, ".") + ".tailed" - ) -- add_size = (sizelimit + filestat[stat.ST_SIZE] -- - current_size) -+ add_size = sizelimit + file_size - current_size - self.add_string_as_file(tail(_file, add_size), strfile) - rel_path = os.path.relpath('/', os.path.dirname(_file)) - link_path = os.path.join(rel_path, 'sos_strings', --- -2.26.2 - diff --git a/SOURCES/sos-bz1912821-sos-collector-declare-sysroot.patch b/SOURCES/sos-bz1912821-sos-collector-declare-sysroot.patch deleted file mode 100644 index 569d680..0000000 --- a/SOURCES/sos-bz1912821-sos-collector-declare-sysroot.patch +++ /dev/null @@ -1,32 +0,0 @@ -From debb61f8137c53bdaf8d4473756c68c5e4d5cca2 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 5 Jan 2021 13:35:34 +0100 -Subject: [PATCH] [collector] declare sysroot for each component - -Commit 7f72a36 requires self.sysroot to exist for each component, -but it is not set for sos-collector. Let pre-fill self.sysroot -every time. - -Resolves: #2358 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/component.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/sos/component.py b/sos/component.py -index bd008761..223c3812 100644 ---- a/sos/component.py -+++ b/sos/component.py -@@ -108,6 +108,7 @@ class SoSComponent(): - try: - import sos.policies - self.policy = sos.policies.load(sysroot=self.opts.sysroot) -+ self.sysroot = self.policy.host_sysroot() - except KeyboardInterrupt: - self._exit(0) - self._is_root = self.policy.is_root() --- -2.26.2 - diff --git a/SOURCES/sos-bz1912889-plugopts-ignored-in-configfile.patch b/SOURCES/sos-bz1912889-plugopts-ignored-in-configfile.patch deleted file mode 100644 index 7a26fac..0000000 --- a/SOURCES/sos-bz1912889-plugopts-ignored-in-configfile.patch +++ /dev/null @@ -1,33 +0,0 @@ -From cd56e096afc8ef06c215c45cbf025bda60f0169c Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 5 Jan 2021 15:06:24 +0100 -Subject: [PATCH] [component] honour plugopts from config file - -Currently, config file plugopts are ignored as we overwrite it -in apply_options_from_cmdline by empty list default value from -cmdline. - -Resolves: #2359 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/component.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/component.py b/sos/component.py -index 7774c05a..bd008761 100644 ---- a/sos/component.py -+++ b/sos/component.py -@@ -192,7 +192,7 @@ class SoSComponent(): - for opt, val in codict.items(): - if opt not in cmdopts.arg_defaults.keys(): - continue -- if val is not None and val != opts.arg_defaults[opt]: -+ if val not in [None, [], ''] and val != opts.arg_defaults[opt]: - setattr(opts, opt, val) - - return opts --- -2.26.2 - diff --git a/SOURCES/sos-bz1912910-empty-file-stops-collecting.patch b/SOURCES/sos-bz1912910-empty-file-stops-collecting.patch deleted file mode 100644 index 82ba360..0000000 --- a/SOURCES/sos-bz1912910-empty-file-stops-collecting.patch +++ /dev/null @@ -1,33 +0,0 @@ -From 03642cf2e5619f11c762b63c61c9c69fb2b00cdf Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 8 Dec 2020 19:33:07 +0100 -Subject: [PATCH] [plugins] Dont stop collecting by empty specfile when - sizelimit=0 - -When sizelimit=0, collecting an empty file would set limit_reached -wrongly and stop collecting further files. Let fix this corner case. - -Resolves: #2330 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/__init__.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index deb46c93..510e116e 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -1483,7 +1483,7 @@ class Plugin(object): - self._add_copy_paths([_file]) - # in the corner case we just reached the sizelimit, we - # should collect the whole file and stop -- limit_reached = (current_size == sizelimit) -+ limit_reached = (sizelimit and current_size == sizelimit) - if self.manifest: - self.manifest.files.append({ - 'specification': copyspec, --- -2.26.2 - diff --git a/SOURCES/sos-bz1916729-ftp-upload-no-passwd.patch b/SOURCES/sos-bz1916729-ftp-upload-no-passwd.patch deleted file mode 100644 index 6b2b2fc..0000000 --- a/SOURCES/sos-bz1916729-ftp-upload-no-passwd.patch +++ /dev/null @@ -1,35 +0,0 @@ -From 486a7918934041306bae8ccc11da2196e8f4c9bb Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 13 Jan 2021 10:57:58 -0500 -Subject: [PATCH] [Policy] Handle additional FTP authentication issues - -It was found that some implementations will return a 530 rather than a -503 as the more specific error for incorrect passwords. Handle this -error code explicitly, and then also add a catch-all for any other -ftplib errors that may get raised. - -Resolves: #2368 - -Signed-off-by: Jake Hunsaker ---- - sos/policies/__init__.py | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py -index c5fb4801e..a4f550c96 100644 ---- a/sos/policies/__init__.py -+++ b/sos/policies/__init__.py -@@ -477,9 +477,13 @@ def upload_ftp(self, url=None, directory=None, user=None, password=None): - errno = str(err).split()[0] - if errno == '503': - raise Exception("could not login as '%s'" % user) -+ if errno == '530': -+ raise Exception("invalid password for user '%s'" % user) - if errno == '550': - raise Exception("could not set upload directory to %s" - % directory) -+ raise Exception("error trying to establish session: %s" -+ % str(err)) - - try: - with open(self.upload_archive, 'rb') as _arcfile: diff --git a/SOURCES/sos-bz1917196-networking-ethtool-e-conditionally.patch b/SOURCES/sos-bz1917196-networking-ethtool-e-conditionally.patch deleted file mode 100644 index c64ffa7..0000000 --- a/SOURCES/sos-bz1917196-networking-ethtool-e-conditionally.patch +++ /dev/null @@ -1,60 +0,0 @@ -From aca8bd83117e177f2beac6b9434d36d446a7de64 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 18 Jan 2021 22:45:43 +0100 -Subject: [PATCH] [networking] Collect 'ethtool -e ' conditionally only - -EEPROM dump collection might hang on specific types of devices, or -negatively impact the system otherwise. As a safe option, sos report -should collect the command when explicitly asked via a plugopt only. - -Resolves: #2376 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/networking.py | 22 +++++++++++----------- - 1 file changed, 11 insertions(+), 11 deletions(-) - -diff --git a/sos/report/plugins/networking.py b/sos/report/plugins/networking.py -index e4236ed9..5bdb697e 100644 ---- a/sos/report/plugins/networking.py -+++ b/sos/report/plugins/networking.py -@@ -27,7 +27,8 @@ class Networking(Plugin): - ("namespaces", "Number of namespaces to collect, 0 for unlimited. " + - "Incompatible with the namespace_pattern plugin option", "slow", 0), - ("ethtool_namespaces", "Define if ethtool commands should be " + -- "collected for namespaces", "slow", True) -+ "collected for namespaces", "slow", True), -+ ("eepromdump", "collect 'ethtool -e' for all devices", "slow", False) - ] - - # switch to enable netstat "wide" (non-truncated) output mode -@@ -141,16 +142,15 @@ class Networking(Plugin): - "ethtool --show-eee " + eth - ], tags=eth) - -- # skip EEPROM collection for 'bnx2x' NICs as this command -- # can pause the NIC and is not production safe. -- bnx_output = { -- "cmd": "ethtool -i %s" % eth, -- "output": "bnx2x" -- } -- bnx_pred = SoSPredicate(self, -- cmd_outputs=bnx_output, -- required={'cmd_outputs': 'none'}) -- self.add_cmd_output("ethtool -e %s" % eth, pred=bnx_pred) -+ # skip EEPROM collection by default, as it might hang or -+ # negatively impact the system on some device types -+ if self.get_option("eepromdump"): -+ cmd = "ethtool -e %s" % eth -+ self._log_warn("WARNING (about to collect '%s'): collecting " -+ "an eeprom dump is known to cause certain NIC " -+ "drivers (e.g. bnx2x/tg3) to interrupt device " -+ "operation" % cmd) -+ self.add_cmd_output(cmd) - - # Collect information about bridges (some data already collected via - # "ip .." commands) --- -2.26.2 - diff --git a/SOURCES/sos-bz1923938-sos-log-effective-options.patch b/SOURCES/sos-bz1923938-sos-log-effective-options.patch new file mode 100644 index 0000000..120df02 --- /dev/null +++ b/SOURCES/sos-bz1923938-sos-log-effective-options.patch @@ -0,0 +1,284 @@ +From 00d12ad3cf24dcc6c73e9bcf63db1d3f17e58bb1 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Thu, 1 Jul 2021 10:50:54 -0400 +Subject: [PATCH] [sosnode] Properly format skip-commands and skip-files on + nodes + +Fixes an issue where options provided for `skip-commands` and +`skip-files` were not properly formatted, thus causing an exception +during the finalization of the node's sos command. + +Signed-off-by: Jake Hunsaker +--- + sos/collector/sosnode.py | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py +index 6597d236..426edcba 100644 +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -734,11 +734,12 @@ class SosNode(): + if self.check_sos_version('4.1'): + if self.opts.skip_commands: + sos_opts.append( +- '--skip-commands=%s' % (quote(self.opts.skip_commands)) ++ '--skip-commands=%s' % ( ++ quote(','.join(self.opts.skip_commands))) + ) + if self.opts.skip_files: + sos_opts.append( +- '--skip-files=%s' % (quote(self.opts.skip_files)) ++ '--skip-files=%s' % (quote(','.join(self.opts.skip_files))) + ) + + if self.check_sos_version('4.2'): +-- +2.31.1 + +From de7edce3f92ed50abcb28dd0dbcbeb104dc7c679 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Fri, 2 Jul 2021 09:52:11 +0200 +Subject: [PATCH] [collector] fix a typo in --plugin-option + +Sos report uses --plugin-option or --plugopts. + +Relevant: #2606 + +Signed-off-by: Pavel Moravec +--- + sos/collector/__init__.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py +index 6d96d692..f072287e 100644 +--- a/sos/collector/__init__.py ++++ b/sos/collector/__init__.py +@@ -272,7 +272,7 @@ class SoSCollector(SoSComponent): + help="chroot executed commands to SYSROOT") + sos_grp.add_argument('-e', '--enable-plugins', action="extend", + help='Enable specific plugins for sosreport') +- sos_grp.add_argument('-k', '--plugin-options', action="extend", ++ sos_grp.add_argument('-k', '--plugin-option', action="extend", + help='Plugin option as plugname.option=value') + sos_grp.add_argument('--log-size', default=0, type=int, + help='Limit the size of individual logs (in MiB)') +-- +2.31.1 + +From 24a79ae8df8f29276f6139c68d4ba9b05114f951 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Fri, 2 Jul 2021 09:53:47 +0200 +Subject: [PATCH] [options] allow variant option names in config file + +While cmdline allows --plugin-option as well as --plugopts, +it stores the value under `plugopts` key. Therefore parsing +config file ignores --plugin-option. + +Similarly for --name/--label and --profile/--profiles. + +When processing config file, we must unify those potentially duplicit +keys. + +Resolves: #2606 + +Signed-off-by: Pavel Moravec +--- + sos/options.py | 9 +++++++++ + 1 file changed, 9 insertions(+) + +diff --git a/sos/options.py b/sos/options.py +index 1eda55d6..a014a022 100644 +--- a/sos/options.py ++++ b/sos/options.py +@@ -186,9 +186,18 @@ class SoSOptions(): + if 'verbose' in odict.keys(): + odict['verbosity'] = int(odict.pop('verbose')) + # convert options names ++ # unify some of them if multiple variants of the ++ # cmdoption exist ++ rename_opts = { ++ 'name': 'label', ++ 'plugin_option': 'plugopts', ++ 'profile': 'profiles' ++ } + for key in list(odict): + if '-' in key: + odict[key.replace('-', '_')] = odict.pop(key) ++ if key in rename_opts: ++ odict[rename_opts[key]] = odict.pop(key) + # set the values according to the config file + for key, val in odict.items(): + if isinstance(val, str): +-- +2.31.1 + +From c7d3644c0c64e9e5439806250592a55c8e2de26f Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Thu, 1 Jul 2021 08:11:15 +0200 +Subject: [PATCH] [report,collect] unify --map-file arguments + +Unify --map[-file] argument among report/collect/clean. + +Resolves: #2602 + +Signed-off-by: Pavel Moravec +--- + sos/cleaner/__init__.py | 2 +- + sos/collector/__init__.py | 2 +- + sos/report/__init__.py | 2 +- + 3 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py +index 7414b55e0..4c9837826 100644 +--- a/sos/cleaner/__init__.py ++++ b/sos/cleaner/__init__.py +@@ -192,7 +192,7 @@ def add_parser_options(cls, parser): + 'file for obfuscation')) + clean_grp.add_argument('--no-update', dest='no_update', default=False, + action='store_true', +- help='Do not update the --map file with new ' ++ help='Do not update the --map-file with new ' + 'mappings from this run') + clean_grp.add_argument('--keep-binary-files', default=False, + action='store_true', +diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py +index 7b8cfcf72..6d96d6923 100644 +--- a/sos/collector/__init__.py ++++ b/sos/collector/__init__.py +@@ -427,7 +427,7 @@ def add_parser_options(cls, parser): + cleaner_grp.add_argument('--no-update', action='store_true', + default=False, dest='no_update', + help='Do not update the default cleaner map') +- cleaner_grp.add_argument('--map', dest='map_file', ++ cleaner_grp.add_argument('--map-file', dest='map_file', + default='/etc/sos/cleaner/default_mapping', + help=('Provide a previously generated mapping' + ' file for obfuscation')) +diff --git a/sos/report/__init__.py b/sos/report/__init__.py +index 7ad2d24a4..411c4eb03 100644 +--- a/sos/report/__init__.py ++++ b/sos/report/__init__.py +@@ -341,7 +341,7 @@ def add_parser_options(cls, parser): + cleaner_grp.add_argument('--no-update', action='store_true', + default=False, dest='no_update', + help='Do not update the default cleaner map') +- cleaner_grp.add_argument('--map', dest='map_file', ++ cleaner_grp.add_argument('--map-file', dest='map_file', + default='/etc/sos/cleaner/default_mapping', + help=('Provide a previously generated mapping' + ' file for obfuscation')) +From fd75745e7a5a6c5def8e6d23190227872b9912c3 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 11 Aug 2021 10:48:41 -0400 +Subject: [PATCH] [sosnode] Fix passing of plugin options when using + `--only-plugins` + +Fixes the handling of plugin options passed by `sos collect` to each +node by first aligning the SoSOption name to those of `report` +(`plugopts`), and second re-arranges the handling of plugin options and +preset options passed by the user when also using `--only-plugins` so +that the former are preserved and passed only with the `--only-plugins` +option value. + +Resolves: #2641 + +Signed-off-by: Jake Hunsaker +--- + sos/collector/__init__.py | 5 +++-- + sos/collector/sosnode.py | 34 +++++++++++++++++----------------- + 2 files changed, 20 insertions(+), 19 deletions(-) + +diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py +index 57ef074e..70b7a69e 100644 +--- a/sos/collector/__init__.py ++++ b/sos/collector/__init__.py +@@ -84,7 +84,7 @@ class SoSCollector(SoSComponent): + 'only_plugins': [], + 'password': False, + 'password_per_node': False, +- 'plugin_options': [], ++ 'plugopts': [], + 'plugin_timeout': None, + 'cmd_timeout': None, + 'preset': '', +@@ -273,7 +273,8 @@ class SoSCollector(SoSComponent): + help="chroot executed commands to SYSROOT") + sos_grp.add_argument('-e', '--enable-plugins', action="extend", + help='Enable specific plugins for sosreport') +- sos_grp.add_argument('-k', '--plugin-option', action="extend", ++ sos_grp.add_argument('-k', '--plugin-option', '--plugopts', ++ action="extend", dest='plugopts', + help='Plugin option as plugname.option=value') + sos_grp.add_argument('--log-size', default=0, type=int, + help='Limit the size of individual logs (in MiB)') +diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py +index 426edcba..5d05c297 100644 +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -667,10 +667,10 @@ class SosNode(): + + if self.cluster.sos_plugin_options: + for opt in self.cluster.sos_plugin_options: +- if not any(opt in o for o in self.plugin_options): ++ if not any(opt in o for o in self.plugopts): + option = '%s=%s' % (opt, + self.cluster.sos_plugin_options[opt]) +- self.plugin_options.append(option) ++ self.plugopts.append(option) + + # set master-only options + if self.cluster.check_node_is_master(self): +@@ -688,7 +688,7 @@ class SosNode(): + self.only_plugins = list(self.opts.only_plugins) + self.skip_plugins = list(self.opts.skip_plugins) + self.enable_plugins = list(self.opts.enable_plugins) +- self.plugin_options = list(self.opts.plugin_options) ++ self.plugopts = list(self.opts.plugopts) + self.preset = list(self.opts.preset) + + def finalize_sos_cmd(self): +@@ -754,6 +754,20 @@ class SosNode(): + os.path.join(self.host.sos_bin_path, self.sos_bin) + ) + ++ if self.plugopts: ++ opts = [o for o in self.plugopts ++ if self._plugin_exists(o.split('.')[0]) ++ and self._plugin_option_exists(o.split('=')[0])] ++ if opts: ++ sos_opts.append('-k %s' % quote(','.join(o for o in opts))) ++ ++ if self.preset: ++ if self._preset_exists(self.preset): ++ sos_opts.append('--preset=%s' % quote(self.preset)) ++ else: ++ self.log_debug('Requested to enable preset %s but preset does ' ++ 'not exist on node' % self.preset) ++ + if self.only_plugins: + plugs = [o for o in self.only_plugins if self._plugin_exists(o)] + if len(plugs) != len(self.only_plugins): +@@ -792,20 +806,6 @@ class SosNode(): + if enable: + sos_opts.append('--enable-plugins=%s' % quote(enable)) + +- if self.plugin_options: +- opts = [o for o in self.plugin_options +- if self._plugin_exists(o.split('.')[0]) +- and self._plugin_option_exists(o.split('=')[0])] +- if opts: +- sos_opts.append('-k %s' % quote(','.join(o for o in opts))) +- +- if self.preset: +- if self._preset_exists(self.preset): +- sos_opts.append('--preset=%s' % quote(self.preset)) +- else: +- self.log_debug('Requested to enable preset %s but preset does ' +- 'not exist on node' % self.preset) +- + self.sos_cmd = "%s %s" % (sos_cmd, ' '.join(sos_opts)) + self.log_info('Final sos command set to %s' % self.sos_cmd) + self.manifest.add_field('final_sos_command', self.sos_cmd) +-- +2.31.1 + diff --git a/SOURCES/sos-bz1925419-all-gluster-files.patch b/SOURCES/sos-bz1925419-all-gluster-files.patch new file mode 100644 index 0000000..ab24429 --- /dev/null +++ b/SOURCES/sos-bz1925419-all-gluster-files.patch @@ -0,0 +1,39 @@ +From 4fb834ec862228afb276ccbd45aa86c66044ea66 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Mon, 15 Mar 2021 09:09:51 +0100 +Subject: [PATCH] [gluster] collect public keys from the right dir + +Collection of glusterfind dir is achieved by /var/lib/gluster +so it doesn't be collected explicitly. + +/var/lib/glusterd/glusterfind/.keys/ subdir is required to be +explicitly collected, as add_copy_spec uses glob.glob() that skips +hidden files. + +Resolves: #2451 + +Signed-off-by: Pavel Moravec +Signed-off-by: Jake Hunsaker +--- + sos/report/plugins/gluster.py | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/sos/report/plugins/gluster.py b/sos/report/plugins/gluster.py +index e1a89df2..952cab63 100644 +--- a/sos/report/plugins/gluster.py ++++ b/sos/report/plugins/gluster.py +@@ -76,9 +76,8 @@ class Gluster(Plugin, RedHatPlugin): + "/var/lib/glusterd/", + # collect nfs-ganesha related configuration + "/run/gluster/shared_storage/nfs-ganesha/", +- # collect status files and public ssh keys +- "/var/lib/glusterd/.keys/", +- "/var/lib/glusterd/glusterfind/" ++ # collect public ssh keys (a_s_c skips implicit hidden files) ++ "/var/lib/glusterd/glusterfind/.keys/", + ] + glob.glob('/run/gluster/*tier-dht/*')) + + if not self.get_option("all_logs"): +-- +2.26.3 + diff --git a/SOURCES/sos-bz1925419-gluster-pubkeys-statusfile.patch b/SOURCES/sos-bz1925419-gluster-pubkeys-statusfile.patch deleted file mode 100644 index 4a5a448..0000000 --- a/SOURCES/sos-bz1925419-gluster-pubkeys-statusfile.patch +++ /dev/null @@ -1,95 +0,0 @@ -From 51e8213fd3a83e717fe7ef35d48d5c541b077c5f Mon Sep 17 00:00:00 2001 -From: Jose Castillo -Date: Mon, 8 Feb 2021 16:25:34 +0100 -Subject: [PATCH] [gluster] Add glusterd public keys and status files - -This patch helps capture some missing files in the -gluster plugin, i.e.: - -Files inside /var/lib/glusterd/glusterfind, like -*.status files, that store the required timestamps, -*.pem.pub files, that store ssh public keys. -We also need to omit the glusterfind_*_secret.pem, -which contains the openssh private key. - -Files inside /var/lib/glusterd/.keys, that contains -*.pem.pub, the ssh public key. - -Closes: RHBZ#1925035, RHBZ#1925419 - -Resolves: #2411 - -Signed-off-by: Jose Castillo -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/gluster.py | 8 +++++++- - 1 file changed, 7 insertions(+), 1 deletion(-) - -diff --git a/sos/report/plugins/gluster.py b/sos/report/plugins/gluster.py -index 7b6a9298..e1a89df2 100644 ---- a/sos/report/plugins/gluster.py -+++ b/sos/report/plugins/gluster.py -@@ -55,6 +55,9 @@ class Gluster(Plugin, RedHatPlugin): - - def setup(self): - self.add_forbidden_path("/var/lib/glusterd/geo-replication/secret.pem") -+ self.add_forbidden_path( -+ "/var/lib/glusterd/glusterfind/glusterfind_*_secret.pem" -+ ) - - self.add_cmd_output([ - "gluster peer status", -@@ -72,7 +75,10 @@ class Gluster(Plugin, RedHatPlugin): - "/etc/glusterfs", - "/var/lib/glusterd/", - # collect nfs-ganesha related configuration -- "/run/gluster/shared_storage/nfs-ganesha/" -+ "/run/gluster/shared_storage/nfs-ganesha/", -+ # collect status files and public ssh keys -+ "/var/lib/glusterd/.keys/", -+ "/var/lib/glusterd/glusterfind/" - ] + glob.glob('/run/gluster/*tier-dht/*')) - - if not self.get_option("all_logs"): --- -2.26.2 - -From 4fb834ec862228afb276ccbd45aa86c66044ea66 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 15 Mar 2021 09:09:51 +0100 -Subject: [PATCH] [gluster] collect public keys from the right dir - -Collection of glusterfind dir is achieved by /var/lib/gluster -so it doesn't be collected explicitly. - -/var/lib/glusterd/glusterfind/.keys/ subdir is required to be -explicitly collected, as add_copy_spec uses glob.glob() that skips -hidden files. - -Resolves: #2451 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/gluster.py | 5 ++--- - 1 file changed, 2 insertions(+), 3 deletions(-) - -diff --git a/sos/report/plugins/gluster.py b/sos/report/plugins/gluster.py -index e1a89df2..952cab63 100644 ---- a/sos/report/plugins/gluster.py -+++ b/sos/report/plugins/gluster.py -@@ -76,9 +76,8 @@ class Gluster(Plugin, RedHatPlugin): - "/var/lib/glusterd/", - # collect nfs-ganesha related configuration - "/run/gluster/shared_storage/nfs-ganesha/", -- # collect status files and public ssh keys -- "/var/lib/glusterd/.keys/", -- "/var/lib/glusterd/glusterfind/" -+ # collect public ssh keys (a_s_c skips implicit hidden files) -+ "/var/lib/glusterd/glusterfind/.keys/", - ] + glob.glob('/run/gluster/*tier-dht/*')) - - if not self.get_option("all_logs"): --- -2.26.2 - diff --git a/SOURCES/sos-bz1928650-powerpc-nhv-scsi-logs.patch b/SOURCES/sos-bz1928650-powerpc-nhv-scsi-logs.patch deleted file mode 100644 index 46bc02c..0000000 --- a/SOURCES/sos-bz1928650-powerpc-nhv-scsi-logs.patch +++ /dev/null @@ -1,101 +0,0 @@ -From 271c35b9be95cf4957150fd702823fbb46ddaa6b Mon Sep 17 00:00:00 2001 -From: Mamatha Inamdar -Date: Tue, 19 Jan 2021 19:54:26 +0530 -Subject: [PATCH 1/2] [powerpc]:Add support to collect HNV infomation - -This patch is to update powerpc plugin to collect -Hyper-V Network Virtualization information. - -/var/log/hcnmgr -- hybridnetwork debugging messages. Must collect -/var/ct/IBM.DRM.stderr -- DynamicRM log messages -/var/ct/IW/log/mc/IBM.DRM/trace* -- IBM DRM traces -lsdevinfo -ournalctl - -Signed-off-by: Mamatha Inamdar ---- - sos/report/plugins/kernel.py | 1 - - sos/report/plugins/powerpc.py | 9 +++++++-- - 2 files changed, 7 insertions(+), 3 deletions(-) - -diff --git a/sos/report/plugins/kernel.py b/sos/report/plugins/kernel.py -index 27e0e4d00..febe2ad0a 100644 ---- a/sos/report/plugins/kernel.py -+++ b/sos/report/plugins/kernel.py -@@ -106,7 +106,6 @@ def setup(self): - "/proc/misc", - "/var/log/dmesg", - "/sys/fs/pstore", -- "/var/log/hcnmgr", - clocksource_path + "available_clocksource", - clocksource_path + "current_clocksource" - ]) -diff --git a/sos/report/plugins/powerpc.py b/sos/report/plugins/powerpc.py -index d29eb0a63..c63551cad 100644 ---- a/sos/report/plugins/powerpc.py -+++ b/sos/report/plugins/powerpc.py -@@ -63,7 +63,10 @@ def setup(self): - "/proc/ppc64/systemcfg", - "/var/log/platform", - "/var/log/drmgr", -- "/var/log/drmgr.0" -+ "/var/log/drmgr.0", -+ "/var/log/hcnmgr", -+ "/var/ct/IBM.DRM.stderr", -+ "/var/ct/IW/log/mc/IBM.DRM/trace*" - ]) - ctsnap_path = self.get_cmd_output_path(name="ctsnap", make=True) - self.add_cmd_output([ -@@ -74,8 +77,10 @@ def setup(self): - "serv_config -l", - "bootlist -m both -r", - "lparstat -i", -- "ctsnap -xrunrpttr -d %s" % (ctsnap_path) -+ "ctsnap -xrunrpttr -d %s" % (ctsnap_path), -+ "lsdevinfo" - ]) -+ self.add_service_status("hcn-init") - - if isPowerNV: - self.add_copy_spec([ - -From 692eba8eeec6254bdb356a6bfdc8cfa1f77bfbbc Mon Sep 17 00:00:00 2001 -From: Mamatha Inamdar -Date: Tue, 19 Jan 2021 19:58:53 +0530 -Subject: [PATCH 2/2] [scsi]:Add support to collect SCSI debugging logs - -This patch updates scsi plugin to collect -additional logs for SCSI devices - -Signed-off-by: Mamatha Inamdar ---- - sos/report/plugins/scsi.py | 16 ++++++++++++---- - 1 file changed, 12 insertions(+), 4 deletions(-) - -diff --git a/sos/report/plugins/scsi.py b/sos/report/plugins/scsi.py -index 50cfca0f7..28d1396c6 100644 ---- a/sos/report/plugins/scsi.py -+++ b/sos/report/plugins/scsi.py -@@ -29,10 +29,18 @@ def setup(self): - ]) - - self.add_cmd_output("lsscsi -i", suggest_filename="lsscsi") -- self.add_cmd_output("sg_map -x") -- self.add_cmd_output("lspath") -- self.add_cmd_output("lsmap -all") -- self.add_cmd_output("lsnports") -+ -+ self.add_cmd_output([ -+ "sg_map -x", -+ "lspath", -+ "lsmap -all", -+ "lsnports", -+ "lsscsi -H", -+ "lsscsi -g", -+ "lsscsi -d", -+ "lsscsi -s", -+ "lsscsi -L" -+ ]) - - scsi_hosts = glob("/sys/class/scsi_host/*") - self.add_blockdev_cmd("udevadm info -a %(dev)s", devices=scsi_hosts, diff --git a/SOURCES/sos-bz1930181-collect-cleaning-consistency.patch b/SOURCES/sos-bz1930181-collect-cleaning-consistency.patch new file mode 100644 index 0000000..0ded10a --- /dev/null +++ b/SOURCES/sos-bz1930181-collect-cleaning-consistency.patch @@ -0,0 +1,243 @@ +From fc0218638f3e865c4315823e72aef2f46d012d07 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 14 Apr 2021 11:55:03 -0400 +Subject: [PATCH 1/2] [clean] Load maps from all archives before obfuscation + loop + +Previously, maps were being prepped via archives after extraction. This +reduced the amount of file IO being done, but made it so that necessary +obfuscations from later archives in a series would not be obfuscated in +the archives obfuscated before those later archives were extracted. + +Fix this by extracting the map prep files into memory for each archive +to prep the maps before we enter the obfuscation loop entirely. + +Closes: #2490 +Related: RHBZ#1930181 +Resolves: #2492 + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/__init__.py | 69 +++++++++++++++----------- + sos/cleaner/parsers/username_parser.py | 13 +++-- + 2 files changed, 45 insertions(+), 37 deletions(-) + +diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py +index b9eb61ef..d10cdc55 100644 +--- a/sos/cleaner/__init__.py ++++ b/sos/cleaner/__init__.py +@@ -292,6 +292,7 @@ third party. + + # we have at least one valid target to obfuscate + self.completed_reports = [] ++ self.preload_all_archives_into_maps() + self.obfuscate_report_paths() + + if not self.completed_reports: +@@ -473,6 +474,44 @@ third party. + self.ui_log.info("Exiting on user cancel") + os._exit(130) + ++ def preload_all_archives_into_maps(self): ++ """Before doing the actual obfuscation, if we have multiple archives ++ to obfuscate then we need to preload each of them into the mappings ++ to ensure that node1 is obfuscated in node2 as well as node2 being ++ obfuscated in node1's archive. ++ """ ++ self.log_info("Pre-loading multiple archives into obfuscation maps") ++ for _arc in self.report_paths: ++ is_dir = os.path.isdir(_arc) ++ if is_dir: ++ _arc_name = _arc ++ else: ++ archive = tarfile.open(_arc) ++ _arc_name = _arc.split('/')[-1].split('.tar')[0] ++ # for each parser, load the map_prep_file into memory, and then ++ # send that for obfuscation. We don't actually obfuscate the file ++ # here, do that in the normal archive loop ++ for _parser in self.parsers: ++ if not _parser.prep_map_file: ++ continue ++ _arc_path = os.path.join(_arc_name, _parser.prep_map_file) ++ try: ++ if is_dir: ++ _pfile = open(_arc_path, 'r') ++ content = _pfile.read() ++ else: ++ _pfile = archive.extractfile(_arc_path) ++ content = _pfile.read().decode('utf-8') ++ _pfile.close() ++ if isinstance(_parser, SoSUsernameParser): ++ _parser.load_usernames_into_map(content) ++ for line in content.splitlines(): ++ if isinstance(_parser, SoSHostnameParser): ++ _parser.load_hostname_into_map(line) ++ self.obfuscate_line(line, _parser.prep_map_file) ++ except Exception as err: ++ self.log_debug("Could not prep %s: %s" % (_arc_path, err)) ++ + def obfuscate_report(self, report): + """Individually handle each archive or directory we've discovered by + running through each file therein. +@@ -493,7 +532,6 @@ third party. + start_time = datetime.now() + arc_md.add_field('start_time', start_time) + archive.extract() +- self.prep_maps_from_archive(archive) + archive.report_msg("Beginning obfuscation...") + + file_list = archive.get_file_list() +@@ -542,35 +580,6 @@ third party. + self.ui_log.info("Exception while processing %s: %s" + % (report, err)) + +- def prep_maps_from_archive(self, archive): +- """Open specific files from an archive and try to load those values +- into our mappings before iterating through the entire archive. +- +- Positional arguments: +- +- :param archive SoSObfuscationArchive: An open archive object +- """ +- for parser in self.parsers: +- if not parser.prep_map_file: +- continue +- prep_file = archive.get_file_path(parser.prep_map_file) +- if not prep_file: +- self.log_debug("Could not prepare %s: %s does not exist" +- % (parser.name, parser.prep_map_file), +- caller=archive.archive_name) +- continue +- # this is a bit clunky, but we need to load this particular +- # parser in a different way due to how hostnames are validated for +- # obfuscation +- if isinstance(parser, SoSHostnameParser): +- with open(prep_file, 'r') as host_file: +- hostname = host_file.readline().strip() +- parser.load_hostname_into_map(hostname) +- if isinstance(parser, SoSUsernameParser): +- parser.load_usernames_into_map(prep_file) +- self.obfuscate_file(prep_file, parser.prep_map_file, +- archive.archive_name) +- + def obfuscate_file(self, filename, short_name=None, arc_name=None): + """Obfuscate and individual file, line by line. + +diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py +index 5223c018..2bb6c7f3 100644 +--- a/sos/cleaner/parsers/username_parser.py ++++ b/sos/cleaner/parsers/username_parser.py +@@ -39,16 +39,15 @@ class SoSUsernameParser(SoSCleanerParser): + super(SoSUsernameParser, self).__init__(conf_file) + self.mapping.load_names_from_options(opt_names) + +- def load_usernames_into_map(self, fname): ++ def load_usernames_into_map(self, content): + """Since we don't get the list of usernames from a straight regex for + this parser, we need to override the initial parser prepping here. + """ +- with open(fname, 'r') as lastfile: +- for line in lastfile.read().splitlines()[1:]: +- user = line.split()[0] +- if user in self.skip_list: +- continue +- self.mapping.get(user) ++ for line in content.splitlines()[1:]: ++ user = line.split()[0] ++ if user in self.skip_list: ++ continue ++ self.mapping.get(user) + + def parse_line(self, line): + count = 0 +-- +2.26.3 + + +From b713f458bfa92427147de754ea36054bfde53d71 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 14 Apr 2021 12:22:28 -0400 +Subject: [PATCH 2/2] [clean] Remove duplicate file skipping within + obfuscate_line() + +A redundant file skipping check was being executed within +`obfuscate_line()` that would cause subsequent archives being obfuscated +to skip line obfuscation within a file, despite iterating through the +entire file. + +Remove this redundant check, thus allowing proper obfuscation. + +Closes: #2490 +Related: RHBZ#1930181 +Resolves: #2492 + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/__init__.py | 11 +++-------- + sos/cleaner/obfuscation_archive.py | 2 -- + 2 files changed, 3 insertions(+), 10 deletions(-) + +diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py +index d10cdc55..bdd24f95 100644 +--- a/sos/cleaner/__init__.py ++++ b/sos/cleaner/__init__.py +@@ -508,7 +508,7 @@ third party. + for line in content.splitlines(): + if isinstance(_parser, SoSHostnameParser): + _parser.load_hostname_into_map(line) +- self.obfuscate_line(line, _parser.prep_map_file) ++ self.obfuscate_line(line) + except Exception as err: + self.log_debug("Could not prep %s: %s" % (_arc_path, err)) + +@@ -606,7 +606,7 @@ third party. + if not line.strip(): + continue + try: +- line, count = self.obfuscate_line(line, short_name) ++ line, count = self.obfuscate_line(line) + subs += count + tfile.write(line) + except Exception as err: +@@ -631,7 +631,7 @@ third party. + pass + return string_data + +- def obfuscate_line(self, line, filename): ++ def obfuscate_line(self, line): + """Run a line through each of the obfuscation parsers, keeping a + cumulative total of substitutions done on that particular line. + +@@ -639,16 +639,11 @@ third party. + + :param line str: The raw line as read from the file being + processed +- :param filename str: Filename the line was read from + + Returns the fully obfuscated line and the number of substitutions made + """ + count = 0 + for parser in self.parsers: +- if filename and any([ +- re.match(_s, filename) for _s in parser.skip_files +- ]): +- continue + try: + line, _count = parser.parse_line(line) + count += _count +diff --git a/sos/cleaner/obfuscation_archive.py b/sos/cleaner/obfuscation_archive.py +index 84ca30cd..c64ab13b 100644 +--- a/sos/cleaner/obfuscation_archive.py ++++ b/sos/cleaner/obfuscation_archive.py +@@ -219,8 +219,6 @@ class SoSObfuscationArchive(): + :param filename str: Filename relative to the extracted + archive root + """ +- if filename in self.file_sub_list: +- return True + + if not os.path.isfile(self.get_file_path(filename)): + return True +-- +2.26.3 + diff --git a/SOURCES/sos-bz1935603-manpages-see-also.patch b/SOURCES/sos-bz1935603-manpages-see-also.patch new file mode 100644 index 0000000..6486b48 --- /dev/null +++ b/SOURCES/sos-bz1935603-manpages-see-also.patch @@ -0,0 +1,99 @@ +From 3b439fb64d8d65b0c09aa8452bf0181ec20f8bcf Mon Sep 17 00:00:00 2001 +From: Jose Castillo +Date: Wed, 3 Mar 2021 13:03:16 +0100 +Subject: [PATCH] [man] Multiple fixes in man pages + +This patch fixes references to sosreport, to the +preferred 'sos report'. Also adds "SEE ALSO" consistently +for all man pages, and fixes a MAINTAINER line. + +Resolves: #2432 + +Signed-off-by: Jose Castillo +Signed-off-by: Jake Hunsaker +--- + man/en/sos-clean.1 | 5 +++++ + man/en/sos-collect.1 | 1 + + man/en/sos-report.1 | 22 ++++++++++++++-------- + 3 files changed, 20 insertions(+), 8 deletions(-) + +diff --git a/man/en/sos-clean.1 b/man/en/sos-clean.1 +index 0c62ed07..d64a0ec7 100644 +--- a/man/en/sos-clean.1 ++++ b/man/en/sos-clean.1 +@@ -77,6 +77,11 @@ Default: 4 + .TP + .B \-\-no-update + Do not write the mapping file contents to /etc/sos/cleaner/default_mapping ++.SH SEE ALSO ++.BR sos (1) ++.BR sos-report (1) ++.BR sos-collect (1) ++ + .SH MAINTAINER + .nf + Jake Hunsaker +diff --git a/man/en/sos-collect.1 b/man/en/sos-collect.1 +index d4e5e648..da36542d 100644 +--- a/man/en/sos-collect.1 ++++ b/man/en/sos-collect.1 +@@ -330,6 +330,7 @@ Sosreport option. Override the default compression type. + .SH SEE ALSO + .BR sos (1) + .BR sos-report (1) ++.BR sos-clean (1) + + .SH MAINTAINER + Jake Hunsaker +diff --git a/man/en/sos-report.1 b/man/en/sos-report.1 +index e7fae97b..81005959 100644 +--- a/man/en/sos-report.1 ++++ b/man/en/sos-report.1 +@@ -38,11 +38,12 @@ sosreport \- Collect and package diagnostic and support data + [-h|--help]\fR + + .SH DESCRIPTION +-\fBsosreport\fR generates an archive of configuration and diagnostic +-information from the running system. The archive may be stored locally +-or centrally for recording or tracking purposes or may be sent to +-technical support representatives, developers or system administrators +-to assist with technical fault-finding and debugging. ++\fBreport\fR is an sos subcommand that generates an archive of ++configuration and diagnostic information from the running system. ++The archive may be stored locally or centrally for recording or ++tracking purposes or may be sent to technical support representatives, ++developers or system administrators to assist with technical ++fault-finding and debugging. + .LP + Sos is modular in design and is able to collect data from a wide + range of subsystems and packages that may be installed. An +@@ -110,8 +111,8 @@ User defined presets are saved under /var/lib/sos/presets as JSON-formatted file + .B \--add-preset ADD_PRESET [options] + Add a preset with name ADD_PRESET that enables [options] when called. + +-For example, 'sosreport --add-preset mypreset --log-size=50 -n logs' will enable +-a user to run 'sosreport --preset mypreset' that sets the maximum log size to ++For example, 'sos report --add-preset mypreset --log-size=50 -n logs' will enable ++a user to run 'sos report --preset mypreset' that sets the maximum log size to + 50 and disables the logs plugin. + + Note: to set a description for the preset that is displayed with \fB--list-presets\fR, +@@ -343,9 +344,14 @@ been tested for this port or may still be under active development. + .TP + .B \--help + Display usage message. ++.SH SEE ALSO ++.BR sos (1) ++.BR sos-clean (1) ++.BR sos-collect (1) ++ + .SH MAINTAINER + .nf +-Bryn M. Reeves ++Jake Hunsaker + .fi + .SH AUTHORS & CONTRIBUTORS + See \fBAUTHORS\fR file in the package documentation. +-- +2.26.3 + diff --git a/SOURCES/sos-bz1937298-ds-mask-password-in-ldif.patch b/SOURCES/sos-bz1937298-ds-mask-password-in-ldif.patch new file mode 100644 index 0000000..48aa77a --- /dev/null +++ b/SOURCES/sos-bz1937298-ds-mask-password-in-ldif.patch @@ -0,0 +1,50 @@ +From 153c0154050a111fd7e5bcf4a685f906a1dea737 Mon Sep 17 00:00:00 2001 +From: Jose Castillo +Date: Wed, 10 Mar 2021 15:33:50 +0100 +Subject: [PATCH] [ds] Mask password and encription keys in ldif files + +Both /etc/dirsrv/slapd*/dse.ldif{,.startOK} files contain +sensitive information : +- all the nsSymmetricKey entries : symmetric encryption key +- nsslapd-rootpw : the admin password's hash + +This patch masks these entries in the files we collect. + +Resolves: #2442 + +Signed-off-by: Jose Castillo +Signed-off-by: Jake Hunsaker +--- + sos/report/plugins/ds.py | 18 ++++++++++++++++++ + 1 file changed, 18 insertions(+) + +diff --git a/sos/report/plugins/ds.py b/sos/report/plugins/ds.py +index f4d68d6e..d467dc89 100644 +--- a/sos/report/plugins/ds.py ++++ b/sos/report/plugins/ds.py +@@ -74,4 +74,22 @@ class DirectoryServer(Plugin, RedHatPlugin): + + self.add_cmd_output("ls -l /var/lib/dirsrv/slapd-*/db/*") + ++ def postproc(self): ++ # Example for scrubbing rootpw hash ++ # ++ # nsslapd-rootpw: AAAAB3NzaC1yc2EAAAADAQABAAABAQDeXYA3juyPqaUuyfWV2HuIM ++ # v3gebb/5cvx9ehEAFF2yIKvsQN2EJGTV+hBM1DEOB4eyy/H11NqcNwm/2QsagDB3PVwYp ++ # 9VKN3BdhQjlhuoYKhLwgtYUMiGL8AX5g1qxjirIkTRJwjbXkSNuQaXig7wVjmvXnB2o7B ++ # zLtu99DiL1AizfVeZTYA+OVowYKYaXYljVmVKS+g3t29Obaom54ZLpfuoGMmyO64AJrWs ++ # ++ # to ++ # ++ # nsslapd-rootpw:******** ++ ++ regexppass = r"(nsslapd-rootpw(\s)*:(\s)*)(\S+)([\r\n]\s.*)*\n" ++ regexpkey = r"(nsSymmetricKey(\s)*::(\s)*)(\S+)([\r\n]\s.*)*\n" ++ repl = r"\1********\n" ++ self.do_path_regex_sub('/etc/dirsrv/*', regexppass, repl) ++ self.do_path_regex_sub('/etc/dirsrv/*', regexpkey, repl) ++ + # vim: set et ts=4 sw=4 : +-- +2.26.3 + diff --git a/SOURCES/sos-bz1937418-add-cmd-timeout.patch b/SOURCES/sos-bz1937418-add-cmd-timeout.patch new file mode 100644 index 0000000..db84839 --- /dev/null +++ b/SOURCES/sos-bz1937418-add-cmd-timeout.patch @@ -0,0 +1,315 @@ +From 90b6b709e9f4002376b656b155d00d85382f1828 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Mon, 29 Mar 2021 16:23:01 +0200 +Subject: [PATCH] [report] add --cmd-timeout option + +Add --cmd-timeout option to configure command timeout. Plugin-specific +option of the same name (i.e. -k logs.cmd-timeout=60) can control the +timeout per plugin. + +Option defaults and global/plugin-specific option preference follows the +--plugin-timeout rules. + +Resolves: #2466 + +Signed-off-by: Pavel Moravec +Signed-off-by: Jake Hunsaker +--- + man/en/sos-report.1 | 18 +++++++++- + sos/collector/__init__.py | 3 ++ + sos/collector/sosnode.py | 5 +++ + sos/options.py | 3 +- + sos/report/__init__.py | 5 ++- + sos/report/plugins/__init__.py | 63 ++++++++++++++++++++++++---------- + 6 files changed, 76 insertions(+), 21 deletions(-) + +diff --git a/man/en/sos-report.1 b/man/en/sos-report.1 +index 81005959..51cf3436 100644 +--- a/man/en/sos-report.1 ++++ b/man/en/sos-report.1 +@@ -17,6 +17,7 @@ sosreport \- Collect and package diagnostic and support data + [--label label] [--case-id id]\fR + [--threads threads]\fR + [--plugin-timeout TIMEOUT]\fR ++ [--cmd-timeout TIMEOUT]\fR + [-s|--sysroot SYSROOT]\fR + [-c|--chroot {auto|always|never}\fR + [--tmp-dir directory]\fR +@@ -247,7 +248,7 @@ Specify a timeout in seconds to allow each plugin to run for. A value of 0 + means no timeout will be set. A value of -1 is used to indicate the default + timeout of 300 seconds. + +-Note that this options sets the timeout for all plugins. If you want to set ++Note that this option sets the timeout for all plugins. If you want to set + a timeout for a specific plugin, use the 'timeout' plugin option available to + all plugins - e.g. '-k logs.timeout=600'. + +@@ -255,6 +256,21 @@ The plugin-specific timeout option will override this option. For example, using + \'--plugin-timeout=60 -k logs.timeout=600\' will set a timeout of 600 seconds for + the logs plugin and 60 seconds for all other enabled plugins. + .TP ++.B \--cmd-timeout TIMEOUT ++Specify a timeout limit in seconds for a command execution. Same defaults logic ++from --plugin-timeout applies here. ++ ++This option sets the command timeout for all plugins. If you want to set a cmd ++timeout for a specific plugin, use the 'cmd-timeout' plugin option available to ++all plugins - e.g. '-k logs.cmd-timeout=600'. ++ ++Again, the same plugin/global precedence logic as for --plugin-timeout applies ++here. ++ ++Note that setting --cmd-timeout (or -k logs.cmd-timeout) high should be followed ++by increasing the --plugin-timeout equivalent, otherwise the plugin can easily ++timeout on slow commands execution. ++.TP + .B \--case-id NUMBER + Specify a case identifier to associate with the archive. + Identifiers may include alphanumeric characters, commas and periods ('.'). +diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py +index 406c8f35..1ae73508 100644 +--- a/sos/collector/__init__.py ++++ b/sos/collector/__init__.py +@@ -82,6 +82,7 @@ class SoSCollector(SoSComponent): + 'password_per_node': False, + 'plugin_options': [], + 'plugin_timeout': None, ++ 'cmd_timeout': None, + 'preset': '', + 'save_group': '', + 'since': '', +@@ -276,6 +277,8 @@ class SoSCollector(SoSComponent): + help='Do not collect env vars in sosreports') + sos_grp.add_argument('--plugin-timeout', type=int, default=None, + help='Set the global plugin timeout value') ++ sos_grp.add_argument('--cmd-timeout', type=int, default=None, ++ help='Set the global command timeout value') + sos_grp.add_argument('--since', default=None, + help=('Escapes archived files older than date. ' + 'This will also affect --all-logs. ' +diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py +index a1679655..dbbee12e 100644 +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -664,6 +664,11 @@ class SosNode(): + '--skip-files=%s' % (quote(self.opts.skip_files)) + ) + ++ if self.check_sos_version('4.2'): ++ if self.opts.cmd_timeout: ++ sos_opts.append('--cmd-timeout=%s' ++ % quote(str(self.opts.cmd_timeout))) ++ + sos_cmd = sos_cmd.replace( + 'sosreport', + os.path.join(self.host.sos_bin_path, self.sos_bin) +diff --git a/sos/options.py b/sos/options.py +index b82a7d36..1eda55d6 100644 +--- a/sos/options.py ++++ b/sos/options.py +@@ -283,7 +283,8 @@ class SoSOptions(): + if name in ("add_preset", "del_preset", "desc", "note"): + return False + # Exception list for options that still need to be reported when 0 +- if name in ['log_size', 'plugin_timeout'] and value == 0: ++ if name in ['log_size', 'plugin_timeout', 'cmd_timeout'] \ ++ and value == 0: + return True + return has_value(name, value) + +diff --git a/sos/report/__init__.py b/sos/report/__init__.py +index 25478ba7..945d0fc1 100644 +--- a/sos/report/__init__.py ++++ b/sos/report/__init__.py +@@ -107,6 +107,7 @@ class SoSReport(SoSComponent): + 'only_plugins': [], + 'preset': 'auto', + 'plugin_timeout': 300, ++ 'cmd_timeout': 300, + 'profiles': [], + 'since': None, + 'verify': False, +@@ -266,6 +267,8 @@ class SoSReport(SoSComponent): + help="A preset identifier", default="auto") + report_grp.add_argument("--plugin-timeout", default=None, + help="set a timeout for all plugins") ++ report_grp.add_argument("--cmd-timeout", default=None, ++ help="set a command timeout for all plugins") + report_grp.add_argument("-p", "--profile", "--profiles", + action="extend", dest="profiles", type=str, + default=[], +@@ -709,7 +712,7 @@ class SoSReport(SoSComponent): + + self.ui_log.info(_("The following plugin options are available:")) + for (plug, plugname, optname, optparm) in self.all_options: +- if optname in ('timeout', 'postproc'): ++ if optname in ('timeout', 'postproc', 'cmd-timeout'): + continue + # format option value based on its type (int or bool) + if type(optparm["enabled"]) == bool: +diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py +index 02625eb1..779119af 100644 +--- a/sos/report/plugins/__init__.py ++++ b/sos/report/plugins/__init__.py +@@ -472,6 +472,9 @@ class Plugin(object): + _default_plug_opts = [ + ('timeout', 'Timeout in seconds for plugin. The default value (-1) ' + + 'defers to the general plugin timeout, 300 seconds', 'fast', -1), ++ ('cmd-timeout', 'Timeout in seconds for a command execution. The ' + ++ 'default value (-1) defers to the general cmd timeout, 300 ' + ++ 'seconds', 'fast', -1), + ('postproc', 'Enable post-processing collected plugin data', 'fast', + True) + ] +@@ -532,16 +535,15 @@ class Plugin(object): + self.manifest.add_list('commands', []) + self.manifest.add_list('files', []) + +- @property +- def timeout(self): +- """Returns either the default plugin timeout value, the value as +- provided on the commandline via -k plugin.timeout=value, or the value +- of the global --plugin-timeout option. ++ def timeout_from_options(self, optname, plugoptname, default_timeout): ++ """Returns either the default [plugin|cmd] timeout value, the value as ++ provided on the commandline via -k plugin.[|cmd-]timeout=value, or the ++ value of the global --[plugin|cmd]-timeout option. + """ + _timeout = None + try: +- opt_timeout = self.get_option('plugin_timeout') +- own_timeout = int(self.get_option('timeout')) ++ opt_timeout = self.get_option(optname) ++ own_timeout = int(self.get_option(plugoptname)) + if opt_timeout is None: + _timeout = own_timeout + elif opt_timeout is not None and own_timeout == -1: +@@ -551,10 +553,30 @@ class Plugin(object): + else: + return None + except ValueError: +- return self.plugin_timeout # Default to known safe value ++ return default_timeout # Default to known safe value + if _timeout is not None and _timeout > -1: + return _timeout +- return self.plugin_timeout ++ return default_timeout ++ ++ @property ++ def timeout(self): ++ """Returns either the default plugin timeout value, the value as ++ provided on the commandline via -k plugin.timeout=value, or the value ++ of the global --plugin-timeout option. ++ """ ++ _timeout = self.timeout_from_options('plugin_timeout', 'timeout', ++ self.plugin_timeout) ++ return _timeout ++ ++ @property ++ def cmdtimeout(self): ++ """Returns either the default command timeout value, the value as ++ provided on the commandline via -k plugin.cmd-timeout=value, or the ++ value of the global --cmd-timeout option. ++ """ ++ _cmdtimeout = self.timeout_from_options('cmd_timeout', 'cmd-timeout', ++ self.cmd_timeout) ++ return _cmdtimeout + + def set_timeout_hit(self): + self._timeout_hit = True +@@ -1235,8 +1257,8 @@ class Plugin(object): + """ + + global_options = ( +- 'all_logs', 'allow_system_changes', 'log_size', 'plugin_timeout', +- 'since', 'verify' ++ 'all_logs', 'allow_system_changes', 'cmd_timeout', 'log_size', ++ 'plugin_timeout', 'since', 'verify' + ) + + if optionname in global_options: +@@ -1505,7 +1527,7 @@ class Plugin(object): + 'tags': _spec_tags + }) + +- def add_blockdev_cmd(self, cmds, devices='block', timeout=300, ++ def add_blockdev_cmd(self, cmds, devices='block', timeout=None, + sizelimit=None, chroot=True, runat=None, env=None, + binary=False, prepend_path=None, whitelist=[], + blacklist=[], tags=[]): +@@ -1569,7 +1591,7 @@ class Plugin(object): + whitelist=whitelist, blacklist=blacklist, + tags=_dev_tags) + +- def _add_device_cmd(self, cmds, devices, timeout=300, sizelimit=None, ++ def _add_device_cmd(self, cmds, devices, timeout=None, sizelimit=None, + chroot=True, runat=None, env=None, binary=False, + prepend_path=None, whitelist=[], blacklist=[], + tags=[]): +@@ -1627,7 +1649,7 @@ class Plugin(object): + changes=soscmd.changes) + + def add_cmd_output(self, cmds, suggest_filename=None, +- root_symlink=None, timeout=cmd_timeout, stderr=True, ++ root_symlink=None, timeout=None, stderr=True, + chroot=True, runat=None, env=None, binary=False, + sizelimit=None, pred=None, subdir=None, + changes=False, foreground=False, tags=[]): +@@ -1849,7 +1871,7 @@ class Plugin(object): + self._log_debug("added string ...'%s' as '%s'" % (summary, filename)) + + def _collect_cmd_output(self, cmd, suggest_filename=None, +- root_symlink=False, timeout=cmd_timeout, ++ root_symlink=False, timeout=None, + stderr=True, chroot=True, runat=None, env=None, + binary=False, sizelimit=None, subdir=None, + changes=False, foreground=False, tags=[]): +@@ -1883,6 +1905,8 @@ class Plugin(object): + if self._timeout_hit: + return + ++ if timeout is None: ++ timeout = self.cmdtimeout + _tags = [] + + if isinstance(tags, str): +@@ -1975,7 +1999,7 @@ class Plugin(object): + return result + + def collect_cmd_output(self, cmd, suggest_filename=None, +- root_symlink=False, timeout=cmd_timeout, ++ root_symlink=False, timeout=None, + stderr=True, chroot=True, runat=None, env=None, + binary=False, sizelimit=None, pred=None, + subdir=None, tags=[]): +@@ -2044,7 +2068,7 @@ class Plugin(object): + tags=tags + ) + +- def exec_cmd(self, cmd, timeout=cmd_timeout, stderr=True, chroot=True, ++ def exec_cmd(self, cmd, timeout=None, stderr=True, chroot=True, + runat=None, env=None, binary=False, pred=None, + foreground=False, container=False, quotecmd=False): + """Execute a command right now and return the output and status, but +@@ -2095,6 +2119,9 @@ class Plugin(object): + if not self.test_predicate(cmd=True, pred=pred): + return _default + ++ if timeout is None: ++ timeout = self.cmdtimeout ++ + if chroot or self.commons['cmdlineopts'].chroot == 'always': + root = self.sysroot + else: +@@ -2331,7 +2358,7 @@ class Plugin(object): + + def add_journal(self, units=None, boot=None, since=None, until=None, + lines=None, allfields=False, output=None, +- timeout=cmd_timeout, identifier=None, catalog=None, ++ timeout=None, identifier=None, catalog=None, + sizelimit=None, pred=None, tags=[]): + """Collect journald logs from one of more units. + +-- +2.26.3 + diff --git a/SOURCES/sos-bz1939963-gather-cups-browsed-logs.patch b/SOURCES/sos-bz1939963-gather-cups-browsed-logs.patch new file mode 100644 index 0000000..3e6c393 --- /dev/null +++ b/SOURCES/sos-bz1939963-gather-cups-browsed-logs.patch @@ -0,0 +1,30 @@ +From 0d56e43299009ffa91f665d85b5a08ba76da9c1f Mon Sep 17 00:00:00 2001 +From: Jose Castillo +Date: Wed, 17 Mar 2021 13:10:36 +0100 +Subject: [PATCH] [cups] Add gathering cups-browsed logs + +Gather logs from the service cups-browsed sent +to the journal. + +Resolves: #2452 + +Signed-off-by: Jose Castillo +Signed-off-by: Bryan Quigley +--- + sos/report/plugins/cups.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/sos/report/plugins/cups.py b/sos/report/plugins/cups.py +index 29a903e8..ab7b6b70 100644 +--- a/sos/report/plugins/cups.py ++++ b/sos/report/plugins/cups.py +@@ -40,5 +40,6 @@ class Cups(Plugin, IndependentPlugin): + ]) + + self.add_journal(units="cups") ++ self.add_journal(units="cups-browsed") + + # vim: set et ts=4 sw=4 : +-- +2.26.3 + diff --git a/SOURCES/sos-bz1940502-sssd-memcache-and-logs.patch b/SOURCES/sos-bz1940502-sssd-memcache-and-logs.patch new file mode 100644 index 0000000..ebc7578 --- /dev/null +++ b/SOURCES/sos-bz1940502-sssd-memcache-and-logs.patch @@ -0,0 +1,62 @@ +From d03c2fa4439c87783293c922b2825cf86e8818bd Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pawe=C5=82=20Po=C5=82awski?= +Date: Fri, 12 Mar 2021 12:42:30 +0100 +Subject: [PATCH] [sssd] Enable collecting SSSD memory cache +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +SSSD plugin by default collects only logs and configuration. +This patch enables collecting memory cache maintained +by SSSD daemon. Cache does not contain any client sensible +data so can be safely included in the sos-report. + +Resolves: #2444 + +Signed-off-by: Paweł Poławski +Signed-off-by: Jake Hunsaker +--- + sos/report/plugins/sssd.py | 8 ++++++-- + 1 file changed, 6 insertions(+), 2 deletions(-) + +diff --git a/sos/report/plugins/sssd.py b/sos/report/plugins/sssd.py +index 9469c41c..aeb68c4f 100644 +--- a/sos/report/plugins/sssd.py ++++ b/sos/report/plugins/sssd.py +@@ -10,6 +10,7 @@ + + from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, + UbuntuPlugin, SoSPredicate) ++from glob import glob + + + class Sssd(Plugin): +@@ -22,11 +23,22 @@ class Sssd(Plugin): + + def setup(self): + self.add_copy_spec([ ++ # main config file + "/etc/sssd/sssd.conf", +- "/var/log/sssd/*", +- "/var/lib/sss/pubconf/krb5.include.d/*", + # SSSD 1.14 +- "/etc/sssd/conf.d/*.conf" ++ "/etc/sssd/conf.d/*.conf", ++ # dynamic Kerberos configuration ++ "/var/lib/sss/pubconf/krb5.include.d/*" ++ ]) ++ ++ # add individual log files ++ self.add_copy_spec(glob("/var/log/sssd/*log*")) ++ ++ # add memory cache ++ self.add_copy_spec([ ++ "/var/lib/sss/mc/passwd", ++ "/var/lib/sss/mc/group", ++ "/var/lib/sss/mc/initgroups" + ]) + + # call sssctl commands only when sssd service is running, +-- +2.26.3 + diff --git a/SOURCES/sos-bz1942276-ibmvNIC-dynamic-debugs.patch b/SOURCES/sos-bz1942276-ibmvNIC-dynamic-debugs.patch new file mode 100644 index 0000000..7bb7fd7 --- /dev/null +++ b/SOURCES/sos-bz1942276-ibmvNIC-dynamic-debugs.patch @@ -0,0 +1,29 @@ +From dddabb07a88d398ed7b8a878e95acfd968af6698 Mon Sep 17 00:00:00 2001 +From: Mamatha Inamdar +Date: Tue, 23 Mar 2021 17:58:30 +0530 +Subject: [PATCH] This patch is to update kernel plugin to collect + dynamic_debug log files for ibmvNIC + +Resolves: #2458 + +Signed-off-by: Mamatha Inamdar +Signed-off-by: Bryan Quigley +--- + sos/report/plugins/kernel.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/sos/report/plugins/kernel.py b/sos/report/plugins/kernel.py +index febe2ad0..dd7b6939 100644 +--- a/sos/report/plugins/kernel.py ++++ b/sos/report/plugins/kernel.py +@@ -106,6 +106,7 @@ class Kernel(Plugin, IndependentPlugin): + "/proc/misc", + "/var/log/dmesg", + "/sys/fs/pstore", ++ "/sys/kernel/debug/dynamic_debug/control", + clocksource_path + "available_clocksource", + clocksource_path + "current_clocksource" + ]) +-- +2.26.3 + diff --git a/SOURCES/sos-bz1956673-pulpcore-plugin.patch b/SOURCES/sos-bz1956673-pulpcore-plugin.patch new file mode 100644 index 0000000..e60a494 --- /dev/null +++ b/SOURCES/sos-bz1956673-pulpcore-plugin.patch @@ -0,0 +1,147 @@ +From 808d9f35ac504a58c337ffed14b39119a591808f Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Tue, 27 Apr 2021 22:16:08 +0200 +Subject: [PATCH] [pulpcore] add plugin for pulp-3 + +Pulp-3 / pulpcore as a revolution from pulp-2 needs a separate +plugin, since both plugins have nothing in common and there might +be deployments where is active both pulp-2 and pulp-3. + +Resolves: #2278 + +Signed-off-by: Pavel Moravec +Signed-off-by: Jake Hunsaker +--- + sos/report/plugins/pulpcore.py | 120 +++++++++++++++++++++++++++++++++ + 1 file changed, 120 insertions(+) + create mode 100644 sos/report/plugins/pulpcore.py + +diff --git a/sos/report/plugins/pulpcore.py b/sos/report/plugins/pulpcore.py +new file mode 100644 +index 00000000..20403814 +--- /dev/null ++++ b/sos/report/plugins/pulpcore.py +@@ -0,0 +1,120 @@ ++# Copyright (C) 2021 Red Hat, Inc., Pavel Moravec ++ ++# This file is part of the sos project: https://github.com/sosreport/sos ++# ++# This copyrighted material is made available to anyone wishing to use, ++# modify, copy, or redistribute it subject to the terms and conditions of ++# version 2 of the GNU General Public License. ++# ++# See the LICENSE file in the source distribution for further information. ++ ++from sos.report.plugins import Plugin, IndependentPlugin ++from pipes import quote ++from re import match ++ ++ ++class PulpCore(Plugin, IndependentPlugin): ++ ++ short_desc = 'Pulp-3 aka pulpcore' ++ ++ plugin_name = "pulpcore" ++ commands = ("pulpcore-manager",) ++ files = ("/etc/pulp/settings.py",) ++ option_list = [ ++ ('task-days', 'days of tasks history', 'fast', 7) ++ ] ++ ++ def parse_settings_config(self): ++ databases_scope = False ++ self.dbhost = "localhost" ++ self.dbport = 5432 ++ self.dbpasswd = "" ++ # TODO: read also redis config (we dont expect much customisations) ++ # TODO: read also db user (pulp) and database name (pulpcore) ++ self.staticroot = "/var/lib/pulp/assets" ++ self.uploaddir = "/var/lib/pulp/media/upload" ++ ++ def separate_value(line, sep=':'): ++ # an auxiliary method to parse values from lines like: ++ # 'HOST': 'localhost', ++ val = line.split(sep)[1].lstrip().rstrip(',') ++ if (val.startswith('"') and val.endswith('"')) or \ ++ (val.startswith('\'') and val.endswith('\'')): ++ val = val[1:-1] ++ return val ++ ++ try: ++ for line in open("/etc/pulp/settings.py").read().splitlines(): ++ # skip empty lines and lines with comments ++ if not line or line[0] == '#': ++ continue ++ if line.startswith("DATABASES"): ++ databases_scope = True ++ continue ++ # example HOST line to parse: ++ # 'HOST': 'localhost', ++ if databases_scope and match(r"\s+'HOST'\s*:\s+\S+", line): ++ self.dbhost = separate_value(line) ++ if databases_scope and match(r"\s+'PORT'\s*:\s+\S+", line): ++ self.dbport = separate_value(line) ++ if databases_scope and match(r"\s+'PASSWORD'\s*:\s+\S+", line): ++ self.dbpasswd = separate_value(line) ++ # if line contains closing '}' database_scope end ++ if databases_scope and '}' in line: ++ databases_scope = False ++ if line.startswith("STATIC_ROOT = "): ++ self.staticroot = separate_value(line, sep='=') ++ if line.startswith("CHUNKED_UPLOAD_DIR = "): ++ self.uploaddir = separate_value(line, sep='=') ++ except IOError: ++ # fallback when the cfg file is not accessible ++ pass ++ # set the password to os.environ when calling psql commands to prevent ++ # printing it in sos logs ++ # we can't set os.environ directly now: other plugins can overwrite it ++ self.env = {"PGPASSWORD": self.dbpasswd} ++ ++ def setup(self): ++ self.parse_settings_config() ++ ++ self.add_copy_spec("/etc/pulp/settings.py") ++ ++ self.add_cmd_output("rq info -u redis://localhost:6379/8", ++ env={"LC_ALL": "en_US.UTF-8"}, ++ suggest_filename="rq_info") ++ self.add_cmd_output("curl -ks https://localhost/pulp/api/v3/status/", ++ suggest_filename="pulp_status") ++ dynaconf_env = {"LC_ALL": "en_US.UTF-8", ++ "PULP_SETTINGS": "/etc/pulp/settings.py", ++ "DJANGO_SETTINGS_MODULE": "pulpcore.app.settings"} ++ self.add_cmd_output("dynaconf list", env=dynaconf_env) ++ for _dir in [self.staticroot, self.uploaddir]: ++ self.add_cmd_output("ls -l %s" % _dir) ++ ++ task_days = self.get_option('task-days') ++ for table in ['core_task', 'core_taskgroup', ++ 'core_reservedresourcerecord', ++ 'core_taskreservedresourcerecord', ++ 'core_groupprogressreport', 'core_progressreport']: ++ _query = "select * from %s where pulp_last_updated > NOW() - " \ ++ "interval '%s days' order by pulp_last_updated" % \ ++ (table, task_days) ++ _cmd = "psql -h %s -p %s -U pulp -d pulpcore -c %s" % \ ++ (self.dbhost, self.dbport, quote(_query)) ++ self.add_cmd_output(_cmd, env=self.env, suggest_filename=table) ++ ++ def postproc(self): ++ # TODO obfuscate from /etc/pulp/settings.py : ++ # SECRET_KEY = "eKfeDkTnvss7p5WFqYdGPWxXfHnsbDBx" ++ # 'PASSWORD': 'tGrag2DmtLqKLTWTQ6U68f6MAhbqZVQj', ++ self.do_path_regex_sub( ++ "/etc/pulp/settings.py", ++ r"(SECRET_KEY\s*=\s*)(.*)", ++ r"\1********") ++ self.do_path_regex_sub( ++ "/etc/pulp/settings.py", ++ r"(PASSWORD\S*\s*:\s*)(.*)", ++ r"\1********") ++ ++ ++# vim: set et ts=4 sw=4 : +-- +2.26.3 + diff --git a/SOURCES/sos-bz1959413-saphana-traceback.patch b/SOURCES/sos-bz1959413-saphana-traceback.patch new file mode 100644 index 0000000..4b784dc --- /dev/null +++ b/SOURCES/sos-bz1959413-saphana-traceback.patch @@ -0,0 +1,30 @@ +From c998ea8c1c950586f91fc9728ee66590740968a5 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Tue, 11 May 2021 15:59:40 +0200 +Subject: [PATCH] [saphana] remove redundant unused argument of get_inst_info + +get_inst_info does not use and isnt called with 'prefix' argument + +Resolves: #2535 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/saphana.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/sos/report/plugins/saphana.py b/sos/report/plugins/saphana.py +index 82c497b4..00e84b59 100644 +--- a/sos/report/plugins/saphana.py ++++ b/sos/report/plugins/saphana.py +@@ -51,7 +51,7 @@ class saphana(Plugin, RedHatPlugin): + inst = inst.strip()[-2:] + self.get_inst_info(sid, sidadm, inst) + +- def get_inst_info(self, prefix, sid, sidadm, inst): ++ def get_inst_info(self, sid, sidadm, inst): + proc_cmd = 'su - %s -c "sapcontrol -nr %s -function GetProcessList"' + status_fname = "%s_%s_status" % (sid, inst) + self.add_cmd_output( +-- +2.26.3 + diff --git a/SOURCES/sos-bz1959598-conversions-and-upgrades.patch b/SOURCES/sos-bz1959598-conversions-and-upgrades.patch new file mode 100644 index 0000000..a39f839 --- /dev/null +++ b/SOURCES/sos-bz1959598-conversions-and-upgrades.patch @@ -0,0 +1,50 @@ +From ee5d9d017b0a1bfeaebee9c21c17e89ef1f909a8 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Mon, 26 Jul 2021 13:30:09 +0200 +Subject: [PATCH] [MigrationResults] collect info about conversions and + upgrades + +A new tiny plugin independent on leapp and convert2rhel is proposed. + +It should collect /etc/migration-results with info about RHEL +conversions and upgrades, whenever the file is present. + +Resolves: #2627 +Relevant to: rhbz#1959598 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/migration_results.py | 21 +++++++++++++++++++++ + 1 file changed, 21 insertions(+) + create mode 100644 sos/report/plugins/migration_results.py + +diff --git a/sos/report/plugins/migration_results.py b/sos/report/plugins/migration_results.py +new file mode 100644 +index 00000000..b67480ba +--- /dev/null ++++ b/sos/report/plugins/migration_results.py +@@ -0,0 +1,21 @@ ++# This file is part of the sos project: https://github.com/sosreport/sos ++# ++# This copyrighted material is made available to anyone wishing to use, ++# modify, copy, or redistribute it subject to the terms and conditions of ++# version 2 of the GNU General Public License. ++# ++# See the LICENSE file in the source distribution for further information. ++ ++from sos.report.plugins import Plugin, RedHatPlugin ++ ++ ++class MigrationResults(Plugin, RedHatPlugin): ++ ++ short_desc = 'Information about conversions and upgrades' ++ ++ plugin_name = 'migration_results' ++ profiles = ('system',) ++ ++ files = ('/etc/migration-results',) ++ ++# vim: et ts=4 sw=4 +-- +2.31.1 + diff --git a/SOURCES/sos-bz1961229-snapper-plugin-and-allocation-failures.patch b/SOURCES/sos-bz1961229-snapper-plugin-and-allocation-failures.patch new file mode 100644 index 0000000..e33a89e --- /dev/null +++ b/SOURCES/sos-bz1961229-snapper-plugin-and-allocation-failures.patch @@ -0,0 +1,121 @@ +From 60105e0705f3483b9a3e8e98dafd6f0e1e277ab7 Mon Sep 17 00:00:00 2001 +From: Mamatha Inamdar +Date: Mon, 19 Apr 2021 16:55:52 +0530 +Subject: [PATCH 1/3] [block]:Patch to update block pluging to collect disk + info + +This patch is to update block plugin to collect +state of sda + +Resolves: #2504 + +Signed-off-by: Mamatha Inamdar +--- + sos/report/plugins/block.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/sos/report/plugins/block.py b/sos/report/plugins/block.py +index f93b3231..c959d667 100644 +--- a/sos/report/plugins/block.py ++++ b/sos/report/plugins/block.py +@@ -38,7 +38,8 @@ class Block(Plugin, IndependentPlugin): + "/run/blkid/blkid.tab", + "/proc/partitions", + "/proc/diskstats", +- "/sys/block/*/queue/" ++ "/sys/block/*/queue/", ++ "/sys/block/sd*/device/state", + ]) + + cmds = [ +-- +2.26.3 + + +From c6e0fe5cebd0d9581950db75fa2d234713b7e15a Mon Sep 17 00:00:00 2001 +From: Mamatha Inamdar +Date: Mon, 26 Apr 2021 23:09:19 +0530 +Subject: [PATCH 2/3] [snapper]:Ptach to update snapper plugin to collect + snapper info + +This patch is to Introduce snapper plugin to collect +/usr/lib/snapper/ information to check executable +permission for installation-helper command + +Resolves: #2504 + +Signed-off-by: Mamatha Inamdar +--- + sos/report/plugins/snapper.py | 27 +++++++++++++++++++++++++++ + 1 file changed, 27 insertions(+) + create mode 100644 sos/report/plugins/snapper.py + +diff --git a/sos/report/plugins/snapper.py b/sos/report/plugins/snapper.py +new file mode 100644 +index 00000000..9ef5fec2 +--- /dev/null ++++ b/sos/report/plugins/snapper.py +@@ -0,0 +1,27 @@ ++# This file is part of the sos project: https://github.com/sosreport/sos ++# ++# This copyrighted material is made available to anyone wishing to use, ++# modify, copy, or redistribute it subject to the terms and conditions of ++# version 2 of the GNU General Public License. ++# ++# See the LICENSE file in the source distribution for further information. ++ ++from sos.report.plugins import Plugin, IndependentPlugin ++ ++ ++class Snapper(Plugin, IndependentPlugin): ++ ++ short_desc = 'System snapper' ++ ++ plugin_name = 'snapper' ++ commands = ("snapper",) ++ ++ def setup(self): ++ ++ self.add_cmd_output([ ++ "ls -la /usr/lib/snapper/", ++ "snapper --version", ++ "snapper list" ++ ]) ++ ++# vim: set et ts=4 sw=4 : +-- +2.26.3 + + +From 61ff5ce165e654a02fe80b9de5ec8e49ed808ec9 Mon Sep 17 00:00:00 2001 +From: Mamatha Inamdar +Date: Mon, 19 Apr 2021 17:49:08 +0530 +Subject: [PATCH 3/3] [kernel]:Patch to update kernel plugin to collect debug + info + +This patch is to update kernel plugin to collect +page allocation failure info + +Resolves: #2504 + +Signed-off-by: Mamatha Inamdar +--- + sos/report/plugins/kernel.py | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/sos/report/plugins/kernel.py b/sos/report/plugins/kernel.py +index dd7b6939..9d53ca03 100644 +--- a/sos/report/plugins/kernel.py ++++ b/sos/report/plugins/kernel.py +@@ -107,6 +107,8 @@ class Kernel(Plugin, IndependentPlugin): + "/var/log/dmesg", + "/sys/fs/pstore", + "/sys/kernel/debug/dynamic_debug/control", ++ "/sys/kernel/debug/extfrag/unusable_index", ++ "/sys/kernel/debug/extfrag/extfrag_index", + clocksource_path + "available_clocksource", + clocksource_path + "current_clocksource" + ]) +-- +2.26.3 + diff --git a/SOURCES/sos-bz1961458-collect-nstat.patch b/SOURCES/sos-bz1961458-collect-nstat.patch new file mode 100644 index 0000000..75b7d29 --- /dev/null +++ b/SOURCES/sos-bz1961458-collect-nstat.patch @@ -0,0 +1,36 @@ +From 575ddeddf2f6e1d6a639922f9ccc51c7e46fbe12 Mon Sep 17 00:00:00 2001 +From: Seiichi Ikarashi +Date: Fri, 14 May 2021 09:49:33 +0900 +Subject: [PATCH] [networking] Add nstat command support + +As netstat command is being deprecated, +we need nstat as an alternative to "netstat -s". + +Signed-off-by: Seiichi Ikarashi +--- + sos/report/plugins/networking.py | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/sos/report/plugins/networking.py b/sos/report/plugins/networking.py +index 8b4614bb..acfa027f 100644 +--- a/sos/report/plugins/networking.py ++++ b/sos/report/plugins/networking.py +@@ -87,6 +87,7 @@ class Networking(Plugin): + root_symlink="netstat") + + self.add_cmd_output([ ++ "nstat -zas", + "netstat -s", + "netstat %s -agn" % self.ns_wide, + "ip route show table all", +@@ -198,6 +199,7 @@ class Networking(Plugin): + ns_cmd_prefix + "netstat %s -neopa" % self.ns_wide, + ns_cmd_prefix + "netstat -s", + ns_cmd_prefix + "netstat %s -agn" % self.ns_wide, ++ ns_cmd_prefix + "nstat -zas", + ]) + + ss_cmd = ns_cmd_prefix + "ss -peaonmi" +-- +2.26.3 + diff --git a/SOURCES/sos-bz1964499-obfuscate-fqdn-from-dnf-log.patch b/SOURCES/sos-bz1964499-obfuscate-fqdn-from-dnf-log.patch new file mode 100644 index 0000000..07e005b --- /dev/null +++ b/SOURCES/sos-bz1964499-obfuscate-fqdn-from-dnf-log.patch @@ -0,0 +1,78 @@ +From b27140a9126ea82efb517d60bf1b8455aaf4f5a6 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Fri, 26 Mar 2021 11:12:33 -0400 +Subject: [PATCH] [cleaner] Only skip packaging-based files for the IP parser + +Files primarily containing package information, e.g. `installed-rpms` or +`installed-debs`, were previously being skipped by all parsers. In +reality, we only need to skip these for the IP parser due to the fact +that version numbers often generate a match for IP address regexes. + +This will also fix a problem where if a system was the build host for +certain packages, the hostname would remain in these files as the +hostname parser was previously not checking these files. + +Closes: #2400 +Resolves: #2464 + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/obfuscation_archive.py | 10 ---------- + sos/cleaner/parsers/ip_parser.py | 16 ++++++++++++++++ + 2 files changed, 16 insertions(+), 10 deletions(-) + +diff --git a/sos/cleaner/obfuscation_archive.py b/sos/cleaner/obfuscation_archive.py +index 981cc05f..84ca30cd 100644 +--- a/sos/cleaner/obfuscation_archive.py ++++ b/sos/cleaner/obfuscation_archive.py +@@ -59,20 +59,10 @@ class SoSObfuscationArchive(): + Returns: list of files and file regexes + """ + return [ +- 'installed-debs', +- 'installed-rpms', +- 'sos_commands/dpkg', +- 'sos_commands/python/pip_list', +- 'sos_commands/rpm', +- 'sos_commands/yum/.*list.*', +- 'sos_commands/snappy/snap_list_--all', +- 'sos_commands/snappy/snap_--version', +- 'sos_commands/vulkan/vulkaninfo', + 'sys/firmware', + 'sys/fs', + 'sys/kernel/debug', + 'sys/module', +- 'var/log/.*dnf.*', + r'.*\.tar$', # TODO: support archive unpacking + # Be explicit with these tar matches to avoid matching commands + r'.*\.tar\.xz', +diff --git a/sos/cleaner/parsers/ip_parser.py b/sos/cleaner/parsers/ip_parser.py +index 3ea7f865..08d1cd05 100644 +--- a/sos/cleaner/parsers/ip_parser.py ++++ b/sos/cleaner/parsers/ip_parser.py +@@ -24,6 +24,22 @@ class SoSIPParser(SoSCleanerParser): + # don't match package versions recorded in journals + r'.*dnf\[.*\]:' + ] ++ ++ skip_files = [ ++ # skip these as version numbers will frequently look like IP addresses ++ # when using regex matching ++ 'installed-debs', ++ 'installed-rpms', ++ 'sos_commands/dpkg', ++ 'sos_commands/python/pip_list', ++ 'sos_commands/rpm', ++ 'sos_commands/yum/.*list.*', ++ 'sos_commands/snappy/snap_list_--all', ++ 'sos_commands/snappy/snap_--version', ++ 'sos_commands/vulkan/vulkaninfo', ++ 'var/log/.*dnf.*' ++ ] ++ + map_file_key = 'ip_map' + prep_map_file = 'sos_commands/networking/ip_-o_addr' + +-- +2.26.3 + diff --git a/SOURCES/sos-bz1965001-fix-avc-copystating-proc-sys.patch b/SOURCES/sos-bz1965001-fix-avc-copystating-proc-sys.patch new file mode 100644 index 0000000..1b62a1a --- /dev/null +++ b/SOURCES/sos-bz1965001-fix-avc-copystating-proc-sys.patch @@ -0,0 +1,135 @@ +From 206d65618f20995b168dcc63090d1e6871450e90 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Wed, 26 May 2021 15:45:26 +0200 +Subject: [PATCH] [archive] skip copying SELinux context for /proc and /sys + everytime + +A supplement of #1399 fix, now also for adding strings or special +device files. + +Also adding a (vendor) test case for it. + +Resolves: #2560 + +Signed-off-by: Pavel Moravec +--- + sos/archive.py | 35 +++++++++++---------- + tests/vendor_tests/redhat/rhbz1965001.py | 39 ++++++++++++++++++++++++ + 2 files changed, 56 insertions(+), 18 deletions(-) + create mode 100644 tests/vendor_tests/redhat/rhbz1965001.py + +diff --git a/sos/archive.py b/sos/archive.py +index 4dd31d75..b02b2475 100644 +--- a/sos/archive.py ++++ b/sos/archive.py +@@ -326,6 +326,20 @@ class FileCacheArchive(Archive): + return None + return dest + ++ def _copy_attributes(self, src, dest): ++ # copy file attributes, skip SELinux xattrs for /sys and /proc ++ try: ++ stat = os.stat(src) ++ if src.startswith("/sys/") or src.startswith("/proc/"): ++ shutil.copymode(src, dest) ++ os.utime(dest, ns=(stat.st_atime_ns, stat.st_mtime_ns)) ++ else: ++ shutil.copystat(src, dest) ++ os.chown(dest, stat.st_uid, stat.st_gid) ++ except Exception as e: ++ self.log_debug("caught '%s' setting attributes of '%s'" ++ % (e, dest)) ++ + def add_file(self, src, dest=None): + with self._path_lock: + if not dest: +@@ -348,18 +362,7 @@ class FileCacheArchive(Archive): + else: + self.log_info("File %s not collected: '%s'" % (src, e)) + +- # copy file attributes, skip SELinux xattrs for /sys and /proc +- try: +- stat = os.stat(src) +- if src.startswith("/sys/") or src.startswith("/proc/"): +- shutil.copymode(src, dest) +- os.utime(dest, ns=(stat.st_atime_ns, stat.st_mtime_ns)) +- else: +- shutil.copystat(src, dest) +- os.chown(dest, stat.st_uid, stat.st_gid) +- except Exception as e: +- self.log_debug("caught '%s' setting attributes of '%s'" +- % (e, dest)) ++ self._copy_attributes(src, dest) + file_name = "'%s'" % src + else: + # Open file case: first rewind the file to obtain +@@ -388,11 +391,7 @@ class FileCacheArchive(Archive): + content = content.decode('utf8', 'ignore') + f.write(content) + if os.path.exists(src): +- try: +- shutil.copystat(src, dest) +- except OSError as e: +- self.log_error("Unable to add '%s' to archive: %s" % +- (dest, e)) ++ self._copy_attributes(src, dest) + self.log_debug("added string at '%s' to FileCacheArchive '%s'" + % (src, self._archive_root)) + +@@ -501,7 +500,7 @@ class FileCacheArchive(Archive): + self.log_info("add_node: %s - mknod '%s'" % (msg, dest)) + return + raise e +- shutil.copystat(path, dest) ++ self._copy_attributes(path, dest) + + def name_max(self): + if 'PC_NAME_MAX' in os.pathconf_names: +diff --git a/tests/vendor_tests/redhat/rhbz1965001.py b/tests/vendor_tests/redhat/rhbz1965001.py +new file mode 100644 +index 00000000..aa16ba81 +--- /dev/null ++++ b/tests/vendor_tests/redhat/rhbz1965001.py +@@ -0,0 +1,39 @@ ++# This file is part of the sos project: https://github.com/sosreport/sos ++# ++# This copyrighted material is made available to anyone wishing to use, ++# modify, copy, or redistribute it subject to the terms and conditions of ++# version 2 of the GNU General Public License. ++# ++# See the LICENSE file in the source distribution for further information. ++ ++ ++import tempfile ++import shutil ++from sos_tests import StageOneReportTest ++ ++ ++class rhbz1965001(StageOneReportTest): ++ """ ++ Copying /proc/sys/vm/{compact_memory,drop_caches} must ignore SELinux ++ context, otherwise an attempt to set the context to files under some ++ directories like /tmp raises an AVC denial, and an ERROR ++ "Unable to add '...' to archive: [Errno 13] Permission denied: '...' ++ is raise. ++ ++ https://bugzilla.redhat.com/show_bug.cgi?id=1965001 ++ ++ :avocado: enable ++ :avocado: tags=stageone ++ """ ++ ++ sos_cmd = '-o system' ++ # it is crucial to run the test case with --tmp-dir=/tmp/... as that is ++ # (an example of) directory exhibiting the relabel permission deny. ++ # /var/tmp directory allows those relabels. ++ # ++ # the directory shouldn't exist at this moment, otherwise ++ # "check to prevent multiple setUp() runs" in sos_tests.py would fail ++ _tmpdir = '/tmp/rhbz1965001_avocado_test' ++ ++ def test_no_permission_denied(self): ++ self.assertSosLogNotContains("Permission denied") +-- +2.26.3 + diff --git a/SOURCES/sos-bz1967613-sssd-common.patch b/SOURCES/sos-bz1967613-sssd-common.patch new file mode 100644 index 0000000..9937972 --- /dev/null +++ b/SOURCES/sos-bz1967613-sssd-common.patch @@ -0,0 +1,36 @@ +From 630dfbee936050698d33b59abd1e243c44e50af8 Mon Sep 17 00:00:00 2001 +From: Jan Jansky +Date: Thu, 3 Jun 2021 15:04:57 +0200 +Subject: [PATCH] [sssd] sssd plugin when sssd-common + +We have reports that sssd logs are not +collected, when we investigated +we found associate wants to collect +sssd related logs also when only +sssd-common package is installed. + +We got this confirmed by sbr-idm. + +Resolves: #2571 + +Signed-off-by: Jan Jansky +--- + sos/report/plugins/sssd.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/sos/report/plugins/sssd.py b/sos/report/plugins/sssd.py +index 17933935..6f98e90c 100644 +--- a/sos/report/plugins/sssd.py ++++ b/sos/report/plugins/sssd.py +@@ -19,7 +19,7 @@ class Sssd(Plugin): + + plugin_name = "sssd" + profiles = ('services', 'security', 'identity') +- packages = ('sssd',) ++ packages = ('sssd', 'sssd-common') + + def setup(self): + self.add_copy_spec([ +-- +2.26.3 + diff --git a/SOURCES/sos-bz1973675-ocp-cluster-cleaner.patch b/SOURCES/sos-bz1973675-ocp-cluster-cleaner.patch new file mode 100644 index 0000000..205152f --- /dev/null +++ b/SOURCES/sos-bz1973675-ocp-cluster-cleaner.patch @@ -0,0 +1,2156 @@ +From 29afda6e4ff90385d34bc61315542e7cb4baaf8d Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Fri, 9 Apr 2021 11:32:14 -0400 +Subject: [PATCH] [cleaner] Do not break iteration of parse_string_for_keys on + first match + +Previously, `parse_string_for_keys()`, called by `obfuscate_string()` +for non-regex based obfuscations, would return on the first match in the +string found for each parser. + +Instead, continue iterating over all items in each parser's dataset +before returning the (now fully) obfuscated string. + +Resolves: #2480 + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/parsers/__init__.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py +index dd0451df..c77300aa 100644 +--- a/sos/cleaner/parsers/__init__.py ++++ b/sos/cleaner/parsers/__init__.py +@@ -104,7 +104,7 @@ class SoSCleanerParser(): + """ + for key, val in self.mapping.dataset.items(): + if key in string_data: +- return string_data.replace(key, val) ++ string_data = string_data.replace(key, val) + return string_data + + def get_map_contents(self): +-- +2.26.3 + +From 52e6b2ae17e128f17a84ee83b7718c2901bcd5bd Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 12 May 2021 12:39:48 -0400 +Subject: [PATCH] [collect] Add options to provide registry auth for pulling + images + +Adds options that allow a user to specify registry authentication, +either via username/password or an authfile, to allow pulling an image +that exists on a non-public registry. + +If a username/password is provided, that will be used. If not, we will +attempt to use an authfile - either provided by the user or by a cluster +profile. + +Also adds an option to forcibly pull a new(er) version of the specified +image, to alleviate conditions where a too-old version of the image +already exists on the host. + +Closes: #2534 + +Signed-off-by: Jake Hunsaker +--- + man/en/sos-collect.1 | 30 +++++++++++++++++++++++ + sos/collector/__init__.py | 17 +++++++++++++ + sos/collector/sosnode.py | 40 +++++++++++++++++++++++++++---- + sos/policies/distros/__init__.py | 16 ++++++++++++- + sos/policies/distros/redhat.py | 25 ++++++++++++------- + sos/policies/runtimes/__init__.py | 25 +++++++++++++++++++ + 6 files changed, 140 insertions(+), 13 deletions(-) + +diff --git a/man/en/sos-collect.1 b/man/en/sos-collect.1 +index 286bfe71..cdbc3257 100644 +--- a/man/en/sos-collect.1 ++++ b/man/en/sos-collect.1 +@@ -26,6 +26,11 @@ sos collect \- Collect sosreports from multiple (cluster) nodes + [\-\-no\-pkg\-check] + [\-\-no\-local] + [\-\-master MASTER] ++ [\-\-image IMAGE] ++ [\-\-force-pull-image] ++ [\-\-registry-user USER] ++ [\-\-registry-password PASSWORD] ++ [\-\-registry-authfile FILE] + [\-o ONLY_PLUGINS] + [\-p SSH_PORT] + [\-\-password] +@@ -245,6 +250,31 @@ Specify a master node for the cluster. + If provided, then sos collect will check the master node, not localhost, for determining + the type of cluster in use. + .TP ++\fB\-\-image IMAGE\fR ++Specify an image to use for the temporary container created for collections on ++containerized host, if you do not want to use the default image specifed by the ++host's policy. Note that this should include the registry. ++.TP ++\fB\-\-force-pull-image\fR ++Use this option to force the container runtime to pull the specified image (even ++if it is the policy default image) even if the image already exists on the host. ++This may be useful to update an older container image on containerized hosts. ++.TP ++\fB\-\-registry-user USER\fR ++Specify the username to authenticate to the registry with in order to pull the container ++image ++.TP ++\fB\-\-registry-password PASSWORD\fR ++Specify the password to authenticate to the registry with in order to pull the container ++image. If no password is required, leave this blank. ++.TP ++\fB\-\-registry-authfile FILE\fR ++Specify the filename to use for providing authentication credentials to the registry ++to pull the container image. ++ ++Note that this file must exist on the node(s) performing the pull operations, not the ++node from which \fBsos collect\fR was run. ++.TP + \fB\-o\fR ONLY_PLUGINS, \fB\-\-only\-plugins\fR ONLY_PLUGINS + Sosreport option. Run ONLY the plugins listed. + +diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py +index 1c742cf5..0624caad 100644 +--- a/sos/collector/__init__.py ++++ b/sos/collector/__init__.py +@@ -63,6 +63,7 @@ class SoSCollector(SoSComponent): + 'encrypt_pass': '', + 'group': None, + 'image': '', ++ 'force_pull_image': False, + 'jobs': 4, + 'keywords': [], + 'keyword_file': None, +@@ -84,6 +85,9 @@ class SoSCollector(SoSComponent): + 'plugin_timeout': None, + 'cmd_timeout': None, + 'preset': '', ++ 'registry_user': None, ++ 'registry_password': None, ++ 'registry_authfile': None, + 'save_group': '', + 'since': '', + 'skip_commands': [], +@@ -319,6 +323,19 @@ class SoSCollector(SoSComponent): + collect_grp.add_argument('--image', + help=('Specify the container image to use for' + ' containerized hosts.')) ++ collect_grp.add_argument('--force-pull-image', '--pull', default=False, ++ action='store_true', ++ help='Force pull the container image even if ' ++ 'it already exists on the host') ++ collect_grp.add_argument('--registry-user', default=None, ++ help='Username to authenticate to the ' ++ 'registry with for pulling an image') ++ collect_grp.add_argument('--registry-password', default=None, ++ help='Password to authenticate to the ' ++ 'registry with for pulling an image') ++ collect_grp.add_argument('--registry-authfile', default=None, ++ help='Use this authfile to provide registry ' ++ 'authentication when pulling an image') + collect_grp.add_argument('-i', '--ssh-key', help='Specify an ssh key') + collect_grp.add_argument('-j', '--jobs', default=4, type=int, + help='Number of concurrent nodes to collect') +diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py +index 48693342..d1c11824 100644 +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -134,9 +134,27 @@ class SosNode(): + """If the host is containerized, create the container we'll be using + """ + if self.host.containerized: +- res = self.run_command(self.host.create_sos_container(), +- need_root=True) +- if res['status'] in [0, 125]: # 125 means container exists ++ cmd = self.host.create_sos_container( ++ image=self.opts.image, ++ auth=self.get_container_auth(), ++ force_pull=self.opts.force_pull_image ++ ) ++ res = self.run_command(cmd, need_root=True) ++ if res['status'] in [0, 125]: ++ if res['status'] == 125: ++ if 'unable to retrieve auth token' in res['stdout']: ++ self.log_error( ++ "Could not pull image. Provide either a username " ++ "and password or authfile" ++ ) ++ raise Exception ++ elif 'unknown: Not found' in res['stdout']: ++ self.log_error('Specified image not found on registry') ++ raise Exception ++ # 'name exists' with code 125 means the container was ++ # created successfully, so ignore it. ++ # initial creations leads to an exited container, restarting it ++ # here will keep it alive for us to exec through + ret = self.run_command(self.host.restart_sos_container(), + need_root=True) + if ret['status'] == 0: +@@ -152,6 +170,20 @@ class SosNode(): + % res['stdout']) + raise Exception + ++ def get_container_auth(self): ++ """Determine what the auth string should be to pull the image used to ++ deploy our temporary container ++ """ ++ if self.opts.registry_user: ++ return self.host.runtimes['default'].fmt_registry_credentials( ++ self.opts.registry_user, ++ self.opts.registry_password ++ ) ++ else: ++ return self.host.runtimes['default'].fmt_registry_authfile( ++ self.opts.registry_authfile or self.host.container_authfile ++ ) ++ + def file_exists(self, fname): + """Checks for the presence of fname on the remote node""" + if not self.local: +@@ -343,7 +375,7 @@ class SosNode(): + % self.commons['policy'].distro) + return self.commons['policy'] + host = load(cache={}, sysroot=self.opts.sysroot, init=InitSystem(), +- probe_runtime=False, remote_exec=self.ssh_cmd, ++ probe_runtime=True, remote_exec=self.ssh_cmd, + remote_check=self.read_file('/etc/os-release')) + if host: + self.log_info("loaded policy %s for host" % host.distro) +diff --git a/sos/policies/distros/__init__.py b/sos/policies/distros/__init__.py +index 9fe31513..f5b9fd5b 100644 +--- a/sos/policies/distros/__init__.py ++++ b/sos/policies/distros/__init__.py +@@ -62,6 +62,7 @@ class LinuxPolicy(Policy): + sos_bin_path = '/usr/bin' + sos_container_name = 'sos-collector-tmp' + container_version_command = None ++ container_authfile = None + + def __init__(self, sysroot=None, init=None, probe_runtime=True): + super(LinuxPolicy, self).__init__(sysroot=sysroot, +@@ -626,13 +627,26 @@ class LinuxPolicy(Policy): + """ + return '' + +- def create_sos_container(self): ++ def create_sos_container(self, image=None, auth=None, force_pull=False): + """Returns the command that will create the container that will be + used for running commands inside a container on hosts that require it. + + This will use the container runtime defined for the host type to + launch a container. From there, we use the defined runtime to exec into + the container's namespace. ++ ++ :param image: The name of the image if not using the policy default ++ :type image: ``str`` or ``None`` ++ ++ :param auth: The auth string required by the runtime to pull an ++ image from the registry ++ :type auth: ``str`` or ``None`` ++ ++ :param force_pull: Should the runtime forcibly pull the image ++ :type force_pull: ``bool`` ++ ++ :returns: The command to execute to launch the temp container ++ :rtype: ``str`` + """ + return '' + +diff --git a/sos/policies/distros/redhat.py b/sos/policies/distros/redhat.py +index 241d3f13..20afbcc4 100644 +--- a/sos/policies/distros/redhat.py ++++ b/sos/policies/distros/redhat.py +@@ -452,15 +452,19 @@ support representative. + + return self.find_preset(ATOMIC) + +- def create_sos_container(self): ++ def create_sos_container(self, image=None, auth=None, force_pull=False): + _cmd = ("{runtime} run -di --name {name} --privileged --ipc=host" + " --net=host --pid=host -e HOST=/host -e NAME={name} -e " +- "IMAGE={image} -v /run:/run -v /var/log:/var/log -v " ++ "IMAGE={image} {pull} -v /run:/run -v /var/log:/var/log -v " + "/etc/machine-id:/etc/machine-id -v " +- "/etc/localtime:/etc/localtime -v /:/host {image}") ++ "/etc/localtime:/etc/localtime -v /:/host {auth} {image}") ++ _image = image or self.container_image ++ _pull = '--pull=always' if force_pull else '' + return _cmd.format(runtime=self.container_runtime, + name=self.sos_container_name, +- image=self.container_image) ++ image=_image, ++ pull=_pull, ++ auth=auth or '') + + def set_cleanup_cmd(self): + return 'docker rm --force sos-collector-tmp' +@@ -482,6 +486,7 @@ support representative. + container_image = 'registry.redhat.io/rhel8/support-tools' + sos_path_strip = '/host' + container_version_command = 'rpm -q sos' ++ container_authfile = '/var/lib/kubelet/config.json' + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): +@@ -511,15 +516,19 @@ support representative. + # RH OCP environments. + return self.find_preset(RHOCP) + +- def create_sos_container(self): ++ def create_sos_container(self, image=None, auth=None, force_pull=False): + _cmd = ("{runtime} run -di --name {name} --privileged --ipc=host" + " --net=host --pid=host -e HOST=/host -e NAME={name} -e " +- "IMAGE={image} -v /run:/run -v /var/log:/var/log -v " ++ "IMAGE={image} {pull} -v /run:/run -v /var/log:/var/log -v " + "/etc/machine-id:/etc/machine-id -v " +- "/etc/localtime:/etc/localtime -v /:/host {image}") ++ "/etc/localtime:/etc/localtime -v /:/host {auth} {image}") ++ _image = image or self.container_image ++ _pull = '--pull=always' if force_pull else '' + return _cmd.format(runtime=self.container_runtime, + name=self.sos_container_name, +- image=self.container_image) ++ image=_image, ++ pull=_pull, ++ auth=auth or '') + + def set_cleanup_cmd(self): + return 'podman rm --force %s' % self.sos_container_name +diff --git a/sos/policies/runtimes/__init__.py b/sos/policies/runtimes/__init__.py +index 1a61b644..f28d6a1d 100644 +--- a/sos/policies/runtimes/__init__.py ++++ b/sos/policies/runtimes/__init__.py +@@ -157,6 +157,31 @@ class ContainerRuntime(): + quoted_cmd = cmd + return "%s %s %s" % (self.run_cmd, container, quoted_cmd) + ++ def fmt_registry_credentials(self, username, password): ++ """Format a string to pass to the 'run' command of the runtime to ++ enable authorization for pulling the image during `sos collect`, if ++ needed using username and optional password creds ++ ++ :param username: The name of the registry user ++ :type username: ``str`` ++ ++ :param password: The password of the registry user ++ :type password: ``str`` or ``None`` ++ ++ :returns: The string to use to enable a run command to pull the image ++ :rtype: ``str`` ++ """ ++ return "--creds=%s%s" % (username, ':' + password if password else '') ++ ++ def fmt_registry_authfile(self, authfile): ++ """Format a string to pass to the 'run' command of the runtime to ++ enable authorization for pulling the image during `sos collect`, if ++ needed using an authfile. ++ """ ++ if authfile: ++ return "--authfile %s" % authfile ++ return '' ++ + def get_logs_command(self, container): + """Get the command string used to dump container logs from the + runtime +-- +2.26.3 + +From 3cbbd7df6f0700609eeef3210d7388298b9e0c21 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 12 May 2021 13:26:45 -0400 +Subject: [PATCH] [sosnode] Allow clusters to set options only for master nodes + +Adds a method the `Cluster` that allows a profile to set sos options +specifically for master nodes. + +Signed-off-by: Jake Hunsaker +--- + sos/collector/clusters/__init__.py | 21 +++++++++++++++++++++ + sos/collector/sosnode.py | 6 ++++++ + 2 files changed, 27 insertions(+) + +diff --git a/sos/collector/clusters/__init__.py b/sos/collector/clusters/__init__.py +index 5c002bae..bfa3aad3 100644 +--- a/sos/collector/clusters/__init__.py ++++ b/sos/collector/clusters/__init__.py +@@ -137,6 +137,27 @@ class Cluster(): + """ + self.cluster_ssh_key = key + ++ def set_master_options(self, node): ++ """If there is a need to set specific options in the sos command being ++ run on the cluster's master nodes, override this method in the cluster ++ profile and do that here. ++ ++ :param node: The master node ++ :type node: ``SoSNode`` ++ """ ++ pass ++ ++ def check_node_is_master(self, node): ++ """In the event there are multiple masters, or if the collect command ++ is being run from a system that is technically capable of enumerating ++ nodes but the cluster profiles needs to specify master-specific options ++ for other nodes, override this method in the cluster profile ++ ++ :param node: The node for the cluster to check ++ :type node: ``SoSNode`` ++ """ ++ return node.address == self.master.address ++ + def exec_master_cmd(self, cmd, need_root=False): + """Used to retrieve command output from a (master) node in a cluster + +diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py +index d1c11824..62666635 100644 +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -647,6 +647,10 @@ class SosNode(): + self.cluster.sos_plugin_options[opt]) + self.opts.plugin_options.append(option) + ++ # set master-only options ++ if self.cluster.check_node_is_master(self): ++ self.cluster.set_master_options(self) ++ + def finalize_sos_cmd(self): + """Use host facts and compare to the cluster type to modify the sos + command if needed""" +@@ -707,6 +711,8 @@ class SosNode(): + os.path.join(self.host.sos_bin_path, self.sos_bin) + ) + ++ self.update_cmd_from_cluster() ++ + if self.opts.only_plugins: + plugs = [o for o in self.opts.only_plugins + if self._plugin_exists(o)] +-- +2.26.3 + +From cae9dd79a59107aa92db5f90aed356e093985bd9 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 12 May 2021 16:06:29 -0400 +Subject: [PATCH] [sosnode] Don't fail on sos-less bastion nodes used for node + lists + +If the master node is determined to not have sos installed, that is not +necessarily a fatal error for scenarios where the 'master' node is only +being used to enumerate node lists and is not actually part of the +cluster. This can happen when a user is using a bastion node to +enumerate and connect to the cluster environment, or if the local host +is being used to enumerate nodes via cluster client tooling. + +Signed-off-by: Jake Hunsaker +--- + sos/collector/sosnode.py | 17 ++++++++++++----- + 1 file changed, 12 insertions(+), 5 deletions(-) + +diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py +index 62666635..7e56483d 100644 +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -287,13 +287,20 @@ class SosNode(): + # use the containerized policy's command + pkgs = self.run_command(self.host.container_version_command, + use_container=True, need_root=True) +- ver = pkgs['stdout'].strip().split('-')[1] +- if ver: +- self.sos_info['version'] = ver +- if 'version' in self.sos_info: ++ if pkgs['status'] == 0: ++ ver = pkgs['stdout'].strip().split('-')[1] ++ if ver: ++ self.sos_info['version'] = ver ++ else: ++ self.sos_info['version'] = None ++ if self.sos_info['version']: + self.log_info('sos version is %s' % self.sos_info['version']) + else: +- self.log_error('sos is not installed on this node') ++ if not self.address == self.opts.master: ++ # in the case where the 'master' enumerates nodes but is not ++ # intended for collection (bastions), don't worry about sos not ++ # being present ++ self.log_error('sos is not installed on this node') + self.connected = False + return False + cmd = 'sosreport -l' +-- +2.26.3 + +From cc5abe563d855dea9ac25f56de2e493228b48bf7 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 12 May 2021 18:26:09 -0400 +Subject: [PATCH] [sosnode] Mark sos commands as explicitly needing root for + containers + +Fixes an issue where the sos inspection commands were not properly +marked as needing to be run as root (either directly or via sudo) for +containerized hosts, which would lead to incorrect sos command +formatting. + +Mark those commands, and the final container removal command, as +explicitly needing root permissions. + +Signed-off-by: Jake Hunsaker +--- + sos/collector/sosnode.py | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py +index 7e56483d..1fc03076 100644 +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -304,7 +304,7 @@ class SosNode(): + self.connected = False + return False + cmd = 'sosreport -l' +- sosinfo = self.run_command(cmd, use_container=True) ++ sosinfo = self.run_command(cmd, use_container=True, need_root=True) + if sosinfo['status'] == 0: + self._load_sos_plugins(sosinfo['stdout']) + if self.check_sos_version('3.6'): +@@ -312,7 +312,7 @@ class SosNode(): + + def _load_sos_presets(self): + cmd = 'sosreport --list-presets' +- res = self.run_command(cmd, use_container=True) ++ res = self.run_command(cmd, use_container=True, need_root=True) + if res['status'] == 0: + for line in res['stdout'].splitlines(): + if line.strip().startswith('name:'): +@@ -996,7 +996,7 @@ class SosNode(): + self.remove_file(self.sos_path + '.md5') + cleanup = self.host.set_cleanup_cmd() + if cleanup: +- self.run_command(cleanup) ++ self.run_command(cleanup, need_root=True) + + def collect_extra_cmd(self, filenames): + """Collect the file created by a cluster outside of sos""" +-- +2.26.3 + +From 55e77ad4c7e90ba14b10c5fdf18b65aa5d6b9cf8 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 12 May 2021 18:55:31 -0400 +Subject: [PATCH] [ocp] Add cluster profile for OCP4 + +Removes the previous OCP cluster profile and replaces it with an updated +one for OCP4 which is entirely separated from the kubernetes profile. + +Resolves: #2544 + +Signed-off-by: Jake Hunsaker +--- + sos/collector/clusters/kubernetes.py | 8 -- + sos/collector/clusters/ocp.py | 109 +++++++++++++++++++++++++++ + 2 files changed, 109 insertions(+), 8 deletions(-) + create mode 100644 sos/collector/clusters/ocp.py + +diff --git a/sos/collector/clusters/kubernetes.py b/sos/collector/clusters/kubernetes.py +index 6a867e31..08fd9554 100644 +--- a/sos/collector/clusters/kubernetes.py ++++ b/sos/collector/clusters/kubernetes.py +@@ -44,11 +44,3 @@ class kubernetes(Cluster): + return nodes + else: + raise Exception('Node enumeration did not return usable output') +- +- +-class openshift(kubernetes): +- +- cluster_name = 'OpenShift Container Platform' +- packages = ('atomic-openshift',) +- sos_preset = 'ocp' +- cmd = 'oc' +diff --git a/sos/collector/clusters/ocp.py b/sos/collector/clusters/ocp.py +new file mode 100644 +index 00000000..283fcfd1 +--- /dev/null ++++ b/sos/collector/clusters/ocp.py +@@ -0,0 +1,109 @@ ++# Copyright Red Hat 2021, Jake Hunsaker ++ ++# This file is part of the sos project: https://github.com/sosreport/sos ++# ++# This copyrighted material is made available to anyone wishing to use, ++# modify, copy, or redistribute it subject to the terms and conditions of ++# version 2 of the GNU General Public License. ++# ++# See the LICENSE file in the source distribution for further information. ++ ++from pipes import quote ++from sos.collector.clusters import Cluster ++ ++ ++class ocp(Cluster): ++ """OpenShift Container Platform v4""" ++ ++ cluster_name = 'OpenShift Container Platform v4' ++ packages = ('openshift-hyperkube', 'openshift-clients') ++ ++ option_list = [ ++ ('label', '', 'Colon delimited list of labels to select nodes with'), ++ ('role', '', 'Colon delimited list of roles to select nodes with'), ++ ('kubeconfig', '', 'Path to the kubeconfig file') ++ ] ++ ++ def fmt_oc_cmd(self, cmd): ++ """Format the oc command to optionall include the kubeconfig file if ++ one is specified ++ """ ++ if self.get_option('kubeconfig'): ++ return "oc --config %s %s" % (self.get_option('kubeconfig'), cmd) ++ return "oc %s" % cmd ++ ++ def check_enabled(self): ++ if super(ocp, self).check_enabled(): ++ return True ++ _who = self.fmt_oc_cmd('whoami') ++ return self.exec_master_cmd(_who)['status'] == 0 ++ ++ def _build_dict(self, nodelist): ++ """From the output of get_nodes(), construct an easier-to-reference ++ dict of nodes that will be used in determining labels, master status, ++ etc... ++ ++ :param nodelist: The split output of `oc get nodes` ++ :type nodelist: ``list`` ++ ++ :returns: A dict of nodes with `get nodes` columns as keys ++ :rtype: ``dict`` ++ """ ++ nodes = {} ++ if 'NAME' in nodelist[0]: ++ # get the index of the fields ++ statline = nodelist.pop(0).split() ++ idx = {} ++ for state in ['status', 'roles', 'version', 'os-image']: ++ try: ++ idx[state] = statline.index(state.upper()) ++ except Exception: ++ pass ++ for node in nodelist: ++ _node = node.split() ++ nodes[_node[0]] = {} ++ for column in idx: ++ nodes[_node[0]][column] = _node[idx[column]] ++ return nodes ++ ++ def get_nodes(self): ++ nodes = [] ++ self.node_dict = {} ++ cmd = 'get nodes -o wide' ++ if self.get_option('label'): ++ labels = ','.join(self.get_option('label').split(':')) ++ cmd += " -l %s" % quote(labels) ++ res = self.exec_master_cmd(self.fmt_oc_cmd(cmd)) ++ if res['status'] == 0: ++ roles = [r for r in self.get_option('role').split(':')] ++ self.node_dict = self._build_dict(res['stdout'].splitlines()) ++ for node in self.node_dict: ++ if roles: ++ for role in roles: ++ if role in node: ++ nodes.append(node) ++ else: ++ nodes.append(node) ++ else: ++ msg = "'oc' command failed" ++ if 'Missing or incomplete' in res['stdout']: ++ msg = ("'oc' failed due to missing kubeconfig on master node." ++ " Specify one via '-c ocp.kubeconfig='") ++ raise Exception(msg) ++ return nodes ++ ++ def set_node_label(self, node): ++ if node.address not in self.node_dict: ++ return '' ++ for label in ['master', 'worker']: ++ if label in self.node_dict[node.address]['roles']: ++ return label ++ return '' ++ ++ def check_node_is_master(self, sosnode): ++ if sosnode.address not in self.node_dict: ++ return False ++ return 'master' in self.node_dict[sosnode.address]['roles'] ++ ++ def set_master_options(self, node): ++ node.opts.enable_plugins.append('openshift') +-- +2.26.3 + +From a3c1caad21160545eda87ea1fde93e972a6fbf88 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 26 May 2021 11:55:24 -0400 +Subject: [PATCH] [cleaner] Don't strip empty lines from substituted files + +Fixes an issue where empty lines would be stripped from files that have +other obfuscations in them. Those empty lines may be important for file +structure and/or readability, so we should instead simply not pass empty +lines to the parsers rather than skipping them wholesale in the flow of +writing obfuscations to a temp file before replacing the source file +with a potentially changed temp file. + +Resolves: #2562 + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/__init__.py | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py +index bdd24f95..55465b85 100644 +--- a/sos/cleaner/__init__.py ++++ b/sos/cleaner/__init__.py +@@ -603,8 +603,6 @@ third party. + tfile = tempfile.NamedTemporaryFile(mode='w', dir=self.tmpdir) + with open(filename, 'r') as fname: + for line in fname: +- if not line.strip(): +- continue + try: + line, count = self.obfuscate_line(line) + subs += count +@@ -642,7 +640,11 @@ third party. + + Returns the fully obfuscated line and the number of substitutions made + """ ++ # don't iterate over blank lines, but still write them to the tempfile ++ # to maintain the same structure when we write a scrubbed file back + count = 0 ++ if not line.strip(): ++ return line, count + for parser in self.parsers: + try: + line, _count = parser.parse_line(line) +-- +2.26.3 + +From 892bbd8114703f5a4d23aa77ba5829b7ba59446f Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 5 May 2021 17:02:04 -0400 +Subject: [PATCH] [cleaner] Remove binary files by default + +Binary files generally speaking cannot be obfuscated, and as such we +should remove them from archives being obfuscated by default so that +sensitive data is not mistakenly included in an obfuscated archive. + +This commits adds a new `--keep-binary-files` option that if used will +keep any encountered binary files in the final archive. The default +option of `false` will ensure that encountered binary files are removed. + +The number of removed binary files per archive is reported when +obfuscation is completed for that archive. + +Closes: #2478 +Resolves: #2524 + +Signed-off-by: Jake Hunsaker +--- + man/en/sos-clean.1 | 12 ++++ + sos/cleaner/__init__.py | 21 +++++- + sos/cleaner/obfuscation_archive.py | 67 ++++++++++++++++++-- + sos/collector/__init__.py | 5 ++ + sos/report/__init__.py | 6 ++ + 8 files changed, 167 insertions(+), 7 deletions(-) + +diff --git a/man/en/sos-clean.1 b/man/en/sos-clean.1 +index 4856b43b..b77bc63c 100644 +--- a/man/en/sos-clean.1 ++++ b/man/en/sos-clean.1 +@@ -9,6 +9,7 @@ sos clean - Obfuscate sensitive data from one or more sosreports + [\-\-map-file] + [\-\-jobs] + [\-\-no-update] ++ [\-\-keep-binary-files] + + .SH DESCRIPTION + \fBsos clean\fR or \fBsos mask\fR is an sos subcommand used to obfuscate sensitive information from +@@ -77,6 +78,17 @@ Default: 4 + .TP + .B \-\-no-update + Do not write the mapping file contents to /etc/sos/cleaner/default_mapping ++.TP ++.B \-\-keep-binary-files ++Keep unprocessable binary files in the archive, rather than removing them. ++ ++Note that binary files cannot be obfuscated, and thus keeping them in the archive ++may result in otherwise sensitive information being included in the final archive. ++Users should review any archive that keeps binary files in place before sending to ++a third party. ++ ++Default: False (remove encountered binary files) ++ + .SH SEE ALSO + .BR sos (1) + .BR sos-report (1) +diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py +index 55465b85..f88ff8a0 100644 +--- a/sos/cleaner/__init__.py ++++ b/sos/cleaner/__init__.py +@@ -47,6 +47,7 @@ class SoSCleaner(SoSComponent): + 'keyword_file': None, + 'map_file': '/etc/sos/cleaner/default_mapping', + 'no_update': False, ++ 'keep_binary_files': False, + 'target': '', + 'usernames': [] + } +@@ -183,6 +184,11 @@ third party. + action='store_true', + help='Do not update the --map file with new ' + 'mappings from this run') ++ clean_grp.add_argument('--keep-binary-files', default=False, ++ action='store_true', ++ dest='keep_binary_files', ++ help='Keep unprocessable binary files in the ' ++ 'archive instead of removing them') + clean_grp.add_argument('--usernames', dest='usernames', default=[], + action='extend', + help='List of usernames to obfuscate') +@@ -467,6 +473,11 @@ third party. + "%s concurrently\n" + % (len(self.report_paths), self.opts.jobs)) + self.ui_log.info(msg) ++ if self.opts.keep_binary_files: ++ self.ui_log.warning( ++ "WARNING: binary files that potentially contain sensitive " ++ "information will NOT be removed from the final archive\n" ++ ) + pool = ThreadPoolExecutor(self.opts.jobs) + pool.map(self.obfuscate_report, self.report_paths, chunksize=1) + pool.shutdown(wait=True) +@@ -539,6 +550,10 @@ third party. + short_name = fname.split(archive.archive_name + '/')[1] + if archive.should_skip_file(short_name): + continue ++ if (not self.opts.keep_binary_files and ++ archive.should_remove_file(short_name)): ++ archive.remove_file(short_name) ++ continue + try: + count = self.obfuscate_file(fname, short_name, + archive.archive_name) +@@ -574,7 +589,11 @@ third party. + arc_md.add_field('files_obfuscated', len(archive.file_sub_list)) + arc_md.add_field('total_substitutions', archive.total_sub_count) + self.completed_reports.append(archive) +- archive.report_msg("Obfuscation completed") ++ rmsg = '' ++ if archive.removed_file_count: ++ rmsg = " [removed %s unprocessable files]" ++ rmsg = rmsg % archive.removed_file_count ++ archive.report_msg("Obfuscation completed%s" % rmsg) + + except Exception as err: + self.ui_log.info("Exception while processing %s: %s" +diff --git a/sos/cleaner/obfuscation_archive.py b/sos/cleaner/obfuscation_archive.py +index c64ab13b..76841b51 100644 +--- a/sos/cleaner/obfuscation_archive.py ++++ b/sos/cleaner/obfuscation_archive.py +@@ -28,6 +28,7 @@ class SoSObfuscationArchive(): + + file_sub_list = [] + total_sub_count = 0 ++ removed_file_count = 0 + + def __init__(self, archive_path, tmpdir): + self.archive_path = archive_path +@@ -62,11 +63,7 @@ class SoSObfuscationArchive(): + 'sys/firmware', + 'sys/fs', + 'sys/kernel/debug', +- 'sys/module', +- r'.*\.tar$', # TODO: support archive unpacking +- # Be explicit with these tar matches to avoid matching commands +- r'.*\.tar\.xz', +- '.*.gz' ++ 'sys/module' + ] + + @property +@@ -76,6 +73,17 @@ class SoSObfuscationArchive(): + except Exception: + return False + ++ def remove_file(self, fname): ++ """Remove a file from the archive. This is used when cleaner encounters ++ a binary file, which we cannot reliably obfuscate. ++ """ ++ full_fname = self.get_file_path(fname) ++ # don't call a blank remove() here ++ if full_fname: ++ self.log_info("Removing binary file '%s' from archive" % fname) ++ os.remove(full_fname) ++ self.removed_file_count += 1 ++ + def extract(self): + if self.is_tarfile: + self.report_msg("Extracting...") +@@ -227,3 +235,52 @@ class SoSObfuscationArchive(): + if filename.startswith(_skip) or re.match(_skip, filename): + return True + return False ++ ++ def should_remove_file(self, fname): ++ """Determine if the file should be removed or not, due to an inability ++ to reliably obfuscate that file based on the filename. ++ ++ :param fname: Filename relative to the extracted archive root ++ :type fname: ``str`` ++ ++ :returns: ``True`` if the file cannot be reliably obfuscated ++ :rtype: ``bool`` ++ """ ++ obvious_removes = [ ++ r'.*\.gz', # TODO: support flat gz/xz extraction ++ r'.*\.xz', ++ r'.*\.bzip2', ++ r'.*\.tar\..*', # TODO: support archive unpacking ++ r'.*\.txz$', ++ r'.*\.tgz$', ++ r'.*\.bin', ++ r'.*\.journal', ++ r'.*\~$' ++ ] ++ ++ # if the filename matches, it is obvious we can remove them without ++ # doing the read test ++ for _arc_reg in obvious_removes: ++ if re.match(_arc_reg, fname): ++ return True ++ ++ return self.file_is_binary(fname) ++ ++ def file_is_binary(self, fname): ++ """Determine if the file is a binary file or not. ++ ++ ++ :param fname: Filename relative to the extracted archive root ++ :type fname: ``str`` ++ ++ :returns: ``True`` if file is binary, else ``False`` ++ :rtype: ``bool`` ++ """ ++ with open(self.get_file_path(fname), 'tr') as tfile: ++ try: ++ # when opened as above (tr), reading binary content will raise ++ # an exception ++ tfile.read(1) ++ return False ++ except UnicodeDecodeError: ++ return True +diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py +index 9884836c..469db60d 100644 +--- a/sos/collector/__init__.py ++++ b/sos/collector/__init__.py +@@ -67,6 +67,7 @@ class SoSCollector(SoSComponent): + 'jobs': 4, + 'keywords': [], + 'keyword_file': None, ++ 'keep_binary_files': False, + 'label': '', + 'list_options': False, + 'log_size': 0, +@@ -410,6 +411,10 @@ class SoSCollector(SoSComponent): + dest='clean', + default=False, action='store_true', + help='Obfuscate sensistive information') ++ cleaner_grp.add_argument('--keep-binary-files', default=False, ++ action='store_true', dest='keep_binary_files', ++ help='Keep unprocessable binary files in the ' ++ 'archive instead of removing them') + cleaner_grp.add_argument('--domains', dest='domains', default=[], + action='extend', + help='Additional domain names to obfuscate') +diff --git a/sos/report/__init__.py b/sos/report/__init__.py +index d4345409..2cedc76e 100644 +--- a/sos/report/__init__.py ++++ b/sos/report/__init__.py +@@ -82,6 +82,7 @@ class SoSReport(SoSComponent): + 'case_id': '', + 'chroot': 'auto', + 'clean': False, ++ 'keep_binary_files': False, + 'desc': '', + 'domains': [], + 'dry_run': False, +@@ -344,6 +345,11 @@ class SoSReport(SoSComponent): + default='/etc/sos/cleaner/default_mapping', + help=('Provide a previously generated mapping' + ' file for obfuscation')) ++ cleaner_grp.add_argument('--keep-binary-files', default=False, ++ action='store_true', ++ dest='keep_binary_files', ++ help='Keep unprocessable binary files in the ' ++ 'archive instead of removing them') + cleaner_grp.add_argument('--usernames', dest='usernames', default=[], + action='extend', + help='List of usernames to obfuscate') + +From aed0102a1d6ef9a030c9e5349f092b51b9d1f22d Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Fri, 11 Jun 2021 23:20:59 -0400 +Subject: [PATCH 01/10] [SoSNode] Allow individually setting node options + +Like we now do for primary nodes, add the ability to individually set +node options via a new `set_node_options()` method for when blanket +setting options across all nodes via the options class attrs is not +sufficient. + +Signed-off-by: Jake Hunsaker +--- + sos/collector/clusters/__init__.py | 10 ++++++++++ + sos/collector/sosnode.py | 6 ++++-- + 2 files changed, 14 insertions(+), 2 deletions(-) + +diff --git a/sos/collector/clusters/__init__.py b/sos/collector/clusters/__init__.py +index 90e62d79..c4da1ab8 100644 +--- a/sos/collector/clusters/__init__.py ++++ b/sos/collector/clusters/__init__.py +@@ -137,6 +137,16 @@ class Cluster(): + """ + self.cluster_ssh_key = key + ++ def set_node_options(self, node): ++ """If there is a need to set specific options on ONLY the non-primary ++ nodes in a collection, override this method in the cluster profile ++ and do that here. ++ ++ :param node: The non-primary node ++ :type node: ``SoSNode`` ++ """ ++ pass ++ + def set_master_options(self, node): + """If there is a need to set specific options in the sos command being + run on the cluster's master nodes, override this method in the cluster +diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py +index 1fc03076..7e784aa1 100644 +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -657,6 +657,8 @@ class SosNode(): + # set master-only options + if self.cluster.check_node_is_master(self): + self.cluster.set_master_options(self) ++ else: ++ self.cluster.set_node_options(self) + + def finalize_sos_cmd(self): + """Use host facts and compare to the cluster type to modify the sos +@@ -713,13 +715,13 @@ class SosNode(): + sos_opts.append('--cmd-timeout=%s' + % quote(str(self.opts.cmd_timeout))) + ++ self.update_cmd_from_cluster() ++ + sos_cmd = sos_cmd.replace( + 'sosreport', + os.path.join(self.host.sos_bin_path, self.sos_bin) + ) + +- self.update_cmd_from_cluster() +- + if self.opts.only_plugins: + plugs = [o for o in self.opts.only_plugins + if self._plugin_exists(o)] +-- +2.26.3 + + +From 96f166699d12704cc7cf73cb8b13278675f68730 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Sat, 12 Jun 2021 00:02:36 -0400 +Subject: [PATCH 02/10] [sosnode] Support passing env vars to `run_command()` + +Updates `run_command()` to support passing new environment variables to +the command being run, for that command alone. This parameter takes a +dict, and if set we will first copy the existing set of env vars on the +node and then update that set of variables using the passed dict. + +Additionally, `execute_sos_command()` will now try to pass a new +`sos_env_vars` dict (default empty) so that clusters may set environment +variables specifically for the sos command being run, without having to +modify the actual sos command being executed. + +Signed-off-by: Jake Hunsaker +--- + sos/collector/sosnode.py | 27 ++++++++++++++++++++++++--- + 1 file changed, 24 insertions(+), 3 deletions(-) + +diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py +index 7e784aa1..40472a4e 100644 +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -45,6 +45,8 @@ class SosNode(): + self.host = None + self.cluster = None + self.hostname = None ++ self.sos_env_vars = {} ++ self._env_vars = {} + self._password = password or self.opts.password + if not self.opts.nopasswd_sudo and not self.opts.sudo_pw: + self.opts.sudo_pw = self._password +@@ -109,6 +111,21 @@ class SosNode(): + def _fmt_msg(self, msg): + return '{:<{}} : {}'.format(self._hostname, self.hostlen + 1, msg) + ++ @property ++ def env_vars(self): ++ if not self._env_vars: ++ if self.local: ++ self._env_vars = os.environ.copy() ++ else: ++ ret = self.run_command("env --null") ++ if ret['status'] == 0: ++ for ln in ret['output'].split('\x00'): ++ if not ln: ++ continue ++ _val = ln.split('=') ++ self._env_vars[_val[0]] = _val[1] ++ return self._env_vars ++ + def set_node_manifest(self, manifest): + """Set the manifest section that this node will write to + """ +@@ -404,7 +421,7 @@ class SosNode(): + return self.host.package_manager.pkg_by_name(pkg) is not None + + def run_command(self, cmd, timeout=180, get_pty=False, need_root=False, +- force_local=False, use_container=False): ++ force_local=False, use_container=False, env=None): + """Runs a given cmd, either via the SSH session or locally + + Arguments: +@@ -446,7 +463,10 @@ class SosNode(): + else: + if get_pty: + cmd = "/bin/bash -c %s" % quote(cmd) +- res = pexpect.spawn(cmd, encoding='utf-8') ++ if env: ++ _cmd_env = self.env_vars ++ _cmd_env.update(env) ++ res = pexpect.spawn(cmd, encoding='utf-8', env=_cmd_env) + if need_root: + if self.need_sudo: + res.sendline(self.opts.sudo_pw) +@@ -830,7 +850,8 @@ class SosNode(): + res = self.run_command(self.sos_cmd, + timeout=self.opts.timeout, + get_pty=True, need_root=True, +- use_container=True) ++ use_container=True, ++ env=self.sos_env_vars) + if res['status'] == 0: + for line in res['stdout'].splitlines(): + if fnmatch.fnmatch(line, '*sosreport-*tar*'): +-- +2.26.3 + + +From a9e1632113406a646bdd7525982b699cf790aedb Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Tue, 15 Jun 2021 12:43:27 -0400 +Subject: [PATCH 03/10] [collect|sosnode] Avoiding clobbering sos options + between nodes + +This commit overhauls the function of `finalize_sos_cmd()` in several +ways. + +First, assign the sos report plugin related options directly to private +copies of those values for each node, so that the shared cluster profile +does not clober options between nodes. + +Second, provide a default Lock mechanism for clusters that need to +perform some node-comparison logic when assigning options based on node +role. + +Finally, finalize the sos command for each node _prior_ to the call to +`SoSNode.sosreport()` so that we can be sure that clusters are able to +appropriately compare and assign sos options across nodes before some +nodes have already started and/or finished their own sos report +collections. + +Signed-off-by: Jake Hunsaker +--- + sos/collector/__init__.py | 14 +++++ + sos/collector/clusters/__init__.py | 2 + + sos/collector/sosnode.py | 89 +++++++++++++++++------------- + 3 files changed, 67 insertions(+), 38 deletions(-) + +diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py +index 469db60d..7b8cfcf7 100644 +--- a/sos/collector/__init__.py ++++ b/sos/collector/__init__.py +@@ -1186,6 +1186,10 @@ this utility or remote systems that it connects to. + "concurrently\n" + % (self.report_num, self.opts.jobs)) + ++ npool = ThreadPoolExecutor(self.opts.jobs) ++ npool.map(self._finalize_sos_cmd, self.client_list, chunksize=1) ++ npool.shutdown(wait=True) ++ + pool = ThreadPoolExecutor(self.opts.jobs) + pool.map(self._collect, self.client_list, chunksize=1) + pool.shutdown(wait=True) +@@ -1217,6 +1221,16 @@ this utility or remote systems that it connects to. + except Exception as err: + self.ui_log.error("Upload attempt failed: %s" % err) + ++ def _finalize_sos_cmd(self, client): ++ """Calls finalize_sos_cmd() on each node so that we have the final ++ command before we thread out the actual execution of sos ++ """ ++ try: ++ client.finalize_sos_cmd() ++ except Exception as err: ++ self.log_error("Could not finalize sos command for %s: %s" ++ % (client.address, err)) ++ + def _collect(self, client): + """Runs sosreport on each node""" + try: +diff --git a/sos/collector/clusters/__init__.py b/sos/collector/clusters/__init__.py +index c4da1ab8..bb728bc0 100644 +--- a/sos/collector/clusters/__init__.py ++++ b/sos/collector/clusters/__init__.py +@@ -11,6 +11,7 @@ + import logging + + from sos.options import ClusterOption ++from threading import Lock + + + class Cluster(): +@@ -66,6 +67,7 @@ class Cluster(): + if cls.__name__ != 'Cluster': + self.cluster_type.append(cls.__name__) + self.node_list = None ++ self.lock = Lock() + self.soslog = logging.getLogger('sos') + self.ui_log = logging.getLogger('sos_ui') + self.options = [] +diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py +index 40472a4e..1c25cc34 100644 +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -38,6 +38,7 @@ class SosNode(): + self.address = address.strip() + self.commons = commons + self.opts = commons['cmdlineopts'] ++ self._assign_config_opts() + self.tmpdir = commons['tmpdir'] + self.hostlen = commons['hostlen'] + self.need_sudo = commons['need_sudo'] +@@ -465,8 +466,8 @@ class SosNode(): + cmd = "/bin/bash -c %s" % quote(cmd) + if env: + _cmd_env = self.env_vars +- _cmd_env.update(env) +- res = pexpect.spawn(cmd, encoding='utf-8', env=_cmd_env) ++ env = _cmd_env.update(env) ++ res = pexpect.spawn(cmd, encoding='utf-8', env=env) + if need_root: + if self.need_sudo: + res.sendline(self.opts.sudo_pw) +@@ -484,9 +485,6 @@ class SosNode(): + + def sosreport(self): + """Run a sosreport on the node, then collect it""" +- self.sos_cmd = self.finalize_sos_cmd() +- self.log_info('Final sos command set to %s' % self.sos_cmd) +- self.manifest.add_field('final_sos_command', self.sos_cmd) + try: + path = self.execute_sos_command() + if path: +@@ -656,29 +654,42 @@ class SosNode(): + This will NOT override user supplied options. + """ + if self.cluster.sos_preset: +- if not self.opts.preset: +- self.opts.preset = self.cluster.sos_preset ++ if not self.preset: ++ self.preset = self.cluster.sos_preset + else: + self.log_info('Cluster specified preset %s but user has also ' + 'defined a preset. Using user specification.' + % self.cluster.sos_preset) + if self.cluster.sos_plugins: + for plug in self.cluster.sos_plugins: +- if plug not in self.opts.enable_plugins: +- self.opts.enable_plugins.append(plug) ++ if plug not in self.enable_plugins: ++ self.enable_plugins.append(plug) + + if self.cluster.sos_plugin_options: + for opt in self.cluster.sos_plugin_options: +- if not any(opt in o for o in self.opts.plugin_options): ++ if not any(opt in o for o in self.plugin_options): + option = '%s=%s' % (opt, + self.cluster.sos_plugin_options[opt]) +- self.opts.plugin_options.append(option) ++ self.plugin_options.append(option) + + # set master-only options + if self.cluster.check_node_is_master(self): +- self.cluster.set_master_options(self) ++ with self.cluster.lock: ++ self.cluster.set_master_options(self) + else: +- self.cluster.set_node_options(self) ++ with self.cluster.lock: ++ self.cluster.set_node_options(self) ++ ++ def _assign_config_opts(self): ++ """From the global opts configuration, assign those values locally ++ to this node so that they may be acted on individually. ++ """ ++ # assign these to new, private copies ++ self.only_plugins = list(self.opts.only_plugins) ++ self.skip_plugins = list(self.opts.skip_plugins) ++ self.enable_plugins = list(self.opts.enable_plugins) ++ self.plugin_options = list(self.opts.plugin_options) ++ self.preset = list(self.opts.preset) + + def finalize_sos_cmd(self): + """Use host facts and compare to the cluster type to modify the sos +@@ -742,59 +753,61 @@ class SosNode(): + os.path.join(self.host.sos_bin_path, self.sos_bin) + ) + +- if self.opts.only_plugins: +- plugs = [o for o in self.opts.only_plugins +- if self._plugin_exists(o)] +- if len(plugs) != len(self.opts.only_plugins): +- not_only = list(set(self.opts.only_plugins) - set(plugs)) ++ if self.only_plugins: ++ plugs = [o for o in self.only_plugins if self._plugin_exists(o)] ++ if len(plugs) != len(self.only_plugins): ++ not_only = list(set(self.only_plugins) - set(plugs)) + self.log_debug('Requested plugins %s were requested to be ' + 'enabled but do not exist' % not_only) +- only = self._fmt_sos_opt_list(self.opts.only_plugins) ++ only = self._fmt_sos_opt_list(self.only_plugins) + if only: + sos_opts.append('--only-plugins=%s' % quote(only)) +- return "%s %s" % (sos_cmd, ' '.join(sos_opts)) ++ self.sos_cmd = "%s %s" % (sos_cmd, ' '.join(sos_opts)) ++ self.log_info('Final sos command set to %s' % self.sos_cmd) ++ self.manifest.add_field('final_sos_command', self.sos_cmd) ++ return + +- if self.opts.skip_plugins: ++ if self.skip_plugins: + # only run skip-plugins for plugins that are enabled +- skip = [o for o in self.opts.skip_plugins +- if self._check_enabled(o)] +- if len(skip) != len(self.opts.skip_plugins): +- not_skip = list(set(self.opts.skip_plugins) - set(skip)) ++ skip = [o for o in self.skip_plugins if self._check_enabled(o)] ++ if len(skip) != len(self.skip_plugins): ++ not_skip = list(set(self.skip_plugins) - set(skip)) + self.log_debug('Requested to skip plugins %s, but plugins are ' + 'already not enabled' % not_skip) + skipln = self._fmt_sos_opt_list(skip) + if skipln: + sos_opts.append('--skip-plugins=%s' % quote(skipln)) + +- if self.opts.enable_plugins: ++ if self.enable_plugins: + # only run enable for plugins that are disabled +- opts = [o for o in self.opts.enable_plugins +- if o not in self.opts.skip_plugins ++ opts = [o for o in self.enable_plugins ++ if o not in self.skip_plugins + and self._check_disabled(o) and self._plugin_exists(o)] +- if len(opts) != len(self.opts.enable_plugins): +- not_on = list(set(self.opts.enable_plugins) - set(opts)) ++ if len(opts) != len(self.enable_plugins): ++ not_on = list(set(self.enable_plugins) - set(opts)) + self.log_debug('Requested to enable plugins %s, but plugins ' + 'are already enabled or do not exist' % not_on) + enable = self._fmt_sos_opt_list(opts) + if enable: + sos_opts.append('--enable-plugins=%s' % quote(enable)) + +- if self.opts.plugin_options: +- opts = [o for o in self.opts.plugin_options ++ if self.plugin_options: ++ opts = [o for o in self.plugin_options + if self._plugin_exists(o.split('.')[0]) + and self._plugin_option_exists(o.split('=')[0])] + if opts: + sos_opts.append('-k %s' % quote(','.join(o for o in opts))) + +- if self.opts.preset: +- if self._preset_exists(self.opts.preset): +- sos_opts.append('--preset=%s' % quote(self.opts.preset)) ++ if self.preset: ++ if self._preset_exists(self.preset): ++ sos_opts.append('--preset=%s' % quote(self.preset)) + else: + self.log_debug('Requested to enable preset %s but preset does ' +- 'not exist on node' % self.opts.preset) ++ 'not exist on node' % self.preset) + +- _sos_cmd = "%s %s" % (sos_cmd, ' '.join(sos_opts)) +- return _sos_cmd ++ self.sos_cmd = "%s %s" % (sos_cmd, ' '.join(sos_opts)) ++ self.log_info('Final sos command set to %s' % self.sos_cmd) ++ self.manifest.add_field('final_sos_command', self.sos_cmd) + + def determine_sos_label(self): + """Determine what, if any, label should be added to the sosreport""" +-- +2.26.3 + + +From 7e6c078e51143f7064190b316a251ddd8d431495 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Tue, 15 Jun 2021 18:38:34 -0400 +Subject: [PATCH 04/10] [cleaner] Improve handling of symlink obfuscation + +Improves handling of symlink obfuscation by only performing the +obfuscaiton on the ultimate target of any symlinks encountered. Now, +when a symlink is encountered, clean will obfuscate the link name and +re-write it in the archive, pointing to the (potentially obfuscated) +target name. + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/__init__.py | 65 +++++++++++++++++++++++++++++------------ + 1 file changed, 46 insertions(+), 19 deletions(-) + +diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py +index abfb684b..b38c8dfc 100644 +--- a/sos/cleaner/__init__.py ++++ b/sos/cleaner/__init__.py +@@ -612,28 +612,55 @@ third party. + if not filename: + # the requested file doesn't exist in the archive + return +- self.log_debug("Obfuscating %s" % short_name or filename, +- caller=arc_name) + subs = 0 +- tfile = tempfile.NamedTemporaryFile(mode='w', dir=self.tmpdir) +- with open(filename, 'r') as fname: +- for line in fname: +- try: +- line, count = self.obfuscate_line(line) +- subs += count +- tfile.write(line) +- except Exception as err: +- self.log_debug("Unable to obfuscate %s: %s" +- % (short_name, err), caller=arc_name) +- tfile.seek(0) +- if subs: +- shutil.copy(tfile.name, filename) +- tfile.close() +- _ob_filename = self.obfuscate_string(short_name) +- if _ob_filename != short_name: ++ if not os.path.islink(filename): ++ # don't run the obfuscation on the link, but on the actual file ++ # at some other point. ++ self.log_debug("Obfuscating %s" % short_name or filename, ++ caller=arc_name) ++ tfile = tempfile.NamedTemporaryFile(mode='w', dir=self.tmpdir) ++ with open(filename, 'r') as fname: ++ for line in fname: ++ try: ++ line, count = self.obfuscate_line(line) ++ subs += count ++ tfile.write(line) ++ except Exception as err: ++ self.log_debug("Unable to obfuscate %s: %s" ++ % (short_name, err), caller=arc_name) ++ tfile.seek(0) ++ if subs: ++ shutil.copy(tfile.name, filename) ++ tfile.close() ++ ++ _ob_short_name = self.obfuscate_string(short_name.split('/')[-1]) ++ _ob_filename = short_name.replace(short_name.split('/')[-1], ++ _ob_short_name) ++ _sym_changed = False ++ if os.path.islink(filename): ++ _link = os.readlink(filename) ++ _ob_link = self.obfuscate_string(_link) ++ if _ob_link != _link: ++ _sym_changed = True ++ ++ if (_ob_filename != short_name) or _sym_changed: + arc_path = filename.split(short_name)[0] + _ob_path = os.path.join(arc_path, _ob_filename) +- os.rename(filename, _ob_path) ++ # ensure that any plugin subdirs that contain obfuscated strings ++ # get created with obfuscated counterparts ++ if not os.path.islink(filename): ++ os.rename(filename, _ob_path) ++ else: ++ # generate the obfuscated name of the link target ++ _target_ob = self.obfuscate_string(os.readlink(filename)) ++ # remove the unobfuscated original symlink first, in case the ++ # symlink name hasn't changed but the target has ++ os.remove(filename) ++ # create the newly obfuscated symlink, pointing to the ++ # obfuscated target name, which may not exist just yet, but ++ # when the actual file is obfuscated, will be created ++ os.symlink(_target_ob, _ob_path) ++ + return subs + + def obfuscate_string(self, string_data): +-- +2.26.3 + + +From b5d166ac9ff79bc3740c5e66f16d60762f9a0ac0 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Tue, 15 Jun 2021 22:56:19 -0400 +Subject: [PATCH 05/10] [cleaner] Iterate over matches with most precise match + first + +When matching strings in parsers to do obfuscation, we should be using +the most precise matches found first, rather than matching in the order +a match is hit. This ensures that we correctly obfuscate an entire +string, rather than potentially only partial substring(s) that exist +within the entire match. + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/parsers/__init__.py | 10 +++++++--- + sos/cleaner/parsers/keyword_parser.py | 2 +- + sos/cleaner/parsers/username_parser.py | 2 +- + 3 files changed, 9 insertions(+), 5 deletions(-) + +diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py +index c77300aa..cfa20b95 100644 +--- a/sos/cleaner/parsers/__init__.py ++++ b/sos/cleaner/parsers/__init__.py +@@ -82,10 +82,12 @@ class SoSCleanerParser(): + for pattern in self.regex_patterns: + matches = [m[0] for m in re.findall(pattern, line, re.I)] + if matches: ++ matches.sort(reverse=True, key=lambda x: len(x)) + count += len(matches) + for match in matches: +- new_match = self.mapping.get(match.strip()) +- line = line.replace(match.strip(), new_match) ++ match = match.strip() ++ new_match = self.mapping.get(match) ++ line = line.replace(match, new_match) + return line, count + + def parse_string_for_keys(self, string_data): +@@ -102,7 +104,9 @@ class SoSCleanerParser(): + :returns: The obfuscated line + :rtype: ``str`` + """ +- for key, val in self.mapping.dataset.items(): ++ for pair in sorted(self.mapping.dataset.items(), reverse=True, ++ key=lambda x: len(x[0])): ++ key, val = pair + if key in string_data: + string_data = string_data.replace(key, val) + return string_data +diff --git a/sos/cleaner/parsers/keyword_parser.py b/sos/cleaner/parsers/keyword_parser.py +index 3dc2b7f0..9134f82d 100644 +--- a/sos/cleaner/parsers/keyword_parser.py ++++ b/sos/cleaner/parsers/keyword_parser.py +@@ -42,7 +42,7 @@ class SoSKeywordParser(SoSCleanerParser): + + def parse_line(self, line): + count = 0 +- for keyword in self.user_keywords: ++ for keyword in sorted(self.user_keywords, reverse=True): + if keyword in line: + line = line.replace(keyword, self.mapping.get(keyword)) + count += 1 +diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py +index 2bb6c7f3..0c3bbac4 100644 +--- a/sos/cleaner/parsers/username_parser.py ++++ b/sos/cleaner/parsers/username_parser.py +@@ -51,7 +51,7 @@ class SoSUsernameParser(SoSCleanerParser): + + def parse_line(self, line): + count = 0 +- for username in self.mapping.dataset.keys(): ++ for username in sorted(self.mapping.dataset.keys(), reverse=True): + if username in line: + count = line.count(username) + line = line.replace(username, self.mapping.get(username)) +-- +2.26.3 + + +From 7ed138fcd2ee6ece3e7fbd9e48293b212e0b4e41 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 16 Jun 2021 01:15:45 -0400 +Subject: [PATCH 06/10] [cleaner] Explicitly obfuscate directory names within + archives + +This commits adds a step to `obfuscate_report()` that explicitly walks +through all directories in the archive, and obfuscates the directory +names if necessary. + +Since this uses `obfuscate_string()` for the directory names, a +`skip_keys` list has been added to maps to allow parsers/maps to +specify matched keys (such as short names for the hostname parser) that +should not be considered when obfuscating directory names (e.g. 'www'). + +Closes: #2465 + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/__init__.py | 26 ++++++++++++++++++++++++++ + sos/cleaner/mappings/__init__.py | 4 +++- + sos/cleaner/mappings/hostname_map.py | 5 +++++ + sos/cleaner/obfuscation_archive.py | 20 ++++++++++++++++++-- + sos/cleaner/parsers/__init__.py | 2 ++ + 5 files changed, 54 insertions(+), 3 deletions(-) + +diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py +index b38c8dfc..88d4d0ea 100644 +--- a/sos/cleaner/__init__.py ++++ b/sos/cleaner/__init__.py +@@ -562,6 +562,11 @@ third party. + except Exception as err: + self.log_debug("Unable to parse file %s: %s" + % (short_name, err)) ++ try: ++ self.obfuscate_directory_names(archive) ++ except Exception as err: ++ self.log_info("Failed to obfuscate directories: %s" % err, ++ caller=archive.archive_name) + + # if the archive was already a tarball, repack it + method = archive.get_compression() +@@ -663,6 +668,27 @@ third party. + + return subs + ++ def obfuscate_directory_names(self, archive): ++ """For all directories that exist within the archive, obfuscate the ++ directory name if it contains sensitive strings found during execution ++ """ ++ self.log_info("Obfuscating directory names in archive %s" ++ % archive.archive_name) ++ for dirpath in sorted(archive.get_directory_list(), reverse=True): ++ for _name in os.listdir(dirpath): ++ _dirname = os.path.join(dirpath, _name) ++ _arc_dir = _dirname.split(archive.extracted_path)[-1] ++ if os.path.isdir(_dirname): ++ _ob_dirname = self.obfuscate_string(_name) ++ if _ob_dirname != _name: ++ _ob_arc_dir = _arc_dir.rstrip(_name) ++ _ob_arc_dir = os.path.join( ++ archive.extracted_path, ++ _ob_arc_dir.lstrip('/'), ++ _ob_dirname ++ ) ++ os.rename(_dirname, _ob_arc_dir) ++ + def obfuscate_string(self, string_data): + for parser in self.parsers: + try: +diff --git a/sos/cleaner/mappings/__init__.py b/sos/cleaner/mappings/__init__.py +index dd464e5a..5cf5c8b2 100644 +--- a/sos/cleaner/mappings/__init__.py ++++ b/sos/cleaner/mappings/__init__.py +@@ -20,8 +20,10 @@ class SoSMap(): + corresponding SoSMap() object, to allow for easy retrieval of obfuscated + items. + """ +- ++ # used for regex skips in parser.parse_line() + ignore_matches = [] ++ # used for filename obfuscations in parser.parse_string_for_keys() ++ skip_keys = [] + + def __init__(self): + self.dataset = {} +diff --git a/sos/cleaner/mappings/hostname_map.py b/sos/cleaner/mappings/hostname_map.py +index e0b7bf1d..c9a44d8d 100644 +--- a/sos/cleaner/mappings/hostname_map.py ++++ b/sos/cleaner/mappings/hostname_map.py +@@ -35,6 +35,11 @@ class SoSHostnameMap(SoSMap): + '^com..*' + ] + ++ skip_keys = [ ++ 'www', ++ 'api' ++ ] ++ + host_count = 0 + domain_count = 0 + _domains = {} +diff --git a/sos/cleaner/obfuscation_archive.py b/sos/cleaner/obfuscation_archive.py +index 88f978d9..90188358 100644 +--- a/sos/cleaner/obfuscation_archive.py ++++ b/sos/cleaner/obfuscation_archive.py +@@ -202,10 +202,22 @@ class SoSObfuscationArchive(): + """Return a list of all files within the archive""" + self.file_list = [] + for dirname, dirs, files in os.walk(self.extracted_path): ++ for _dir in dirs: ++ _dirpath = os.path.join(dirname, _dir) ++ # catch dir-level symlinks ++ if os.path.islink(_dirpath) and os.path.isdir(_dirpath): ++ self.file_list.append(_dirpath) + for filename in files: + self.file_list.append(os.path.join(dirname, filename)) + return self.file_list + ++ def get_directory_list(self): ++ """Return a list of all directories within the archive""" ++ dir_list = [] ++ for dirname, dirs, files in os.walk(self.extracted_path): ++ dir_list.append(dirname) ++ return dir_list ++ + def update_sub_count(self, fname, count): + """Called when a file has finished being parsed and used to track + total substitutions made and number of files that had changes made +@@ -230,7 +242,8 @@ class SoSObfuscationArchive(): + archive root + """ + +- if not os.path.isfile(self.get_file_path(filename)): ++ if (not os.path.isfile(self.get_file_path(filename)) and not ++ os.path.islink(self.get_file_path(filename))): + return True + + for _skip in self.skip_list: +@@ -266,7 +279,10 @@ class SoSObfuscationArchive(): + if re.match(_arc_reg, fname): + return True + +- return self.file_is_binary(fname) ++ if os.path.isfile(self.get_file_path(fname)): ++ return self.file_is_binary(fname) ++ # don't fail on dir-level symlinks ++ return False + + def file_is_binary(self, fname): + """Determine if the file is a binary file or not. +diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py +index cfa20b95..84874475 100644 +--- a/sos/cleaner/parsers/__init__.py ++++ b/sos/cleaner/parsers/__init__.py +@@ -107,6 +107,8 @@ class SoSCleanerParser(): + for pair in sorted(self.mapping.dataset.items(), reverse=True, + key=lambda x: len(x[0])): + key, val = pair ++ if key in self.mapping.skip_keys: ++ continue + if key in string_data: + string_data = string_data.replace(key, val) + return string_data +-- +2.26.3 + + +From f180150277b706e72f2445287f3d0b6943efa252 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 16 Jun 2021 02:24:51 -0400 +Subject: [PATCH 07/10] [hostname parser,map] Attempt to detect strings with + FQDN substrings + +This commit updates the hostname parser and associated map to be able to +better detect and obfuscate FQDN substrings within file content and file +names, particularly when the regex patterns failed to match a hostname +that is formatted with '_' characters rather than '.' characters. + +The `get()` method has been updated to alow preserve characters and +certain extensions that are not part of the FQDN, but are brought in by +the regex pattern due to the fact that we need to use word boundary +indicators within the pattern. + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/mappings/hostname_map.py | 59 +++++++++++++++++++++++--- + sos/cleaner/parsers/__init__.py | 3 +- + sos/cleaner/parsers/hostname_parser.py | 30 ++++++++++--- + 3 files changed, 81 insertions(+), 11 deletions(-) + +diff --git a/sos/cleaner/mappings/hostname_map.py b/sos/cleaner/mappings/hostname_map.py +index c9a44d8d..d4b2c88e 100644 +--- a/sos/cleaner/mappings/hostname_map.py ++++ b/sos/cleaner/mappings/hostname_map.py +@@ -104,7 +104,7 @@ class SoSHostnameMap(SoSMap): + host = domain.split('.') + if len(host) == 1: + # don't block on host's shortname +- return True ++ return host[0] in self.hosts.keys() + else: + domain = host[0:-1] + for known_domain in self._domains: +@@ -113,12 +113,59 @@ class SoSHostnameMap(SoSMap): + return False + + def get(self, item): +- if item.startswith(('.', '_')): +- item = item.lstrip('._') +- item = item.strip() ++ prefix = '' ++ suffix = '' ++ final = None ++ # The regex pattern match may include a leading and/or trailing '_' ++ # character due to the need to use word boundary matching, so we need ++ # to strip these from the string during processing, but still keep them ++ # in the returned string to not mangle the string replacement in the ++ # context of the file or filename ++ while item.startswith(('.', '_')): ++ prefix += item[0] ++ item = item[1:] ++ while item.endswith(('.', '_')): ++ suffix += item[-1] ++ item = item[0:-1] + if not self.domain_name_in_loaded_domains(item.lower()): + return item +- return super(SoSHostnameMap, self).get(item) ++ if item.endswith(('.yaml', '.yml', '.crt', '.key', '.pem')): ++ ext = '.' + item.split('.')[-1] ++ item = item.replace(ext, '') ++ suffix += ext ++ if item not in self.dataset.keys(): ++ # try to account for use of '-' in names that include hostnames ++ # and don't create new mappings for each of these ++ for _existing in sorted(self.dataset.keys(), reverse=True, ++ key=lambda x: len(x)): ++ _host_substr = False ++ _test = item.split(_existing) ++ _h = _existing.split('.') ++ # avoid considering a full FQDN match as a new match off of ++ # the hostname of an existing match ++ if _h[0] and _h[0] in self.hosts.keys(): ++ _host_substr = True ++ if len(_test) == 1 or not _test[0]: ++ # does not match existing obfuscation ++ continue ++ elif _test[0].endswith('.') and not _host_substr: ++ # new hostname in known domain ++ final = super(SoSHostnameMap, self).get(item) ++ break ++ elif item.split(_test[0]): ++ # string that includes existing FQDN obfuscation substring ++ # so, only obfuscate the FQDN part ++ try: ++ itm = item.split(_test[0])[1] ++ final = _test[0] + super(SoSHostnameMap, self).get(itm) ++ break ++ except Exception: ++ # fallback to still obfuscating the entire item ++ pass ++ ++ if not final: ++ final = super(SoSHostnameMap, self).get(item) ++ return prefix + final + suffix + + def sanitize_item(self, item): + host = item.split('.') +@@ -146,6 +193,8 @@ class SoSHostnameMap(SoSMap): + """Obfuscate the short name of the host with an incremented counter + based on the total number of obfuscated host names + """ ++ if not hostname: ++ return hostname + if hostname not in self.hosts: + ob_host = "host%s" % self.host_count + self.hosts[hostname] = ob_host +diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py +index 84874475..57d2020a 100644 +--- a/sos/cleaner/parsers/__init__.py ++++ b/sos/cleaner/parsers/__init__.py +@@ -87,7 +87,8 @@ class SoSCleanerParser(): + for match in matches: + match = match.strip() + new_match = self.mapping.get(match) +- line = line.replace(match, new_match) ++ if new_match != match: ++ line = line.replace(match, new_match) + return line, count + + def parse_string_for_keys(self, string_data): +diff --git a/sos/cleaner/parsers/hostname_parser.py b/sos/cleaner/parsers/hostname_parser.py +index 9982024b..3de6bb08 100644 +--- a/sos/cleaner/parsers/hostname_parser.py ++++ b/sos/cleaner/parsers/hostname_parser.py +@@ -18,7 +18,7 @@ class SoSHostnameParser(SoSCleanerParser): + map_file_key = 'hostname_map' + prep_map_file = 'sos_commands/host/hostname' + regex_patterns = [ +- r'(((\b|_)[a-zA-Z0-9-\.]{1,200}\.[a-zA-Z]{1,63}\b))' ++ r'(((\b|_)[a-zA-Z0-9-\.]{1,200}\.[a-zA-Z]{1,63}(\b|_)))' + ] + + def __init__(self, conf_file=None, opt_domains=None): +@@ -66,10 +66,30 @@ class SoSHostnameParser(SoSCleanerParser): + """Override the default parse_line() method to also check for the + shortname of the host derived from the hostname. + """ ++ ++ def _check_line(ln, count, search, repl=None): ++ """Perform a second manual check for substrings that may have been ++ missed by regex matching ++ """ ++ if search in self.mapping.skip_keys: ++ return ln, count ++ if search in ln: ++ count += ln.count(search) ++ ln = ln.replace(search, self.mapping.get(repl or search)) ++ return ln, count ++ + count = 0 + line, count = super(SoSHostnameParser, self).parse_line(line) +- for short_name in self.short_names: +- if short_name in line: +- count += 1 +- line = line.replace(short_name, self.mapping.get(short_name)) ++ # make an additional pass checking for '_' formatted substrings that ++ # the regex patterns won't catch ++ hosts = [h for h in self.mapping.dataset.keys() if '.' in h] ++ for host in sorted(hosts, reverse=True, key=lambda x: len(x)): ++ fqdn = host ++ for c in '.-': ++ fqdn = fqdn.replace(c, '_') ++ line, count = _check_line(line, count, fqdn, host) ++ ++ for short_name in sorted(self.short_names, reverse=True): ++ line, count = _check_line(line, count, short_name) ++ + return line, count +-- +2.26.3 + + +From ec46e6a8fac58ed757344be3751eb1f925eab981 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Mon, 14 Jun 2021 09:31:07 -0400 +Subject: [PATCH 08/10] [ocp] Refine OCP node options in cluster profile + +Adds explicit setting of primary/node sos options for the `openshift` +plugin within the cluster, rather than relying on default configurations +and best practices to avoid duplicate collections. + +Signed-off-by: Jake Hunsaker +--- + sos/collector/clusters/ocp.py | 65 +++++++++++++++++++++++++++++++++-- + sos/collector/sosnode.py | 4 +-- + 2 files changed, 65 insertions(+), 4 deletions(-) + +diff --git a/sos/collector/clusters/ocp.py b/sos/collector/clusters/ocp.py +index 283fcfd1..ddff84a4 100644 +--- a/sos/collector/clusters/ocp.py ++++ b/sos/collector/clusters/ocp.py +@@ -8,6 +8,8 @@ + # + # See the LICENSE file in the source distribution for further information. + ++import os ++ + from pipes import quote + from sos.collector.clusters import Cluster + +@@ -18,10 +20,14 @@ class ocp(Cluster): + cluster_name = 'OpenShift Container Platform v4' + packages = ('openshift-hyperkube', 'openshift-clients') + ++ api_collect_enabled = False ++ token = None ++ + option_list = [ + ('label', '', 'Colon delimited list of labels to select nodes with'), + ('role', '', 'Colon delimited list of roles to select nodes with'), +- ('kubeconfig', '', 'Path to the kubeconfig file') ++ ('kubeconfig', '', 'Path to the kubeconfig file'), ++ ('token', '', 'Service account token to use for oc authorization') + ] + + def fmt_oc_cmd(self, cmd): +@@ -32,9 +38,20 @@ class ocp(Cluster): + return "oc --config %s %s" % (self.get_option('kubeconfig'), cmd) + return "oc %s" % cmd + ++ def _attempt_oc_login(self): ++ """Attempt to login to the API using the oc command using a provided ++ token ++ """ ++ _res = self.exec_primary_cmd("oc login --insecure-skip-tls-verify=True" ++ " --token=%s" % self.token) ++ return _res['status'] == 0 ++ + def check_enabled(self): + if super(ocp, self).check_enabled(): + return True ++ self.token = self.get_option('token') or os.getenv('SOSOCPTOKEN', None) ++ if self.token: ++ self._attempt_oc_login() + _who = self.fmt_oc_cmd('whoami') + return self.exec_master_cmd(_who)['status'] == 0 + +@@ -106,4 +123,48 @@ class ocp(Cluster): + return 'master' in self.node_dict[sosnode.address]['roles'] + + def set_master_options(self, node): +- node.opts.enable_plugins.append('openshift') ++ node.enable_plugins.append('openshift') ++ if self.api_collect_enabled: ++ # a primary has already been enabled for API collection, disable ++ # it among others ++ node.plugin_options.append('openshift.no-oc=on') ++ else: ++ _oc_cmd = 'oc' ++ if node.host.containerized: ++ _oc_cmd = '/host/bin/oc' ++ # when run from a container, the oc command does not inherit ++ # the default config, so if it's present then pass it here to ++ # detect a funcitonal oc command. This is sidestepped in sos ++ # report by being able to chroot the `oc` execution which we ++ # cannot do remotely ++ if node.file_exists('/root/.kube/config', need_root=True): ++ _oc_cmd += ' --kubeconfig /host/root/.kube/config' ++ can_oc = node.run_command("%s whoami" % _oc_cmd, ++ use_container=node.host.containerized, ++ # container is available only to root ++ # and if rhel, need to run sos as root ++ # anyways which will run oc as root ++ need_root=True) ++ if can_oc['status'] == 0: ++ # the primary node can already access the API ++ self.api_collect_enabled = True ++ elif self.token: ++ node.sos_env_vars['SOSOCPTOKEN'] = self.token ++ self.api_collect_enabled = True ++ elif self.get_option('kubeconfig'): ++ kc = self.get_option('kubeconfig') ++ if node.file_exists(kc): ++ if node.host.containerized: ++ kc = "/host/%s" % kc ++ node.sos_env_vars['KUBECONFIG'] = kc ++ self.api_collect_enabled = True ++ if self.api_collect_enabled: ++ msg = ("API collections will be performed on %s\nNote: API " ++ "collections may extend runtime by 10s of minutes\n" ++ % node.address) ++ self.soslog.info(msg) ++ self.ui_log.info(msg) ++ ++ def set_node_options(self, node): ++ # don't attempt OC API collections on non-primary nodes ++ node.plugin_options.append('openshift.no-oc=on') +diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py +index 1c25cc34..6597d236 100644 +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -202,11 +202,11 @@ class SosNode(): + self.opts.registry_authfile or self.host.container_authfile + ) + +- def file_exists(self, fname): ++ def file_exists(self, fname, need_root=False): + """Checks for the presence of fname on the remote node""" + if not self.local: + try: +- res = self.run_command("stat %s" % fname) ++ res = self.run_command("stat %s" % fname, need_root=need_root) + return res['status'] == 0 + except Exception: + return False +-- +2.26.3 + + +From eea8e15845a8bcba91b93a5310ba693e8c20ab9c Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Thu, 17 Jun 2021 09:52:36 -0400 +Subject: [PATCH 09/10] [cleaner] Don't obfuscate default 'core' user + +The 'core' user is a common default user on containerized hosts, and +obfuscation of it is not advantageous, much like the default 'ubuntu' +user for that distribution. + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/parsers/username_parser.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py +index 0c3bbac4..64843205 100644 +--- a/sos/cleaner/parsers/username_parser.py ++++ b/sos/cleaner/parsers/username_parser.py +@@ -28,6 +28,7 @@ class SoSUsernameParser(SoSCleanerParser): + prep_map_file = 'sos_commands/login/lastlog_-u_1000-60000' + regex_patterns = [] + skip_list = [ ++ 'core', + 'nobody', + 'nfsnobody', + 'root' +-- +2.26.3 + + +From 581429ca65131711c96f9d56bf2f0e18779aec2e Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Fri, 18 Jun 2021 14:26:55 -0400 +Subject: [PATCH 10/10] [cleaner] Fix checksum and archive pruning from archive + list + +Fixes an issue where checksums may have gotten into the list of archives +to be cleaned, which would cause further issues later. Additionally, +prevents nested sosreports from top-level archives (such as from +`collect`) from being removed for being a binary file when that +top-level archive gets obfuscated. +--- + sos/cleaner/__init__.py | 5 +++-- + sos/cleaner/obfuscation_archive.py | 1 + + 2 files changed, 4 insertions(+), 2 deletions(-) + +diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py +index 88d4d0ea..8280bc50 100644 +--- a/sos/cleaner/__init__.py ++++ b/sos/cleaner/__init__.py +@@ -226,8 +226,7 @@ third party. + nested_archives = [] + for _file in archive.getmembers(): + if (re.match('sosreport-.*.tar', _file.name.split('/')[-1]) and not +- (_file.name.endswith('.md5') or +- _file.name.endswith('.sha256'))): ++ (_file.name.endswith(('.md5', '.sha256')))): + nested_archives.append(_file.name.split('/')[-1]) + + if nested_archives: +@@ -235,6 +234,8 @@ third party. + nested_path = self.extract_archive(archive) + for arc_file in os.listdir(nested_path): + if re.match('sosreport.*.tar.*', arc_file): ++ if arc_file.endswith(('.md5', '.sha256')): ++ continue + self.report_paths.append(os.path.join(nested_path, + arc_file)) + # add the toplevel extracted archive +diff --git a/sos/cleaner/obfuscation_archive.py b/sos/cleaner/obfuscation_archive.py +index 90188358..e357450b 100644 +--- a/sos/cleaner/obfuscation_archive.py ++++ b/sos/cleaner/obfuscation_archive.py +@@ -58,6 +58,7 @@ class SoSObfuscationArchive(): + Returns: list of files and file regexes + """ + return [ ++ 'sosreport-', + 'sys/firmware', + 'sys/fs', + 'sys/kernel/debug', +-- +2.26.3 + diff --git a/SOURCES/sos-bz1985037-cleaner-AD-users-obfuscation.patch b/SOURCES/sos-bz1985037-cleaner-AD-users-obfuscation.patch new file mode 100644 index 0000000..2e5835a --- /dev/null +++ b/SOURCES/sos-bz1985037-cleaner-AD-users-obfuscation.patch @@ -0,0 +1,142 @@ +From 7e471676fe41dab155a939c60446cc7b7dab773b Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Tue, 20 Jul 2021 11:09:29 -0400 +Subject: [PATCH] [username parser] Load usernames from `last` for LDAP users + +AD/LDAP users are not reported into `lastlog` generally, however they +are reported in `last`. Conversely, `last` does not report local users +who have not logged in but still exist. + +In order to obfuscate both kinds of users, we need to look at both +sources. + +For this, first allow parsers to specify multiple prep files. Second, +update the username parser to search through all `lastlog` collections +as well as the `last` collection. + +Also includes a small update to the username parser's prep loading logic +to ensure we are iterating over each username discovered only once. + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/__init__.py | 38 ++++++++++++++------------ + sos/cleaner/parsers/__init__.py | 2 +- + sos/cleaner/parsers/username_parser.py | 24 +++++++++++++--- + 3 files changed, 42 insertions(+), 22 deletions(-) + +diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py +index ca5f93e5..6aadfe79 100644 +--- a/sos/cleaner/__init__.py ++++ b/sos/cleaner/__init__.py +@@ -518,23 +518,27 @@ third party. + for _parser in self.parsers: + if not _parser.prep_map_file: + continue +- _arc_path = os.path.join(_arc_name, _parser.prep_map_file) +- try: +- if is_dir: +- _pfile = open(_arc_path, 'r') +- content = _pfile.read() +- else: +- _pfile = archive.extractfile(_arc_path) +- content = _pfile.read().decode('utf-8') +- _pfile.close() +- if isinstance(_parser, SoSUsernameParser): +- _parser.load_usernames_into_map(content) +- for line in content.splitlines(): +- if isinstance(_parser, SoSHostnameParser): +- _parser.load_hostname_into_map(line) +- self.obfuscate_line(line) +- except Exception as err: +- self.log_debug("Could not prep %s: %s" % (_arc_path, err)) ++ if isinstance(_parser.prep_map_file, str): ++ _parser.prep_map_file = [_parser.prep_map_file] ++ for parse_file in _parser.prep_map_file: ++ _arc_path = os.path.join(_arc_name, parse_file) ++ try: ++ if is_dir: ++ _pfile = open(_arc_path, 'r') ++ content = _pfile.read() ++ else: ++ _pfile = archive.extractfile(_arc_path) ++ content = _pfile.read().decode('utf-8') ++ _pfile.close() ++ if isinstance(_parser, SoSUsernameParser): ++ _parser.load_usernames_into_map(content) ++ for line in content.splitlines(): ++ if isinstance(_parser, SoSHostnameParser): ++ _parser.load_hostname_into_map(line) ++ self.obfuscate_line(line) ++ except Exception as err: ++ self.log_debug("Could not prep %s: %s" ++ % (_arc_path, err)) + + def obfuscate_report(self, report): + """Individually handle each archive or directory we've discovered by +diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py +index 3076db39..af6e375e 100644 +--- a/sos/cleaner/parsers/__init__.py ++++ b/sos/cleaner/parsers/__init__.py +@@ -50,7 +50,7 @@ class SoSCleanerParser(): + skip_line_patterns = [] + skip_files = [] + map_file_key = 'unset' +- prep_map_file = 'unset' ++ prep_map_file = [] + + def __init__(self, conf_file=None): + # attempt to load previous run data into the mapping for the parser +diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py +index 96ce5f0c..b142e371 100644 +--- a/sos/cleaner/parsers/username_parser.py ++++ b/sos/cleaner/parsers/username_parser.py +@@ -25,13 +25,24 @@ class SoSUsernameParser(SoSCleanerParser + + name = 'Username Parser' + map_file_key = 'username_map' +- prep_map_file = 'sos_commands/login/lastlog_-u_1000-60000' ++ prep_map_file = [ ++ 'sos_commands/login/lastlog_-u_1000-60000', ++ 'sos_commands/login/lastlog_-u_60001-65536', ++ 'sos_commands/login/lastlog_-u_65537-4294967295', ++ # AD users will be reported here, but favor the lastlog files since ++ # those will include local users who have not logged in ++ 'sos_commands/login/last' ++ ] + regex_patterns = [] + skip_list = [ + 'core', + 'nobody', + 'nfsnobody', +- 'root' ++ 'shutdown', ++ 'reboot', ++ 'root', ++ 'ubuntu', ++ 'wtmp' + ] + + def __init__(self, conf_file=None, opt_names=None): +@@ -44,11 +54,17 @@ class SoSUsernameParser(SoSCleanerParser): + """Since we don't get the list of usernames from a straight regex for + this parser, we need to override the initial parser prepping here. + """ ++ users = set() + for line in content.splitlines()[1:]: +- user = line.split()[0] ++ try: ++ user = line.split()[0] ++ except Exception: ++ continue + if user in self.skip_list: + continue +- self.mapping.get(user) ++ users.add(user) ++ for each in users: ++ self.mapping.get(each) + + def parse_line(self, line): + count = 0 +-- +2.31.1 + diff --git a/SOURCES/sos-bz1985986-potential-issues-static-analyse.patch b/SOURCES/sos-bz1985986-potential-issues-static-analyse.patch new file mode 100644 index 0000000..0c359e6 --- /dev/null +++ b/SOURCES/sos-bz1985986-potential-issues-static-analyse.patch @@ -0,0 +1,65 @@ +From 6d5cbe90e17534d53d7fe42dff4d8ca734acf594 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Tue, 29 Jun 2021 15:49:00 -0400 +Subject: [PATCH] [yum] Fix potential traceback when yum history is empty + +Like we did in #969 for `dnf`, fix a potential issue where we would +generate a traceback in the plugin when `yum history` is empty. + +Signed-off-by: Jake Hunsaker +--- + sos/report/plugins/yum.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/sos/report/plugins/yum.py b/sos/report/plugins/yum.py +index 54e222df..aec805e6 100644 +--- a/sos/report/plugins/yum.py ++++ b/sos/report/plugins/yum.py +@@ -91,7 +91,7 @@ class Yum(Plugin, RedHatPlugin): + # packages installed/erased/updated per transaction + if self.get_option("yum-history-info"): + history = self.exec_cmd("yum history") +- transactions = None ++ transactions = -1 + if history['status'] == 0: + for line in history['output'].splitlines(): + try: +-- +2.31.1 + +From a7a4ef73faee6cddba36bf670d4a20ab0521c36f Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Wed, 30 Jun 2021 13:10:56 +0200 +Subject: [PATCH] [plugins] Set default predicate instead of None for + robustness + +Just making the code more robustness, it could be dangerous to +set pred = None and then potentially call log_skipped_cmd that +expects "pred" of SoSPredicate type. + +Currently such a call flow can not happen, but it is worth to +make the code more robust for potential future changes. + +Resolves: #2601 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/__init__.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py +index 6fd1a3b2..b9cd28ed 100644 +--- a/sos/report/plugins/__init__.py ++++ b/sos/report/plugins/__init__.py +@@ -1629,7 +1629,7 @@ class Plugin(object): + + def _add_cmd_output(self, **kwargs): + """Internal helper to add a single command to the collection list.""" +- pred = kwargs.pop('pred') if 'pred' in kwargs else None ++ pred = kwargs.pop('pred') if 'pred' in kwargs else SoSPredicate(self) + soscmd = SoSCommand(**kwargs) + self._log_debug("packed command: " + soscmd.__str__()) + for _skip_cmd in self.skip_commands: +-- +2.31.1 + diff --git a/SOURCES/sos-bz1992957-conversions-and-upgrades.patch b/SOURCES/sos-bz1992957-conversions-and-upgrades.patch deleted file mode 100644 index a39f839..0000000 --- a/SOURCES/sos-bz1992957-conversions-and-upgrades.patch +++ /dev/null @@ -1,50 +0,0 @@ -From ee5d9d017b0a1bfeaebee9c21c17e89ef1f909a8 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 26 Jul 2021 13:30:09 +0200 -Subject: [PATCH] [MigrationResults] collect info about conversions and - upgrades - -A new tiny plugin independent on leapp and convert2rhel is proposed. - -It should collect /etc/migration-results with info about RHEL -conversions and upgrades, whenever the file is present. - -Resolves: #2627 -Relevant to: rhbz#1959598 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/migration_results.py | 21 +++++++++++++++++++++ - 1 file changed, 21 insertions(+) - create mode 100644 sos/report/plugins/migration_results.py - -diff --git a/sos/report/plugins/migration_results.py b/sos/report/plugins/migration_results.py -new file mode 100644 -index 00000000..b67480ba ---- /dev/null -+++ b/sos/report/plugins/migration_results.py -@@ -0,0 +1,21 @@ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from sos.report.plugins import Plugin, RedHatPlugin -+ -+ -+class MigrationResults(Plugin, RedHatPlugin): -+ -+ short_desc = 'Information about conversions and upgrades' -+ -+ plugin_name = 'migration_results' -+ profiles = ('system',) -+ -+ files = ('/etc/migration-results',) -+ -+# vim: et ts=4 sw=4 --- -2.31.1 - diff --git a/SPECS/sos.spec b/SPECS/sos.spec index cb596ae..30e6657 100644 --- a/SPECS/sos.spec +++ b/SPECS/sos.spec @@ -4,8 +4,8 @@ Summary: A set of tools to gather troubleshooting information from a system Name: sos -Version: 4.0 -Release: 12%{?dist} +Version: 4.1 +Release: 5%{?dist} Group: Applications/System Source0: https://github.com/sosreport/sos/archive/%{version}/sos-%{version}.tar.gz Source1: sos-audit-%{auditversion}.tgz @@ -20,29 +20,28 @@ Requires: xz Conflicts: vdsm < 4.40 Obsoletes: sos-collector Recommends: python3-pexpect -Patch1: sos-bz1827801-streamlined-sanitize_item.patch -Patch2: sos-bz1874295-osp-ironic-inspector-configs.patch -Patch3: sos-bz1880372-power-logs.patch -Patch4: sos-bz1881118-crio-conf-d.patch -Patch5: sos-bz1882368-upload-functionality-issues.patch -Patch6: sos-bz1886782-exclude-panfs.patch -Patch7: sos-bz1887390-kdump-logfiles.patch -Patch8: sos-bz1888012-stratis-new-feature-output.patch -Patch9: sos-bz1891562-tmp-dir-relative-path.patch -Patch10: sos-bz1848095-collect-rhev-pki.patch -Patch11: sos-bz1895316-collector--cluster-type.patch -Patch12: sos-bz1904045-preset-ignores-verbosity.patch -Patch13: sos-bz1905657-empty-file-stops-zero-sizelimit.patch -Patch14: sos-bz1906598-collect-broken-symlinks.patch -Patch15: sos-bz1912889-plugopts-ignored-in-configfile.patch -Patch16: sos-bz1912821-sos-collector-declare-sysroot.patch -Patch17: sos-bz1912910-empty-file-stops-collecting.patch -Patch18: sos-bz1917196-networking-ethtool-e-conditionally.patch -Patch19: sos-bz1887402-kexec-logs.patch -Patch20: sos-bz1916729-ftp-upload-no-passwd.patch -Patch21: sos-bz1925419-gluster-pubkeys-statusfile.patch -Patch22: sos-bz1928650-powerpc-nhv-scsi-logs.patch -Patch23: sos-bz1992957-conversions-and-upgrades.patch +Patch1: sos-bz1930181-collect-cleaning-consistency.patch +Patch2: sos-bz1935603-manpages-see-also.patch +Patch3: sos-bz1937418-add-cmd-timeout.patch +Patch4: sos-bz1937298-ds-mask-password-in-ldif.patch +Patch5: sos-bz1939963-gather-cups-browsed-logs.patch +Patch6: sos-bz1940502-sssd-memcache-and-logs.patch +Patch7: sos-bz1942276-ibmvNIC-dynamic-debugs.patch +Patch8: sos-bz1956673-pulpcore-plugin.patch +Patch9: sos-bz1959413-saphana-traceback.patch +Patch10: sos-bz1961458-collect-nstat.patch +Patch11: sos-bz1961229-snapper-plugin-and-allocation-failures.patch +Patch12: sos-bz1925419-all-gluster-files.patch +Patch13: sos-bz1964499-obfuscate-fqdn-from-dnf-log.patch +Patch14: sos-bz1886711-enhance-tc-hw-offload.patch +Patch15: sos-bz1965001-fix-avc-copystating-proc-sys.patch +Patch16: sos-bz1967613-sssd-common.patch +Patch17: sos-bz1973675-ocp-cluster-cleaner.patch +Patch18: sos-bz1923938-sos-log-effective-options.patch +Patch19: sos-bz1985986-potential-issues-static-analyse.patch +Patch20: sos-bz1959598-conversions-and-upgrades.patch +Patch21: sos-bz1665947-rhui-plugin.patch +Patch22: sos-bz1985037-cleaner-AD-users-obfuscation.patch %description @@ -76,7 +75,6 @@ support technicians and developers. %patch20 -p1 %patch21 -p1 %patch22 -p1 -%patch23 -p1 %build %py3_build @@ -92,12 +90,15 @@ cd %{name}-audit-%{auditversion} DESTDIR=%{buildroot} ./install.sh cd .. -mkdir -p %{buildroot}%{_sysconfdir}/sos/cleaner +mkdir -p %{buildroot}%{_sysconfdir}/sos/{cleaner,presets.d,extras.d,groups.d} %files -f %{name}.lang %{_sbindir}/sosreport %{_sbindir}/sos %{_sbindir}/sos-collector +%dir /etc/sos/presets.d +%dir /etc/sos/extras.d +%dir /etc/sos/groups.d %{python3_sitelib}/* %{_mandir}/man1/sosreport.1.gz %{_mandir}/man1/sos-clean.1.gz @@ -111,6 +112,7 @@ mkdir -p %{buildroot}%{_sysconfdir}/sos/cleaner %license LICENSE %config(noreplace) %{_sysconfdir}/sos/sos.conf %config(noreplace) %{_sysconfdir}/sos/cleaner +%config /usr/config/sos.conf %package audit Summary: Audit use of some commands for support purposes @@ -121,7 +123,7 @@ Group: Application/System Sos-audit provides configuration files for the Linux Auditing System to track the use of some commands capable of changing the configuration -of the system. Currently storage and filesystem commands are audited. +of the system. Currently storage and filesystem commands are audited. %post audit %{_sbindir}/sos-audit.sh @@ -139,21 +141,63 @@ of the system. Currently storage and filesystem commands are audited. %ghost /etc/audit/rules.d/40-sos-storage.rules %changelog -* Thu Aug 12 2021 Pavel Moravec = 4.0-12 -- [MigrationResults] collect info about conversions+upgrades - Resolves: bz1992957 - -* Wed Mar 17 2021 Pavel Moravec = 4.0-11 +* Wed Aug 11 2021 Pavel Moravec = 4.1-5 +- [report,collect] unify --map-file arguments + Resolves: bz1923938 +- [rhui] add new plugin for RHUI 4 + Resolves: bz1665947 +- [username parser] Load usernames from `last` for LDAP users + Resolves: bz1985037 + +* Mon Jul 26 2021 Pavel Moravec = 4.1-4 +- [options] allow variant option names in config file + Resolves: bz1923938 +- [plugins] Set default predicate instead of None + Resolves: bz1985986 +- [MigrationResults] collect info about conversions + Resolves: bz1959598 + +* Mon Jun 21 2021 Pavel Moravec = 4.1-3 - [gluster] collect public keys from the right dir Resolves: bz1925419 - -* Thu Mar 11 2021 Pavel Moravec = 4.0-10 -- [powerpc] Collect logs for power specific components (HNV and SCSI) - Resolves: bz1928650 - -* Fri Mar 05 2021 Pavel Moravec = 4.0-9 -- [gluster] Add glusterd public keys and status files - Resolves: bz1925419 +- [cleaner] Only skip packaging-based files for the IP parse + Resolves: bz1964499 +- [networking] collect also tc filter show ingress + Resolves: bz1886711 +- [archive] skip copying SELinux context for /proc and /sys + Resolves: bz1965001 +- [sssd] sssd plugin when sssd-common + Resolves: bz1967613 +- Various OCP/cluster/cleanup enhancements + Resolves: bz1973675 + +* Tue May 18 2021 Pavel Moravec = 4.1-2 +- Load maps from all archives before obfuscation + Resolves: bz1930181 +- Multiple fixes in man pages + Resolves: bz1935603 +- [ds] Mask password and encryption keys in ldif files + Resolves: bz1937298 +- [report] add --cmd-timeout option + Resolves: bz1937418 +- [cups] Add gathering cups-browsed logs + Resolves: bz1939963 +- [sssd] Collect memory cache / individual logfiles + Resolves: bz1940502 +- Collect ibmvNIC dynamic_debugs + Resolves: bz1942276 +- [pulpcore] add plugin for pulp-3 + Resolves: bz1956673 +- [saphana] remove redundant unused argument of get_inst_info + Resolves: bz1959413 +- [networking] Add nstat command support + Resolves: bz1961458 +- [snapper] add a new plugin + Resolves: bz1961229 + +* Mon Apr 26 2021 Pavel Moravec = 4.1-1 +- Rebase on upstream 4.1 + Resolves: bz1928679 * Tue Feb 16 2021 Pavel Moravec = 4.0-8 - Automatically create directory for sos-cleaner default_mapping @@ -216,751 +260,3 @@ of the system. Currently storage and filesystem commands are audited. * Tue Oct 13 2020 Pavel Moravec = 4.0-1 - Rebase on upstream 4.0 Resolves: bz1827801 - -* Wed Aug 19 2020 Pavel Moravec = 3.9.1-6 -- [networking] remove 'ethtool -e' option for bnx2x NICs - Resolves: bz1869724 - -* Fri Jul 24 2020 Pavel Moravec = 3.9.1-5 -- [logs] collect also non-persistent journal logs - Resolves: bz1850926 -- [block] Fix typo in LUKS detection - Resolves: bz1850554 -- [powerpc] Fix enablement triggers - Resolves: bz1851923 -- [pci] Update gating for lspci commands - Resolves: bz1853700 -- [containers_common] collect user-related commands outputs - Resolves: bz1776549 -- [gluster] remove only dump files + generated state files - Resolves: bz1857590 -- [kubernetes] ignore blank+empty lines in "kubectl get nodes" - Resolves: bz1859888 - -* Tue Jun 23 2020 Pavel Moravec = 3.9.1-4 -- [gluster] fix gluster volume splitlines iteration - Resolves: bz1843562 -- [nfs] merge nfsserver plugin into nfs one - Resolves: bz1844853 -- [pacemaker] Fix scrubbing when password contains an equa - Resolves: bz1845386 -- [powerpc] Add support to collect component logs - Resolves: bz1843754 - -* Wed May 27 2020 Pavel Moravec = 3.9.1-2 -- Rebase on upstream 3.9 - Resolves: bz1826656 -- [redhat] fix RH containers without sysroot Attempting to run - Resolves: bz1825283 -- [containers_common] Add plugin for common containers configs - Resolves: bz1823488 -- [rabbitmq] Call containerised rabbitmqctl report on - Resolves: bz1819662 -- [insights] collect insights-client dump - Resolves: bz1814867 -- [nvmetcli] Add new plugin for NVMe Target CLI - Resolves: bz1785546 -- [containers_common] collect rootless containers info - Resolves: bz1776549 -- [networking] collect iptables when proper kernel modules - Resolves: bz1633006 -- [navicli] replace interactive prompt by plugin option - Resolves: bz1457191 -- [xdp] Add XDP plugin - Resolves: bz1838123 - -* Thu May 21 2020 Pavel Moravec = 3.8-4 -- [container_log] fix unscoped 'logdir' variable - Resolves: bz1834421 - -* Wed May 06 2020 Pavel Moravec = 3.8-3 -- [containers_common] Add plugin for common containers configs - Resolves: bz1823488 - -* Fri Jan 10 2020 Pavel Moravec = 3.8-2 -- [plugins] improve heuristic for applying --since - Resolves: bz1789049 -- [Predicate] Override __bool__ to allow py3 evaluation - Resolves: bz1789018 -- [ceph] Add 'ceph insights' command output - Resolves: bz1783034 -- [dnf] Collect dnf module list - Resolves: bz1781819 -- [kernel,networking] collect bpftool net list for each - Resolves: bz1768956 -- [libreswan] New plugin for "libreswan" IPsec - Resolves: bz1741330 -- [kernel] collect "bpftool net list" - Resolves: bz1721779 -- [grub2] call grub2-config with --no-grubenv-update - Resolves: bz1709682 - -* Wed Dec 11 2019 Pavel Moravec = 3.8-1 -- Rebase on upstream 3.8 - Resolves: bz1779387 - -* Mon Nov 04 2019 Pavel Moravec = 3.7-7 -- [Plugin, kernel] interim sysroot fixes - Resolves: bz1766915 - -* Wed Oct 30 2019 Pavel Moravec = 3.7-6 -- [ovirt_hosted_engine] Add gluster deployment and cleanup log - Resolves: bz1744086 -- [vdsm]: Fix executing shell commands - Resolves: bz1744110 -- [ovn_*] Add support to containerized setups - Resolves: bz1744553 -- [ipa] collect ipa-healthcheck logs, kdcproxy configs, httpd cert - Resolves: bz1688764 - -* Wed Oct 02 2019 Pavel Moravec = 3.7-5 -- [kernel] Don't collect trace file by default - Resolves: bz1738391 - -* Thu Sep 12 2019 Pavel Moravec = 3.7-4 -- [openvswitch] catch all openvswitch2.* packages - Resolves: bz1745017 - -* Tue Jul 30 2019 Pavel Moravec = 3.7-3 -- [openstack] Extract Placement plugin from Nova - Resolves: bz1717882 -- [utilities] Fix high CPU usage and slow command collection - Resolves: bz1733352 -- [peripety] collect proper config file - Resolves: bz1665981 -- [sosreport,plugins] Stop plugin execution after timeout hit - Resolves: bz1733469 -- [nvme] collect config file everytime - Resolves: bz1665929 - -* Tue Jul 09 2019 Pavel Moravec = 3.7-2 -- [sar] collect whole sar log dir - Resolves: bz1714243 -- [archive] convert absolute symlink targets to relative - Resolves: bz1702806 -- [archive] Handle checking container sysroot in _make_leading_paths - Resolves: bz1728214 -- [frr] FRR plugin - Resolves: bz1709906 -- [policies] redhat policy to use hostname instead of rhn id - Resolves: bz1718087 -- Updates to vdsm plugin - Resolves: bz1700780 - -* Wed Jun 12 2019 Pavel Moravec = 3.7-1 -- Rebase on upstream 3.7 - Resolves: bz1684400 -- [buildah] parse container list properly even for scratch ones - Resolves: bz1687954 -- [PATCH] [maas,mysql,npm,pacemaker,postgresql] fix plugopts data types - Resolves: bz1695583 -- [plugins] add vdsm plugin - Resolves: bz1700780 -- [openstack_instack] add ansible.log - Resolves: bz1702806 -- [pcp] collect pmlogger without a sizelimit - Resolves: bz1719884 -- [foreman,satellite] increase plugin default timeouts - Resolves: bz1719885 -- [sosreport] [sosreport] initialize disabled plugins properly - Resolves: bz1719886 -- [katello] support both locations of qpid SSL certs - Resolves: bz1719887 - -* Thu May 02 2019 Pavel Moravec = 3.6-11 -- [composer] Collect sources info for all sources - Resolves: bz1678418 - -* Mon Jan 21 2019 Pavel Moravec = 3.6-10 -- [grub2] Enable plugin by grub2-common package also - Resolves: bz1666214 - -* Mon Jan 14 2019 Pavel Moravec = 3.6-9 -- [block] proper parsing of luks partition on self device - Resolves: bz1638855 -- [networking] Collect NUMA Node of each NIC - Resolves: bz1645085 -- [composer] add missing commas in list in add_copy_spec - Resolves: bz1644062 -- [opendaylight] Update directory for openDaylight logs - Resolves: bz1642377 - -* Fri Dec 13 2018 Pavel Moravec = 3.6-8 -- [plugins] fix exception when collecting empty strings - Resolves: bz1632607 -- [crypto] collect more configs and commands - Resolves: bz1638492 -- [networking] Replace "brctl: by "bridge" commands - Resolves: bz1644021 -- [firewalld] collect nftables ruleset - Resolves: bz1644022 -- [composer] New plugin for lorax-composer - Resolves: bz1644062 -- [Plugin] clean up Plugin.get_option() - Resolves: bz1655984 -- [ovirt_node] New plugin for oVirt Node - Resolves: bz1658937 -- [podman] Add support for gathering information on podman - Resolves: bz1658938 -- [postgresql] Do not limit dump size - Resolves: bz1658939 - -* Fri Oct 12 2018 Pavel Moravec = 3.6-7 -- [plugin,archive] fix remaining add_link issues - Resolves: bz1627543 -- [kernel] dont collect some tracing instance files - Resolves: bz1638637 -- [openstack_*] relax enabling of OSP RedHat plugins - Resolves: bz1638638 -- [powerpc] Add support to collect DLPAR and LPM related logs - Resolves: bz1637127 - -* Mon Sep 10 2018 Pavel Moravec = 3.6-6 -- [archive] fix leading path creation - Resolves: bz1627543 -- [atomic] Define valid preset for RHEL Atomic - Resolves: bz1627546 -- [utilities] wait till AsyncReader p.poll() returns None - Resolves: bz1627544 - -* Thu Aug 23 2018 Pavel Moravec = 3.6-5 -- [rhv-log-collector-analyzer] Add new plugin for RHV - Resolves: bz1620049 -- [kubernetes|etcd] Support OpenShift 3.10 deployments - Resolves: bz1620048 -- [krb5|gssproxy] add new plugin, collect more krb5 files - Resolves: bz1607630 -- [block] collect luksDump for all encrypted devices - Resolves: bz1599739 -- [archive] Dont copystat /sys and /proc paths - Resolves: bz1619234 - -* Fri Aug 10 2018 Pavel Moravec = 3.6-4 -- [apparmor,ceph] fix typo in add_forbidden_path - Resolves: bz1614955 -- [policies] sanitize report label - Resolves: bz1614956 -- [policies,process] make lsof execution optional, dont call on RHOSP - Resolves: bz1614957 -- [sosreport] Add mechanism to encrypt final archive - Resolves: bz1614952 -- [archive] fix stat typo - Resolves: bz1614953 -- [rhui] Fix detection of CDS for RHUI3 - Resolves: bz1614954 -- [archive] fix add_string()/do_*_sub() regression - Resolves: bz1599701 - -* Fri Aug 10 2018 Bryn M. Reeves = 3.6-3 -- Clean up spec file and sources -- Integrate sos-audit subpackage - Resolves: bz1601084 - -* Tue Jul 10 2018 Pavel Moravec = 3.6-2 -- Rebase on upstream 3.6 - Resolves: bz1549522 - -* Fri Feb 09 2018 Fedora Release Engineering - 3.5-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_28_Mass_Rebuild - -* Tue Nov 14 2017 Sandro Bonazzola - 3.5-1 -- Rebase on upstream 3.5 -- Resolves: BZ#1513030 - -* Thu Jul 27 2017 Fedora Release Engineering - 3.4-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_27_Mass_Rebuild - -* Wed Mar 29 2017 Sandro Bonazzola - 3.4-1 -- Rebase on upstream 3.4 -- Resolves: BZ#1436969 -- Resolves: BZ#1427445 - -* Thu Feb 23 2017 Sandro Bonazzola - 3.3-1 -- Rebase on upstream 3.3 -- Resolves: BZ#1411314 - -* Sat Feb 11 2017 Fedora Release Engineering - 3.2-6 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_26_Mass_Rebuild - -* Mon Dec 19 2016 Miro Hrončok - 3.2-5 -- Rebuild for Python 3.6 - -* Tue Jul 19 2016 Fedora Release Engineering - 3.2-4 -- https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages - -* Fri Feb 05 2016 Fedora Release Engineering - 3.2-3 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_24_Mass_Rebuild - -* Wed Dec 16 2015 Bryn M. Reeves = 3.2-2 -- [sosreport] ensure private temporary directory is removed -- [global] sync rawhide package with upstream -- [ceph] collect /var/lib/ceph and /var/run/ceph -- [sosreport] prepare report in a private subdirectory (CVE-2015-7529) -- [docker] collect journald logs for docker unit -- [sosreport] fix command-line report defaults -- [openstack_neutron] obfuscate server_auth in restproxy.ini -- [memory] collect swapon --show output in bytes -- [sosreport] fix command-line report defaults (proper patch ordering) -- [sapnw] call self methods properly -- [openvswitch] capture the logs, db and OVS bridges details -- [logs] fix reference to missing 'rsyslog_conf' variable -- [sapnw] Add check if saphostctrl is not present, dont use Set -- [Plugin] fix handling of symlinks in non-sysroot environments -- [openstack] Ensure openstack passwords and secrets are obfuscated -- [plugin] pass stderr through _collect_cmd_output -- [kubernetes,plugin] Support running sos inside a container -- [openstack] New Openstack Trove (DBaaS) plugin -- [services] Add more diagnostics to applications -- [openstack_neutron] Obscure passwords and secrets -- [ceph] add calamari and ragos logs and configs -- [iprconfig] enable plugin for ppc64* architectures -- [general] verify --profile contains valid plugins only -- [kernel,mpt,memory] additional kernel-related diagnostics -- [cluster] enable crm_report password scrubbing -- [sosreport] fix command-line report defaults -- [virsh] add new plugin, add listing of qemu -- [sap*,vhostmd] new plugins for SAP -- [cluster] crm_report fails to run because dir already exists -- [foreman] Skip collection of generic resources -- [apache] Added collection of conf.modules.d dir for httpd 2.4 -- [pcp] collect /etc/pcp.conf -- [puppet] adding new plugin for puppet -- [block] Don't use parted human readable output -- [general] Better handling --name and --ticket-number in -- [networking] additional ip, firewall and traffic shaping -- [infiniband] add opensm and infiniband-diags support -- [plugins/rabbitmq] Added cluster_status command output -- [networking] re-add 'ip addr' with a root symlink -- [kimchi] add new plugin -- [iprconfig] add plugin for IBM Power RAID adapters -- [ovirt] Collect engine tunables and domain information. -- [activemq] Honour all_logs and get config on RHEL -- [cluster] Add luci to packages for standalone luci servers -- [hpasm] hpasmcli commands hang under timeout -- [mysql] Collect log file -- [chrony] add chrony plugin -- [openstack_sahara] redact secrets from sahara configuration -- [openstack_sahara] add new openstack_sahara plugin -- [openstack_neutron] neutron configuration and logs files not captured -- [ovirt] remove ovirt-engine setup answer file password leak -- [networking] network plugin fails if NetworkManager is disabled -- [cluster] crm_report fails to run because dir already exists -- [mysql] improve handling of dbuser, dbpass and MYSQL_PWD -- [mysql] test for boolean values in dbuser and dbpass -- [plugin] limit path names to PC_NAME_MAX -- [squid] collect files from /var/log/squid -- [sosreport] log plugin exceptions to a file -- [ctdb] fix collection of /etc/sysconfig/ctdb -- [sosreport] fix silent exception handling -- [sosreport] do not make logging calls after OSError -- [sosreport] catch OSError exceptions in SoSReport.execute() -- [anaconda] make useradd password regex tolerant of whitespace -- [mysql] fix handling of mysql.dbpass option -- [navicli] catch exceptions if stdin is unreadable -- [docs] update man page for new options -- [sosreport] make all utf-8 handling user errors=ignore -- [kpatch] do not attempt to collect data if kpatch is not installed -- [archive] drop support for Zip archives -- [sosreport] fix archive permissions regression -- [tomcat] add support for tomcat7 and default log size limits -- [mysql] obtain database password from the environment -- [corosync] add postprocessing for corosync-objctl output -- [ovirt_hosted_engine] fix exception when force-enabled -- [yum] call rhsm-debug with --no-subscriptions -- [powerpc] allow PowerPC plugin to run on ppc64le -- [package] add Obsoletes for sos-plugins-openstack -- [pam] add pam_tally2 and faillock support -- [postgresql] obtain db password from the environment -- [pcp] add Performance Co-Pilot plugin -- [nfsserver] collect /etc/exports.d -- [sosreport] handle --compression-type correctly -- [anaconda] redact passwords in kickstart configurations -- [haproxy] add new plugin -- [keepalived] add new plugin -- [lvm2] set locking_type=0 when calling lvm commands -- [tuned] add new plugin -- [cgroups] collect /etc/sysconfig/cgred -- [plugins] ensure doc text is always displayed for plugins -- [sosreport] fix the distribution version API call -- [docker] add new plugin -- [openstack_*] include broken-out openstack plugins -- [mysql] support MariaDB -- [openstack] do not collect /var/lib/nova -- [grub2] collect grub.cfg on UEFI systems -- [sosreport] handle out-of-space errors gracefully -- [firewalld] new plugin -- [networking] collect NetworkManager status -- [kpatch] new plugin -- [global] update to upstream 3.2 release -- [foreman] add new plugin - -* Tue Nov 10 2015 Fedora Release Engineering - 3.2-0.4.a -- Rebuilt for https://fedoraproject.org/wiki/Changes/python3.5 - -* Fri Jul 17 2015 Miro Hrončok - 3.2-0.3.a -- Use Python 3 (#1014595) -- Use setup.py instead of make -- Remove some deprecated statements - -* Fri Jun 19 2015 Fedora Release Engineering - 3.2-0.2.a -- Rebuilt for https://fedoraproject.org/wiki/Fedora_23_Mass_Rebuild - -* Tue Jun 17 2014 Bryn M. Reeves = 3.2-0.1.a -- Make source URL handling compliant with packaging guidelines -- Update to new upstream pre-release sos-3.2-alpha1 - -* Sun Jun 08 2014 Fedora Release Engineering - 3.1-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_21_Mass_Rebuild - -* Tue Apr 01 2014 Bryn M. Reeves = 3.1-1 -- Update to new upstream release sos-3.1 -- Add collection of grub configuration for UEFI systems -- Raise a TypeError if add_copy_specs() is called with a string -- Add tests for Plugin.add_copy_spec()/add_copy_specs() -- Update Plugin tests to treat copy_paths as a set -- Use a set for Plugin.copy_paths -- Remove references to 'sub' parameter from plugin tests -- Remove 'sub' parameter from Plugin.add_copy_spec*() -- Drop RedHatPlugin from procenv -- Update plugin_tests.py to match new method names -- Remove obsolete checksum reference from utilities_tests.py -- Refactor Plugin.collect() pathway -- Fix x86 arch detection in processor plugin -- Pythonify Plugin._path_in_pathlist() -- Clean up package checks in processor plugin -- Replace self.policy().pkg_by_name() us in Logs plugin -- Convert infiniband to package list -- Dead code removal: PluginException -- Dead code removal: sos.plugins.common_prefix() -- Add vim tags to all python source files -- Dead code removal: utilities.checksum() -- Dead code removal: DirTree -- Dead code removal: sos_relative_path() -- Remove --profile support -- Fix plugin_test exception on six.PY2 -- Call rhsm-debug with the --sos switch -- Do not collect isos in cobbler plugin -- Match plugins against policies -- Update policy_tests.py for validate_plugin change -- Rename validatePlugin to validate_plugin -- Fix broken binary detection in satellite plugin -- Clean up get_cmd_path/make_cmd_path/make_cmd_dirs mess -- Add tuned plugin -- Update systemd support -- Fix remaining use of obsolete 'get_cmd_dir()' in plugins -- Add PowerNV specific debug data -- powerpc: Move VPD related tool under common code -- Remove the rhevm plugin. -- Replace package check with file check in anacron -- Scrub ldap_default_authtok password in sssd plugin -- Eliminate hard-coded /var/log/sa paths in sar plugin -- Remove useless check_enabled() from sar plugin -- Improve error message when cluster.crm_from is invalid -- Fix command output substitution exception -- Add distupgrade plugin -- Fix gluster volume name extraction -- Ensure unused fds are closed when calling subprocesses via Popen -- Pass --no-archive to rhsm-debug script -- postgresql: allow use TCP socket -- postgresql: added license and copyright -- postgresql: add logs about errors / warnings -- postgresql: minor fixes -- Include geo-replication status in gluster plugin -- Make get_cmd_output_now() behaviour match 2.2 -- Add rhsm-debug collection to yum plugin -- Always treat rhevm vdsmlogs option as string -- Fix verbose file logging -- Fix get_option() use in cluster plugin -- Fix cluster postproc regression -- Ensure superclass postproc method is called in ldap plugin -- Remove obsolete diagnostics code from ldap plugin -- Fix cluster module crm_report support - -* Thu Mar 20 2014 Bryn M. Reeves = 3.0-23 -- Call rhsm-debug with the --sos switch - -* Mon Mar 03 2014 Bryn M. Reeves -- Fix package check in anacron plugin - -* Wed Feb 12 2014 Bryn M. Reeves -- Remove obsolete rhel_version() usage from yum plugin - -* Tue Feb 11 2014 Bryn M. Reeves -- Prevent unhandled exception during command output substitution - -* Mon Feb 10 2014 Bryn M. Reeves -- Fix generation of volume names in gluster plugin -- Add distupgrade plugin - -* Tue Feb 04 2014 Bryn M. Reeves -- Prevent file descriptor leaks when using Popen -- Disable zip archive creation when running rhsm-debug -- Include volume geo-replication status in gluster plugin - -* Mon Feb 03 2014 Bryn M. Reeves -- Fix get_option use in cluster plugin -- Fix debug logging to file when given '-v' -- Always treat rhevm plugin's vdsmlogs option as a string -- Run the rhsm-debug script from yum plugin - -* Fri Jan 31 2014 Bryn M. Reeves -- Add new plugin to collect OpenHPI configuration -- Fix cluster plugin crm_report support -- Fix file postprocessing in ldap plugin -- Remove collection of anaconda-ks.cfg from general plugin - -* Fri Jan 24 2014 Bryn M. Reeves -- Remove debug statements from logs plugin -- Make ethernet interface detection more robust -- Fix specifying multiple plugin options on the command line -- Make log and message levels match previous versions -- Log a warning message when external commands time out -- Remove --upload command line option -- Update sos UI text to match upstream - -* Fri Dec 27 2013 Daniel Mach -- Mass rebuild 2013-12-27 - -* Thu Nov 14 2013 Bryn M. Reeves -- Fix regressions introduced with --build option - -* Tue Nov 12 2013 Bryn M. Reeves -- Fix typo in yum plug-in add_forbidden_paths -- Add krb5 plug-in and drop collection of krb5.keytab - -* Fri Nov 8 2013 Bryn M. Reeves -- Add nfs client plug-in -- Fix traceback when sar module force-enabled - -* Thu Nov 7 2013 Bryn M. Reeves -- Restore --build command line option -- Collect saved vmcore-dmesg.txt files -- Normalize temporary directory paths - -* Tue Nov 5 2013 Bryn M. Reeves -- Add domainname output to NIS plug-in -- Collect /var/log/squid in squid plug-in -- Collect mountstats and mountinfo in filesys plug-in -- Add PowerPC plug-in from upstream - -* Thu Oct 31 2013 Bryn M. Reeves -- Remove version checks in gluster plug-in -- Check for usable temporary directory -- Fix --alloptions command line option -- Fix configuration fail regression - -* Wed Oct 30 2013 Bryn M. Reeves -- Include /etc/yaboot.conf in boot plug-in -- Fix collection of brctl output in networking plug-in -- Verify limited set of RPM packages by default -- Do not strip newlines from command output -- Limit default sar data collection - -* Thu Oct 3 2013 Bryn M. Reeves -- Do not attempt to read RPC pseudo files in networking plug-in -- Restrict wbinfo collection to the current domain -- Add obfuscation of luci secrets to cluster plug-in -- Add XFS plug-in -- Fix policy class handling of --tmp-dir -- Do not set batch mode if stdin is not a TTY -- Attempt to continue when reading bad input in interactive mode - -* Wed Aug 14 2013 Bryn M. Reeves -- Add crm_report support to cluster plug-in -- Fix rhel_version() usage in cluster and s390 plug-ins -- Strip trailing newline from command output - -* Mon Jun 10 2013 Bryn M. Reeves -- Silence 'could not run' messages at default verbosity -- New upstream release - -* Thu May 23 2013 Bryn M. Reeves -- Always invoke tar with '-f-' option - -* Mon Jan 21 2013 Bryn M. Reeves -- Fix interactive mode regression when --ticket unspecified - -* Fri Jan 18 2013 Bryn M. Reeves -- Fix propagation of --ticket parameter in interactive mode - -* Thu Jan 17 2013 Bryn M. Reeves -- Revert OpenStack patch - -* Wed Jan 9 2013 Bryn M. Reeves -- Report --name and --ticket values as defaults -- Fix device-mapper command execution logging -- Fix data collection and rename PostreSQL module to pgsql - -* Fri Oct 19 2012 Bryn M. Reeves -- Add support for content delivery hosts to RHUI module - -* Thu Oct 18 2012 Bryn M. Reeves -- Add Red Hat Update Infrastructure module -- Collect /proc/iomem in hardware module -- Collect subscription-manager output in general module -- Collect rhsm log files in general module -- Fix exception in gluster module on non-gluster systems -- Fix exception in psql module when dbname is not given - -* Wed Oct 17 2012 Bryn M. Reeves -- Collect /proc/pagetypeinfo in memory module -- Strip trailing newline from command output -- Add sanlock module -- Do not collect archived accounting files in psacct module -- Call spacewalk-debug from rhn module to collect satellite data - -* Mon Oct 15 2012 Bryn M. Reeves -- Avoid calling volume status when collecting gluster statedumps -- Use a default report name if --name is empty -- Quote tilde characters passed to shell in RPM module -- Collect KDC and named configuration in ipa module -- Sanitize hostname characters before using as report path -- Collect /etc/multipath in device-mapper module -- New plug-in for PostgreSQL -- Add OpenStack module -- Avoid deprecated sysctls in /proc/sys/net -- Fix error logging when calling external programs -- Use ip instead of ifconfig to generate network interface lists - -* Wed May 23 2012 Bryn M. Reeves -- Collect the swift configuration directory in gluster module -- Update IPA module and related plug-ins - -* Fri May 18 2012 Bryn M. Reeves -- Collect mcelog files in the hardware module - -* Wed May 02 2012 Bryn M. Reeves -- Add nfs statedump collection to gluster module - -* Tue May 01 2012 Bryn M. Reeves -- Use wildcard to match possible libvirt log paths - -* Mon Apr 23 2012 Bryn M. Reeves -- Add forbidden paths for new location of gluster private keys - -* Fri Mar 9 2012 Bryn M. Reeves -- Fix katello and aeolus command string syntax -- Remove stray hunk from gluster module patch - -* Thu Mar 8 2012 Bryn M. Reeves -- Correct aeolus debug invocation in CloudForms module -- Update gluster module for gluster-3.3 -- Add additional command output to gluster module -- Add support for collecting gluster configuration and logs - -* Wed Mar 7 2012 Bryn M. Reeves -- Collect additional diagnostic information for realtime systems -- Improve sanitization of RHN user and case number in report name -- Fix verbose output and debug logging -- Add basic support for CloudForms data collection -- Add support for Subscription Asset Manager diagnostics - -* Tue Mar 6 2012 Bryn M. Reeves -- Collect fence_virt.conf in cluster module -- Fix collection of /proc/net directory tree -- Gather output of cpufreq-info when present -- Fix brctl showstp output when bridges contain multiple interfaces -- Add /etc/modprobe.d to kernel module -- Ensure relative symlink targets are correctly handled when copying -- Fix satellite and proxy package detection in rhn plugin -- Collect stderr output from external commands -- Collect /proc/cgroups in the cgroups module - Resolve: bz784874 -- Collect /proc/irq in the kernel module -- Fix installed-rpms formatting for long package names -- Add symbolic links for truncated log files -- Collect non-standard syslog and rsyslog log files -- Use correct paths for tomcat6 in RHN module -- Obscure root password if present in anacond-ks.cfg -- Do not accept embedded forward slashes in RHN usernames -- Add new sunrpc module to collect rpcinfo for gluster systems - -* Tue Nov 1 2011 Bryn M. Reeves -- Do not collect subscription manager keys in general plugin - -* Fri Sep 23 2011 Bryn M. Reeves -- Fix execution of RHN hardware.py from hardware plugin -- Fix hardware plugin to support new lsusb path - -* Fri Sep 09 2011 Bryn M. Reeves -- Fix brctl collection when a bridge contains no interfaces -- Fix up2dateclient path in hardware plugin - -* Mon Aug 15 2011 Bryn M. Reeves -- Collect brctl show and showstp output -- Collect nslcd.conf in ldap plugin - -* Sun Aug 14 2011 Bryn M. Reeves -- Truncate files that exceed specified size limit -- Add support for collecting Red Hat Subscrition Manager configuration -- Collect /etc/init on systems using upstart -- Don't strip whitespace from output of external programs -- Collect ipv6 neighbour table in network module -- Collect basic cgroups configuration data - -* Sat Aug 13 2011 Bryn M. Reeves -- Fix collection of data from LVM2 reporting tools in devicemapper plugin -- Add /proc/vmmemctl collection to vmware plugin - -* Fri Aug 12 2011 Bryn M. Reeves -- Collect yum repository list by default -- Add basic Infiniband plugin -- Add plugin for scsi-target-utils iSCSI target -- Fix autofs plugin LC_ALL usage -- Fix collection of lsusb and add collection of -t and -v outputs -- Extend data collection by qpidd plugin -- Add ethtool pause, coalesce and ring (-a, -c, -g) options to network plugin - -* Thu Apr 07 2011 Bryn M. Reeves -- Use sha256 for report digest when operating in FIPS mode - -* Tue Apr 05 2011 Bryn M. Reeves -- Fix parted and dumpe2fs output on s390 - -* Fri Feb 25 2011 Bryn M. Reeves -- Fix collection of chkconfig output in startup.py -- Collect /etc/dhcp in dhcp.py plugin -- Collect dmsetup ls --tree output in devicemapper.py -- Collect lsblk output in filesys.py - -* Thu Feb 24 2011 Bryn M. Reeves -- Fix collection of logs and config files in sssd.py -- Add support for collecting entitlement certificates in rhn.py - -* Thu Feb 03 2011 Bryn M. Reeves -- Fix cluster plugin dlm lockdump for el6 -- Add sssd plugin to collect configuration and logs -- Collect /etc/anacrontab in system plugin -- Correct handling of redhat-release for el6 - -* Thu Jul 29 2010 Adam Stokes - -* Thu Jun 10 2010 Adam Stokes - -* Wed Apr 28 2010 Adam Stokes - -* Mon Apr 12 2010 Adam Stokes - -* Tue Mar 30 2010 Adam Stokes -- fix setup.py to autocompile translations and man pages -- rebase 1.9 - -* Fri Mar 19 2010 Adam Stokes -- updated translations - -* Thu Mar 04 2010 Adam Stokes -- version bump 1.9 -- replaced compression utility with xz -- strip threading/multiprocessing -- simplified progress indicator -- pylint update -- put global vars in class container -- unittests -- simple profiling -- make use of xgettext as pygettext is deprecated - -* Mon Jan 18 2010 Adam Stokes -- more sanitizing options for log files -- rhbz fixes from RHEL version merged into trunk -- progressbar update -