diff --git a/.gitignore b/.gitignore index a247b61..bcb6c81 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -SOURCES/sos-4.2.tar.gz +SOURCES/sos-4.3.tar.gz SOURCES/sos-audit-0.3.tgz diff --git a/.sos.metadata b/.sos.metadata index 054c91f..74f14fe 100644 --- a/.sos.metadata +++ b/.sos.metadata @@ -1,2 +1,2 @@ -fe82967b0577076aac104412a9fe35cdb444bde4 SOURCES/sos-4.2.tar.gz +6d443271a3eb26af8fb400ed417a4b572730d316 SOURCES/sos-4.3.tar.gz 9d478b9f0085da9178af103078bbf2fd77b0175a SOURCES/sos-audit-0.3.tgz diff --git a/README.debrand b/README.debrand deleted file mode 100644 index 01c46d2..0000000 --- a/README.debrand +++ /dev/null @@ -1,2 +0,0 @@ -Warning: This package was configured for automatic debranding, but the changes -failed to apply. diff --git a/SOURCES/sos-bz1869561-cpuX-individual-sizelimits.patch b/SOURCES/sos-bz1869561-cpuX-individual-sizelimits.patch deleted file mode 100644 index 4d579d7..0000000 --- a/SOURCES/sos-bz1869561-cpuX-individual-sizelimits.patch +++ /dev/null @@ -1,48 +0,0 @@ -From b09ed75b09075d86c184b0a63cce9260f2cee4ca Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 30 Aug 2021 11:27:48 +0200 -Subject: [PATCH] [processor] Apply sizelimit to /sys/devices/system/cpu/cpuX - -Copy /sys/devices/system/cpu/cpuX with separately applied sizelimit. - -This is required for systems with tens/hundreds of CPUs where the -cumulative directory size exceeds 25MB or even 100MB. - -Resolves: #2639 -Closes: #2665 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/processor.py | 9 ++++++++- - 1 file changed, 8 insertions(+), 1 deletion(-) - -diff --git a/sos/report/plugins/processor.py b/sos/report/plugins/processor.py -index 0ddfd126..2df2dc9a 100644 ---- a/sos/report/plugins/processor.py -+++ b/sos/report/plugins/processor.py -@@ -7,6 +7,7 @@ - # See the LICENSE file in the source distribution for further information. - - from sos.report.plugins import Plugin, IndependentPlugin -+import os - - - class Processor(Plugin, IndependentPlugin): -@@ -34,7 +35,13 @@ class Processor(Plugin, IndependentPlugin): - self.add_copy_spec([ - "/proc/cpuinfo", - "/sys/class/cpuid", -- "/sys/devices/system/cpu" -+ ]) -+ # copy /sys/devices/system/cpu/cpuX with separately applied sizelimit -+ # this is required for systems with tens/hundreds of CPUs where the -+ # cumulative directory size exceeds 25MB or even 100MB. -+ cdirs = self.listdir('/sys/devices/system/cpu') -+ self.add_copy_spec([ -+ os.path.join('/sys/devices/system/cpu', cdir) for cdir in cdirs - ]) - - self.add_cmd_output([ --- -2.31.1 - diff --git a/SOURCES/sos-bz2011507-foreman-puma-status.patch b/SOURCES/sos-bz2011507-foreman-puma-status.patch deleted file mode 100644 index 2a80571..0000000 --- a/SOURCES/sos-bz2011507-foreman-puma-status.patch +++ /dev/null @@ -1,69 +0,0 @@ -From 5a9458d318302c1caef862a868745fc8bdf5c741 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 4 Oct 2021 15:52:36 +0200 -Subject: [PATCH] [foreman] Collect puma status and stats - -Collect foreman-puma-status and 'pumactl [gc-|]stats', optionally using -SCL (if detected). - -Resolves: #2712 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/foreman.py | 21 ++++++++++++++++++++- - 1 file changed, 20 insertions(+), 1 deletion(-) - -diff --git a/sos/report/plugins/foreman.py b/sos/report/plugins/foreman.py -index 4539f12b..351794f4 100644 ---- a/sos/report/plugins/foreman.py -+++ b/sos/report/plugins/foreman.py -@@ -13,6 +13,7 @@ from sos.report.plugins import (Plugin, - UbuntuPlugin) - from pipes import quote - from re import match -+from sos.utilities import is_executable - - - class Foreman(Plugin): -@@ -26,7 +27,9 @@ class Foreman(Plugin): - option_list = [ - ('months', 'number of months for dynflow output', 'fast', 1), - ('proxyfeatures', 'collect features of smart proxies', 'slow', False), -+ ('puma-gc', 'collect Puma GC stats', 'fast', False), - ] -+ pumactl = 'pumactl %s -S /usr/share/foreman/tmp/puma.state' - - def setup(self): - # for external DB, search in /etc/foreman/database.yml for: -@@ -134,6 +138,17 @@ class Foreman(Plugin): - suggest_filename='dynflow_sidekiq_status') - self.add_journal(units="dynflow-sidekiq@*") - -+ # Puma stats & status, i.e. foreman-puma-stats, then -+ # pumactl stats -S /usr/share/foreman/tmp/puma.state -+ # and optionally also gc-stats -+ # if on RHEL with Software Collections, wrap the commands accordingly -+ if self.get_option('puma-gc'): -+ self.add_cmd_output(self.pumactl % 'gc-stats', -+ suggest_filename='pumactl_gc-stats') -+ self.add_cmd_output(self.pumactl % 'stats', -+ suggest_filename='pumactl_stats') -+ self.add_cmd_output('/usr/sbin/foreman-puma-status') -+ - # collect tables sizes, ordered - _cmd = self.build_query_cmd( - "SELECT table_name, pg_size_pretty(total_bytes) AS total, " -@@ -297,6 +312,10 @@ class RedHatForeman(Foreman, RedHatPlugin): - self.add_file_tags({ - '/usr/share/foreman/.ssh/ssh_config': 'ssh_foreman_config', - }) -+ # if we are on RHEL7 with scl, wrap some Puma commands by -+ # scl enable tfm 'command' -+ if self.policy.dist_version() == 7 and is_executable('scl'): -+ self.pumactl = "scl enable tfm '%s'" % self.pumactl - - super(RedHatForeman, self).setup() - --- -2.31.1 - diff --git a/SOURCES/sos-bz2011533-unpackaged-recursive-symlink.patch b/SOURCES/sos-bz2011533-unpackaged-recursive-symlink.patch deleted file mode 100644 index 35cc89d..0000000 --- a/SOURCES/sos-bz2011533-unpackaged-recursive-symlink.patch +++ /dev/null @@ -1,42 +0,0 @@ -From e2ca3d02f36c0db4efaacfb2c1b7d502f38e371c Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 30 Aug 2021 10:18:29 +0200 -Subject: [PATCH] [unpackaged] deal with recursive loop of symlinks properly - -When the plugin processes a recursive loop of symlinks, it currently -hangs in an infinite loop trying to follow the symlinks. Use -pathlib.Path.resolve() method to return the target directly. - -Resolves: #2664 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/unpackaged.py | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) - -diff --git a/sos/report/plugins/unpackaged.py b/sos/report/plugins/unpackaged.py -index e5cc6191..9d68077c 100644 ---- a/sos/report/plugins/unpackaged.py -+++ b/sos/report/plugins/unpackaged.py -@@ -10,6 +10,7 @@ from sos.report.plugins import Plugin, RedHatPlugin - - import os - import stat -+from pathlib import Path - - - class Unpackaged(Plugin, RedHatPlugin): -@@ -41,8 +42,8 @@ class Unpackaged(Plugin, RedHatPlugin): - for name in files: - path = os.path.join(root, name) - try: -- while stat.S_ISLNK(os.lstat(path).st_mode): -- path = os.path.abspath(os.readlink(path)) -+ if stat.S_ISLNK(os.lstat(path).st_mode): -+ path = Path(path).resolve() - except Exception: - continue - file_list.append(os.path.realpath(path)) --- -2.31.1 - diff --git a/SOURCES/sos-bz2011534-opacapture-under-allow-system-changes.patch b/SOURCES/sos-bz2011534-opacapture-under-allow-system-changes.patch deleted file mode 100644 index 39f9c8a..0000000 --- a/SOURCES/sos-bz2011534-opacapture-under-allow-system-changes.patch +++ /dev/null @@ -1,49 +0,0 @@ -From 66ebb8256b1326573cbcb2d134545635dfead3bc Mon Sep 17 00:00:00 2001 -From: Jose Castillo -Date: Sun, 29 Aug 2021 15:35:09 +0200 -Subject: [PATCH] [omnipath_client] Ensure opacapture runs only with - allow-system-changes - -While omnipath_client plugin is collecting "opacapture", -`depmod -a` command is executed to regenerates some files -under /usr/lib/modules/$kernel. - -modules.dep -modules.dep.bin -modules.devname -modules.softdep -modules.symbols -modules.symbols.bin - -This patch ensures that the command is only run when -the option --allow-system-changes is used. - -Fixes: RHBZ#1998433 - -Signed-off-by: Jose Castillo ---- - sos/report/plugins/omnipath_client.py | 9 +++++++-- - 1 file changed, 7 insertions(+), 2 deletions(-) - -diff --git a/sos/report/plugins/omnipath_client.py b/sos/report/plugins/omnipath_client.py -index 1ec01384..4e988c5c 100644 ---- a/sos/report/plugins/omnipath_client.py -+++ b/sos/report/plugins/omnipath_client.py -@@ -45,7 +45,12 @@ class OmnipathClient(Plugin, RedHatPlugin): - # rather than storing it somewhere under /var/tmp and copying it via - # add_copy_spec, add it directly to sos_commands/ dir by - # building a path argument using self.get_cmd_output_path(). -- self.add_cmd_output("opacapture %s" % join(self.get_cmd_output_path(), -- "opacapture.tgz")) -+ # This command calls 'depmod -a', so lets make sure we -+ # specified the 'allow-system-changes' option before running it. -+ if self.get_option('allow_system_changes'): -+ self.add_cmd_output("opacapture %s" % -+ join(self.get_cmd_output_path(), -+ "opacapture.tgz"), -+ changes=True) - - # vim: set et ts=4 sw=4 : --- -2.31.1 - diff --git a/SOURCES/sos-bz2011535-kernel-psi.patch b/SOURCES/sos-bz2011535-kernel-psi.patch deleted file mode 100644 index 1a9d5e0..0000000 --- a/SOURCES/sos-bz2011535-kernel-psi.patch +++ /dev/null @@ -1,33 +0,0 @@ -From 23e523b6b9784390c7ce2c5af654ab497fb10aaf Mon Sep 17 00:00:00 2001 -From: Jose Castillo -Date: Wed, 8 Sep 2021 09:25:24 +0200 -Subject: [PATCH] [kernel] Capture Pressure Stall Information - -Kernel 4.20 includes PSI metrics for CPU, memeory and IO. -The feature is enabled after adding "psi=1" as -kernel boot parameter. -The information is captured in files -in the directory /proc/pressure. - -Signed-off-by: Jose Castillo ---- - sos/report/plugins/kernel.py | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/sos/report/plugins/kernel.py b/sos/report/plugins/kernel.py -index 8c5e5e11..803f5e30 100644 ---- a/sos/report/plugins/kernel.py -+++ b/sos/report/plugins/kernel.py -@@ -112,7 +112,8 @@ class Kernel(Plugin, IndependentPlugin): - "/sys/kernel/debug/extfrag/unusable_index", - "/sys/kernel/debug/extfrag/extfrag_index", - clocksource_path + "available_clocksource", -- clocksource_path + "current_clocksource" -+ clocksource_path + "current_clocksource", -+ "/proc/pressure/" - ]) - - if self.get_option("with-timer"): --- -2.31.1 - diff --git a/SOURCES/sos-bz2011536-iptables-based-on-ntf.patch b/SOURCES/sos-bz2011536-iptables-based-on-ntf.patch deleted file mode 100644 index 5ccc61f..0000000 --- a/SOURCES/sos-bz2011536-iptables-based-on-ntf.patch +++ /dev/null @@ -1,303 +0,0 @@ -From 2ab8ba3ecbd52e452cc554d515e0782801dcb4b6 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Wed, 8 Sep 2021 15:31:48 +0200 -Subject: [PATCH] [firewalld] collect nft rules in firewall_tables only - -We collect 'nft list ruleset' in both plugins, while: -- nft is not shipped by firewalld package, so we should not collect -it in firewalld plugin -- running the command requires both nf_tables and nfnetlink kmods, so -we should use both kmods in the predicate - -Resolves: #2679 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/firewall_tables.py | 9 +++++---- - sos/report/plugins/firewalld.py | 8 +------- - 2 files changed, 6 insertions(+), 11 deletions(-) - -diff --git a/sos/report/plugins/firewall_tables.py b/sos/report/plugins/firewall_tables.py -index 56058d3bf9..63a7dddeb5 100644 ---- a/sos/report/plugins/firewall_tables.py -+++ b/sos/report/plugins/firewall_tables.py -@@ -40,10 +40,11 @@ def collect_nftables(self): - """ Collects nftables rulesets with 'nft' commands if the modules - are present """ - -- self.add_cmd_output( -- "nft list ruleset", -- pred=SoSPredicate(self, kmods=['nf_tables']) -- ) -+ # collect nftables ruleset -+ nft_pred = SoSPredicate(self, -+ kmods=['nf_tables', 'nfnetlink'], -+ required={'kmods': 'all'}) -+ self.add_cmd_output("nft list ruleset", pred=nft_pred, changes=True) - - def setup(self): - # collect iptables -t for any existing table, if we can't read the -diff --git a/sos/report/plugins/firewalld.py b/sos/report/plugins/firewalld.py -index ec83527ed7..9401bfd239 100644 ---- a/sos/report/plugins/firewalld.py -+++ b/sos/report/plugins/firewalld.py -@@ -9,7 +9,7 @@ - # - # See the LICENSE file in the source distribution for further information. - --from sos.report.plugins import Plugin, RedHatPlugin, SoSPredicate -+from sos.report.plugins import Plugin, RedHatPlugin - - - class FirewallD(Plugin, RedHatPlugin): -@@ -35,12 +35,6 @@ def setup(self): - "/var/log/firewalld", - ]) - -- # collect nftables ruleset -- nft_pred = SoSPredicate(self, -- kmods=['nf_tables', 'nfnetlink'], -- required={'kmods': 'all'}) -- self.add_cmd_output("nft list ruleset", pred=nft_pred, changes=True) -- - # use a 10s timeout to workaround dbus problems in - # docker containers. - self.add_cmd_output([ --- -2.31.1 - - -From 2a7cf53b61943907dc823cf893530b620a87946c Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Fri, 15 Oct 2021 22:31:36 +0200 -Subject: [PATCH 1/3] [report] Use log_skipped_cmd method inside - collect_cmd_output - -Also, remove obsolete parameters of the log_skipped_cmd method. - -Related: #2724 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/__init__.py | 26 ++++++++------------------ - 1 file changed, 8 insertions(+), 18 deletions(-) - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index ec138f83..b60ab5f6 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -876,8 +876,7 @@ class Plugin(): - return bool(pred) - return False - -- def log_skipped_cmd(self, pred, cmd, kmods=False, services=False, -- changes=False): -+ def log_skipped_cmd(self, cmd, pred, changes=False): - """Log that a command was skipped due to predicate evaluation. - - Emit a warning message indicating that a command was skipped due -@@ -887,21 +886,17 @@ class Plugin(): - message indicating that the missing data can be collected by using - the "--allow-system-changes" command line option will be included. - -- :param pred: The predicate that caused the command to be skipped -- :type pred: ``SoSPredicate`` -- - :param cmd: The command that was skipped - :type cmd: ``str`` - -- :param kmods: Did kernel modules cause the command to be skipped -- :type kmods: ``bool`` -- -- :param services: Did services cause the command to be skipped -- :type services: ``bool`` -+ :param pred: The predicate that caused the command to be skipped -+ :type pred: ``SoSPredicate`` - - :param changes: Is the `--allow-system-changes` enabled - :type changes: ``bool`` - """ -+ if pred is None: -+ pred = SoSPredicate(self) - msg = "skipped command '%s': %s" % (cmd, pred.report_failure()) - - if changes: -@@ -1700,9 +1693,7 @@ class Plugin(): - self.collect_cmds.append(soscmd) - self._log_info("added cmd output '%s'" % soscmd.cmd) - else: -- self.log_skipped_cmd(pred, soscmd.cmd, kmods=bool(pred.kmods), -- services=bool(pred.services), -- changes=soscmd.changes) -+ self.log_skipped_cmd(soscmd.cmd, pred, changes=soscmd.changes) - - def add_cmd_output(self, cmds, suggest_filename=None, - root_symlink=None, timeout=None, stderr=True, -@@ -2112,7 +2103,7 @@ class Plugin(): - root_symlink=False, timeout=None, - stderr=True, chroot=True, runat=None, env=None, - binary=False, sizelimit=None, pred=None, -- subdir=None, tags=[]): -+ changes=False, subdir=None, tags=[]): - """Execute a command and save the output to a file for inclusion in the - report, then return the results for further use by the plugin - -@@ -2163,8 +2154,7 @@ class Plugin(): - :rtype: ``dict`` - """ - if not self.test_predicate(cmd=True, pred=pred): -- self._log_info("skipped cmd output '%s' due to predicate (%s)" % -- (cmd, self.get_predicate(cmd=True, pred=pred))) -+ self.log_skipped_cmd(cmd, pred, changes=changes) - return { - 'status': None, # don't match on if result['status'] checks - 'output': '', --- -2.31.1 - - -From 6b1bea0ffb1df7f8e5001b06cf25f0741b007ddd Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Fri, 15 Oct 2021 22:34:01 +0200 -Subject: [PATCH 2/3] [firewall_tables] call iptables -t based on nft - list - -If iptables are not realy in use, calling iptables -t
-would load corresponding nft table. - -Therefore, call iptables -t only for the tables from "nft list ruleset" -output. - -Example: nft list ruleset contains - -table ip mangle { -.. -} - -so we can collect iptable -t mangle -nvL . - -The same applies to ip6tables as well. - -Resolves: #2724 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/firewall_tables.py | 29 ++++++++++++++++++++------- - 1 file changed, 22 insertions(+), 7 deletions(-) - -diff --git a/sos/report/plugins/firewall_tables.py b/sos/report/plugins/firewall_tables.py -index 63a7ddde..ef04d939 100644 ---- a/sos/report/plugins/firewall_tables.py -+++ b/sos/report/plugins/firewall_tables.py -@@ -44,26 +44,41 @@ class firewall_tables(Plugin, IndependentPlugin): - nft_pred = SoSPredicate(self, - kmods=['nf_tables', 'nfnetlink'], - required={'kmods': 'all'}) -- self.add_cmd_output("nft list ruleset", pred=nft_pred, changes=True) -+ return self.collect_cmd_output("nft list ruleset", pred=nft_pred, -+ changes=True) - - def setup(self): -+ # first, collect "nft list ruleset" as collecting commands like -+ # ip6tables -t mangle -nvL -+ # depends on its output -+ # store in nft_ip_tables lists of ip[|6] tables from nft list -+ nft_list = self.collect_nftables() -+ nft_ip_tables = {'ip': [], 'ip6': []} -+ nft_lines = nft_list['output'] if nft_list['status'] == 0 else '' -+ for line in nft_lines.splitlines(): -+ words = line.split()[0:3] -+ if len(words) == 3 and words[0] == 'table' and \ -+ words[1] in nft_ip_tables.keys(): -+ nft_ip_tables[words[1]].append(words[2]) - # collect iptables -t for any existing table, if we can't read the - # tables, collect 2 default ones (mangle, filter) -+ # do collect them only when relevant nft list ruleset exists -+ default_ip_tables = "mangle\nfilter\n" - try: - ip_tables_names = open("/proc/net/ip_tables_names").read() - except IOError: -- ip_tables_names = "mangle\nfilter\n" -+ ip_tables_names = default_ip_tables - for table in ip_tables_names.splitlines(): -- self.collect_iptable(table) -+ if nft_list['status'] == 0 and table in nft_ip_tables['ip']: -+ self.collect_iptable(table) - # collect the same for ip6tables - try: - ip_tables_names = open("/proc/net/ip6_tables_names").read() - except IOError: -- ip_tables_names = "mangle\nfilter\n" -+ ip_tables_names = default_ip_tables - for table in ip_tables_names.splitlines(): -- self.collect_ip6table(table) -- -- self.collect_nftables() -+ if nft_list['status'] == 0 and table in nft_ip_tables['ip6']: -+ self.collect_ip6table(table) - - # When iptables is called it will load the modules - # iptables_filter (for kernel <= 3) or --- -2.31.1 - - -From 464bd2d2e83f203e369f2ba7671bbb7da53e06f6 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Sun, 24 Oct 2021 16:00:31 +0200 -Subject: [PATCH 3/3] [firewall_tables] Call iptables only when nft ip filter - table exists - -iptables -vnxL creates nft 'ip filter' table if it does not exist, hence -we must guard iptables execution by presence of the nft table. - -An equivalent logic applies to ip6tables. - -Resolves: #2724 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/firewall_tables.py | 26 ++++++++++++++------------ - 1 file changed, 14 insertions(+), 12 deletions(-) - -diff --git a/sos/report/plugins/firewall_tables.py b/sos/report/plugins/firewall_tables.py -index ef04d939..7eafd60f 100644 ---- a/sos/report/plugins/firewall_tables.py -+++ b/sos/report/plugins/firewall_tables.py -@@ -80,19 +80,21 @@ class firewall_tables(Plugin, IndependentPlugin): - if nft_list['status'] == 0 and table in nft_ip_tables['ip6']: - self.collect_ip6table(table) - -- # When iptables is called it will load the modules -- # iptables_filter (for kernel <= 3) or -- # nf_tables (for kernel >= 4) if they are not loaded. -+ # When iptables is called it will load: -+ # 1) the modules iptables_filter (for kernel <= 3) or -+ # nf_tables (for kernel >= 4) if they are not loaded. -+ # 2) nft 'ip filter' table will be created - # The same goes for ipv6. -- self.add_cmd_output( -- "iptables -vnxL", -- pred=SoSPredicate(self, kmods=['iptable_filter', 'nf_tables']) -- ) -- -- self.add_cmd_output( -- "ip6tables -vnxL", -- pred=SoSPredicate(self, kmods=['ip6table_filter', 'nf_tables']) -- ) -+ if nft_list['status'] != 0 or 'filter' in nft_ip_tables['ip']: -+ self.add_cmd_output( -+ "iptables -vnxL", -+ pred=SoSPredicate(self, kmods=['iptable_filter', 'nf_tables']) -+ ) -+ if nft_list['status'] != 0 or 'filter' in nft_ip_tables['ip6']: -+ self.add_cmd_output( -+ "ip6tables -vnxL", -+ pred=SoSPredicate(self, kmods=['ip6table_filter', 'nf_tables']) -+ ) - - self.add_copy_spec([ - "/etc/nftables", --- -2.31.1 - diff --git a/SOURCES/sos-bz2011537-estimate-only-option.patch b/SOURCES/sos-bz2011537-estimate-only-option.patch deleted file mode 100644 index a1a96c4..0000000 --- a/SOURCES/sos-bz2011537-estimate-only-option.patch +++ /dev/null @@ -1,1316 +0,0 @@ -From 5b245b1e449c6a05d09034bcb8290bffded79327 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Wed, 8 Sep 2021 17:04:58 +0200 -Subject: [PATCH] [report] Implement --estimate-only - -Add report option --estimate-only to estimate disk space requirements -when running a sos report. - -Resolves: #2673 - -Signed-off-by: Pavel Moravec ---- - man/en/sos-report.1 | 13 +++++++- - sos/report/__init__.py | 74 ++++++++++++++++++++++++++++++++++++++++-- - 2 files changed, 84 insertions(+), 3 deletions(-) - -diff --git a/man/en/sos-report.1 b/man/en/sos-report.1 -index 36b337df..e8efc8f8 100644 ---- a/man/en/sos-report.1 -+++ b/man/en/sos-report.1 -@@ -14,7 +14,7 @@ sos report \- Collect and package diagnostic and support data - [--preset preset] [--add-preset add_preset]\fR - [--del-preset del_preset] [--desc description]\fR - [--batch] [--build] [--debug] [--dry-run]\fR -- [--label label] [--case-id id]\fR -+ [--estimate-only] [--label label] [--case-id id]\fR - [--threads threads]\fR - [--plugin-timeout TIMEOUT]\fR - [--cmd-timeout TIMEOUT]\fR -@@ -317,6 +317,17 @@ output, or string data from the system. The resulting logs may be used - to understand the actions that sos would have taken without the dry run - option. - .TP -+.B \--estimate-only -+Estimate disk space requirements when running sos report. This can be valuable -+to prevent sosreport working dir to consume all free disk space. No plugin data -+is available at the end. -+ -+Plugins will be collected sequentially, size of collected files and commands outputs -+will be calculated and the plugin files will be immediatelly deleted prior execution -+of the next plugin. This still can consume whole free disk space, though. Please note, -+size estimations may not be accurate for highly utilized systems due to changes between -+an estimate and a real execution. -+.TP - .B \--upload - If specified, attempt to upload the resulting archive to a vendor defined location. - -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index 82484f1d..b033f621 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -86,6 +86,7 @@ class SoSReport(SoSComponent): - 'desc': '', - 'domains': [], - 'dry_run': False, -+ 'estimate_only': False, - 'experimental': False, - 'enable_plugins': [], - 'keywords': [], -@@ -137,6 +138,7 @@ class SoSReport(SoSComponent): - self._args = args - self.sysroot = "/" - self.preset = None -+ self.estimated_plugsizes = {} - - self.print_header() - self._set_debug() -@@ -223,6 +225,11 @@ class SoSReport(SoSComponent): - help="Description for a new preset",) - report_grp.add_argument("--dry-run", action="store_true", - help="Run plugins but do not collect data") -+ report_grp.add_argument("--estimate-only", action="store_true", -+ help="Approximate disk space requirements for " -+ "a real sos run; disables --clean and " -+ "--collect, sets --threads=1 and " -+ "--no-postproc") - report_grp.add_argument("--experimental", action="store_true", - dest="experimental", default=False, - help="enable experimental plugins") -@@ -700,6 +700,33 @@ class SoSReport(SoSComponent): - self.all_options.append((plugin, plugin_name, optname, - optparm)) - -+ def _set_estimate_only(self): -+ # set estimate-only mode by enforcing some options settings -+ # and return a corresponding log messages string -+ msg = "\nEstimate-only mode enabled" -+ ext_msg = [] -+ if self.opts.threads > 1: -+ ext_msg += ["--threads=%s overriden to 1" % self.opts.threads, ] -+ self.opts.threads = 1 -+ if not self.opts.build: -+ ext_msg += ["--build enabled", ] -+ self.opts.build = True -+ if not self.opts.no_postproc: -+ ext_msg += ["--no-postproc enabled", ] -+ self.opts.no_postproc = True -+ if self.opts.clean: -+ ext_msg += ["--clean disabled", ] -+ self.opts.clean = False -+ if self.opts.upload: -+ ext_msg += ["--upload* options disabled", ] -+ self.opts.upload = False -+ if ext_msg: -+ msg += ", which overrides some options:\n " + "\n ".join(ext_msg) -+ else: -+ msg += "." -+ msg += "\n\n" -+ return msg -+ - def _report_profiles_and_plugins(self): - self.ui_log.info("") - if len(self.loaded_plugins): -@@ -875,10 +909,12 @@ class SoSReport(SoSComponent): - return True - - def batch(self): -+ msg = self.policy.get_msg() -+ if self.opts.estimate_only: -+ msg += self._set_estimate_only() - if self.opts.batch: -- self.ui_log.info(self.policy.get_msg()) -+ self.ui_log.info(msg) - else: -- msg = self.policy.get_msg() - msg += _("Press ENTER to continue, or CTRL-C to quit.\n") - try: - input(msg) -@@ -1011,6 +1047,22 @@ class SoSReport(SoSComponent): - self.running_plugs.remove(plugin[1]) - self.loaded_plugins[plugin[0]-1][1].set_timeout_hit() - pool._threads.clear() -+ if self.opts.estimate_only: -+ from pathlib import Path -+ tmpdir_path = Path(self.archive.get_tmp_dir()) -+ self.estimated_plugsizes[plugin[1]] = sum( -+ [f.stat().st_size for f in tmpdir_path.glob('**/*') -+ if (os.path.isfile(f) and not os.path.islink(f))]) -+ # remove whole tmp_dir content - including "sos_commands" and -+ # similar dirs that will be re-created on demand by next plugin -+ # if needed; it is less error-prone approach than skipping -+ # deletion of some dirs but deleting their content -+ for f in os.listdir(self.archive.get_tmp_dir()): -+ f = os.path.join(self.archive.get_tmp_dir(), f) -+ if os.path.isdir(f): -+ rmtree(f) -+ else: -+ os.unlink(f) - return True - - def collect_plugin(self, plugin): -@@ -1330,6 +1382,24 @@ class SoSReport(SoSComponent): - self.policy.display_results(archive, directory, checksum, - map_file=map_file) - -+ if self.opts.estimate_only: -+ from sos.utilities import get_human_readable -+ _sum = get_human_readable(sum(self.estimated_plugsizes.values())) -+ self.ui_log.info("Estimated disk space requirement for whole " -+ "uncompressed sos report directory: %s" % _sum) -+ bigplugins = sorted(self.estimated_plugsizes.items(), -+ key=lambda x: x[1], reverse=True)[:3] -+ bp_out = ", ".join("%s: %s" % -+ (p, get_human_readable(v, precision=0)) -+ for p, v in bigplugins) -+ self.ui_log.info("Three biggest plugins: %s" % bp_out) -+ self.ui_log.info("") -+ self.ui_log.info("Please note the estimation is relevant to the " -+ "current options.") -+ self.ui_log.info("Be aware that the real disk space requirements " -+ "might be different.") -+ self.ui_log.info("") -+ - if self.opts.upload or self.opts.upload_url: - if not self.opts.build: - try: --- -2.31.1 - -From 7ae47e6c0717c0b56c3368008dd99a87f7f436d5 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Wed, 13 Oct 2021 20:21:16 +0200 -Subject: [PATCH] [report] Count with sos_logs and sos_reports in - --estimate-only - -Currently, we estimate just plugins' disk space and ignore sos_logs -or sos_reports directories - although they can occupy nontrivial disk -space as well. - -Resolves: #2723 - -Signed-off-by: Pavel Moravec ---- - sos/report/__init__.py | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index e35c7e8d..7feb31ee 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -1380,6 +1380,14 @@ class SoSReport(SoSComponent): - - if self.opts.estimate_only: - from sos.utilities import get_human_readable -+ from pathlib import Path -+ # add sos_logs, sos_reports dirs, etc., basically everything -+ # that remained in self.tmpdir after plugins' contents removal -+ # that still will be moved to the sos report final directory path -+ tmpdir_path = Path(self.tmpdir) -+ self.estimated_plugsizes['sos_logs_reports'] = sum( -+ [f.stat().st_size for f in tmpdir_path.glob('**/*')]) -+ - _sum = get_human_readable(sum(self.estimated_plugsizes.values())) - self.ui_log.info("Estimated disk space requirement for whole " - "uncompressed sos report directory: %s" % _sum) --- -2.31.1 - -From 4293f3317505661e8f32ba94ad87310996fa1626 Mon Sep 17 00:00:00 2001 -From: Eric Desrochers -Date: Tue, 19 Oct 2021 12:18:40 -0400 -Subject: [PATCH] [report] check for symlink before rmtree when opt - estimate-only is use - -Check if the dir is also symlink before performing rmtree() -method so that unlink() method can be used instead. - -Traceback (most recent call last): - File "./bin/sos", line 22, in - sos.execute() - File "/tmp/sos/sos/__init__.py", line 186, in execute - self._component.execute() -OSError: Cannot call rmtree on a symbolic link - -Closes: #2727 - -Signed-off-by: Eric Desrochers ---- - sos/report/__init__.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index 7feb31ee..1b5bc97d 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -1059,7 +1059,7 @@ class SoSReport(SoSComponent): - # deletion of some dirs but deleting their content - for f in os.listdir(self.archive.get_tmp_dir()): - f = os.path.join(self.archive.get_tmp_dir(), f) -- if os.path.isdir(f): -+ if os.path.isdir(f) and not os.path.islink(f): - rmtree(f) - else: - os.unlink(f) --- -2.31.1 - -From 589d47c93257b55bc796ef6ac25b88c974ee3d72 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 8 Nov 2021 16:38:24 +0100 -Subject: [PATCH] [report] Calculate sizes of dirs, symlinks and manifest in - estimate mode - -Enhance --estimate-mode to calculate sizes of also: -- symlinks -- directories themselves -- manifest.json file - -Use os.lstat() method instead of os.stat() to properly calculate the -sizes (and not destinations of symlinks, e.g.). - -Print five biggest plugins instead of three as sos logs and reports do -stand as one "plugin" in the list, often. - -Resolves: #2752 - -Signed-off-by: Pavel Moravec ---- - sos/report/__init__.py | 56 +++++++++++++++++++++--------------------- - 1 file changed, 28 insertions(+), 28 deletions(-) - -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index 10952566..a4c92acc 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -1050,8 +1050,7 @@ class SoSReport(SoSComponent): - from pathlib import Path - tmpdir_path = Path(self.archive.get_tmp_dir()) - self.estimated_plugsizes[plugin[1]] = sum( -- [f.stat().st_size for f in tmpdir_path.glob('**/*') -- if (os.path.isfile(f) and not os.path.islink(f))]) -+ [f.lstat().st_size for f in tmpdir_path.glob('**/*')]) - # remove whole tmp_dir content - including "sos_commands" and - # similar dirs that will be re-created on demand by next plugin - # if needed; it is less error-prone approach than skipping -@@ -1273,6 +1272,33 @@ class SoSReport(SoSComponent): - short_name='manifest.json' - ) - -+ # print results in estimate mode (to include also just added manifest) -+ if self.opts.estimate_only: -+ from sos.utilities import get_human_readable -+ from pathlib import Path -+ # add sos_logs, sos_reports dirs, etc., basically everything -+ # that remained in self.tmpdir after plugins' contents removal -+ # that still will be moved to the sos report final directory path -+ tmpdir_path = Path(self.tmpdir) -+ self.estimated_plugsizes['sos_logs_reports'] = sum( -+ [f.lstat().st_size for f in tmpdir_path.glob('**/*')]) -+ -+ _sum = get_human_readable(sum(self.estimated_plugsizes.values())) -+ self.ui_log.info("Estimated disk space requirement for whole " -+ "uncompressed sos report directory: %s" % _sum) -+ bigplugins = sorted(self.estimated_plugsizes.items(), -+ key=lambda x: x[1], reverse=True)[:5] -+ bp_out = ", ".join("%s: %s" % -+ (p, get_human_readable(v, precision=0)) -+ for p, v in bigplugins) -+ self.ui_log.info("Five biggest plugins: %s" % bp_out) -+ self.ui_log.info("") -+ self.ui_log.info("Please note the estimation is relevant to the " -+ "current options.") -+ self.ui_log.info("Be aware that the real disk space requirements " -+ "might be different.") -+ self.ui_log.info("") -+ - # package up and compress the results - if not self.opts.build: - old_umask = os.umask(0o077) -@@ -1377,32 +1403,6 @@ class SoSReport(SoSComponent): - self.policy.display_results(archive, directory, checksum, - map_file=map_file) - -- if self.opts.estimate_only: -- from sos.utilities import get_human_readable -- from pathlib import Path -- # add sos_logs, sos_reports dirs, etc., basically everything -- # that remained in self.tmpdir after plugins' contents removal -- # that still will be moved to the sos report final directory path -- tmpdir_path = Path(self.tmpdir) -- self.estimated_plugsizes['sos_logs_reports'] = sum( -- [f.stat().st_size for f in tmpdir_path.glob('**/*')]) -- -- _sum = get_human_readable(sum(self.estimated_plugsizes.values())) -- self.ui_log.info("Estimated disk space requirement for whole " -- "uncompressed sos report directory: %s" % _sum) -- bigplugins = sorted(self.estimated_plugsizes.items(), -- key=lambda x: x[1], reverse=True)[:3] -- bp_out = ", ".join("%s: %s" % -- (p, get_human_readable(v, precision=0)) -- for p, v in bigplugins) -- self.ui_log.info("Three biggest plugins: %s" % bp_out) -- self.ui_log.info("") -- self.ui_log.info("Please note the estimation is relevant to the " -- "current options.") -- self.ui_log.info("Be aware that the real disk space requirements " -- "might be different.") -- self.ui_log.info("") -- - if self.opts.upload or self.opts.upload_url: - if not self.opts.build: - try: --- -2.31.1 - -From c6a5bbb8d75aadd5c7f76d3f469929aba2cf8060 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Wed, 5 Jan 2022 10:33:58 +0100 -Subject: [PATCH] [report] Provide better warning about estimate-mode - -As --estimate-only calculates disk usage based on `stat` data that -differs from outputs of other commands like `du`, enhance the warning -about reliability of the calculated estimation. - -Also add a rule-of-thumb recommendation of real disk space requirements. - -Resolves: #2815 - -Signed-off-by: Pavel Moravec ---- - man/en/sos-report.1 | 10 +++++++--- - sos/report/__init__.py | 3 ++- - 2 files changed, 9 insertions(+), 4 deletions(-) - -diff --git a/man/en/sos-report.1 b/man/en/sos-report.1 -index 464a77e54..e34773986 100644 ---- a/man/en/sos-report.1 -+++ b/man/en/sos-report.1 -@@ -343,9 +343,13 @@ is available at the end. - - Plugins will be collected sequentially, size of collected files and commands outputs - will be calculated and the plugin files will be immediatelly deleted prior execution --of the next plugin. This still can consume whole free disk space, though. Please note, --size estimations may not be accurate for highly utilized systems due to changes between --an estimate and a real execution. -+of the next plugin. This still can consume whole free disk space, though. -+ -+Please note, size estimations may not be accurate for highly utilized systems due to -+changes between an estimate and a real execution. Also some difference between -+estimation (using `stat` command) and other commands used (i.e. `du`). -+ -+A rule of thumb is to reserve at least double the estimation. - .TP - .B \--upload - If specified, attempt to upload the resulting archive to a vendor defined location. -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index ef61fb344..e0617b45e 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -1330,7 +1330,8 @@ def final_work(self): - self.ui_log.info("Please note the estimation is relevant to the " - "current options.") - self.ui_log.info("Be aware that the real disk space requirements " -- "might be different.") -+ "might be different. A rule of thumb is to " -+ "reserve at least double the estimation.") - self.ui_log.info("") - - # package up and compress the results -From f22efe044f1f0565b57d6aeca2081a5227e0312c Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Mon, 14 Feb 2022 09:37:30 -0500 -Subject: [PATCH] [utilities] Don't try to chroot to / - -With the recent fix for sysroot being `None` to always being (correctly) -`/`, we should guard against situations where `sos_get_command_output()` -would now try to chroot to `/` before running any command. Incidentally, -this would also cause our unittests to fail if they were run by a -non-root user. - -Signed-off-by: Jake Hunsaker ---- - sos/utilities.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/utilities.py b/sos/utilities.py -index 6b13415b..d782123a 100644 ---- a/sos/utilities.py -+++ b/sos/utilities.py -@@ -120,7 +120,7 @@ def sos_get_command_output(command, timeout=TIMEOUT_DEFAULT, stderr=False, - # closure are caught in the parent (chroot and chdir are bound from - # the enclosing scope). - def _child_prep_fn(): -- if (chroot): -+ if chroot and chroot != '/': - os.chroot(chroot) - if (chdir): - os.chdir(chdir) --- -2.34.1 -From 3d064102f8ca6662fd9602512e1cb05cf8746dfd Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Mon, 27 Sep 2021 19:01:16 -0400 -Subject: [PATCH] [Systemd, Policy] Correct InitSystem chrooting when chroot is - needed - -This commit resolves a situation in which `sos` is being run in a -container but the `SystemdInit` InitSystem would not properly load -information from the host, thus causing the `Plugin.is_service*()` -methods to erroneously fail or return `False`. - -Fix this scenario by pulling the `_container_init()` and related logic -to check for a containerized host sysroot out of the Red Hat specific -policy and into the base `LinuxPolicy` class so that the init system can -be initialized with the correct sysroot, which is now used to chroot the -calls to the relevant `systemctl` commands. - -For now, this does impose the use of looking for the `container` env var -(automatically set by docker, podman, and crio regardless of -distribution) and the use of the `HOST` env var to read where the host's -`/` filesystem is mounted within the container. If desired in the -future, this can be changed to allow policy-specific overrides. For now -however, this extends host collection via an sos container for all -distributions currently shipping sos. - -Note that this issue only affected the `InitSystem` abstraction for -loading information about local services, and did not affect init system -related commands called by plugins as part of those collections. - -Signed-off-by: Jake Hunsaker ---- - sos/policies/distros/__init__.py | 28 ++++++++++++++++++++++++++- - sos/policies/distros/redhat.py | 27 +------------------------- - sos/policies/init_systems/__init__.py | 13 +++++++++++-- - sos/policies/init_systems/systemd.py | 7 ++++--- - 4 files changed, 43 insertions(+), 32 deletions(-) - -diff --git a/sos/policies/distros/__init__.py b/sos/policies/distros/__init__.py -index f5b9fd5b01..c33a356a75 100644 ---- a/sos/policies/distros/__init__.py -+++ b/sos/policies/distros/__init__.py -@@ -29,6 +29,10 @@ - except ImportError: - REQUESTS_LOADED = False - -+# Container environment variables for detecting if we're in a container -+ENV_CONTAINER = 'container' -+ENV_HOST_SYSROOT = 'HOST' -+ - - class LinuxPolicy(Policy): - """This policy is meant to be an abc class that provides common -@@ -69,10 +73,17 @@ def __init__(self, sysroot=None, init=None, probe_runtime=True): - probe_runtime=probe_runtime) - self.init_kernel_modules() - -+ # need to set _host_sysroot before PackageManager() -+ if sysroot: -+ self._container_init() -+ self._host_sysroot = sysroot -+ else: -+ sysroot = self._container_init() -+ - if init is not None: - self.init_system = init - elif os.path.isdir("/run/systemd/system/"): -- self.init_system = SystemdInit() -+ self.init_system = SystemdInit(chroot=sysroot) - else: - self.init_system = InitSystem() - -@@ -130,6 +141,21 @@ def get_local_name(self): - def sanitize_filename(self, name): - return re.sub(r"[^-a-z,A-Z.0-9]", "", name) - -+ def _container_init(self): -+ """Check if sos is running in a container and perform container -+ specific initialisation based on ENV_HOST_SYSROOT. -+ """ -+ if ENV_CONTAINER in os.environ: -+ if os.environ[ENV_CONTAINER] in ['docker', 'oci', 'podman']: -+ self._in_container = True -+ if ENV_HOST_SYSROOT in os.environ: -+ self._host_sysroot = os.environ[ENV_HOST_SYSROOT] -+ use_sysroot = self._in_container and self._host_sysroot is not None -+ if use_sysroot: -+ host_tmp_dir = os.path.abspath(self._host_sysroot + self._tmp_dir) -+ self._tmp_dir = host_tmp_dir -+ return self._host_sysroot if use_sysroot else None -+ - def init_kernel_modules(self): - """Obtain a list of loaded kernel modules to reference later for plugin - enablement and SoSPredicate checks -diff --git a/sos/policies/distros/redhat.py b/sos/policies/distros/redhat.py -index b3a84336be..3476e21fb2 100644 ---- a/sos/policies/distros/redhat.py -+++ b/sos/policies/distros/redhat.py -@@ -17,7 +17,7 @@ - from sos.presets.redhat import (RHEL_PRESETS, ATOMIC_PRESETS, RHV, RHEL, - CB, RHOSP, RHOCP, RH_CFME, RH_SATELLITE, - ATOMIC) --from sos.policies.distros import LinuxPolicy -+from sos.policies.distros import LinuxPolicy, ENV_HOST_SYSROOT - from sos.policies.package_managers.rpm import RpmPackageManager - from sos import _sos as _ - -@@ -56,12 +56,6 @@ def __init__(self, sysroot=None, init=None, probe_runtime=True, - super(RedHatPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime) - self.usrmove = False -- # need to set _host_sysroot before PackageManager() -- if sysroot: -- self._container_init() -- self._host_sysroot = sysroot -- else: -- sysroot = self._container_init() - - self.package_manager = RpmPackageManager(chroot=sysroot, - remote_exec=remote_exec) -@@ -140,21 +134,6 @@ def transform_path(path): - else: - return files - -- def _container_init(self): -- """Check if sos is running in a container and perform container -- specific initialisation based on ENV_HOST_SYSROOT. -- """ -- if ENV_CONTAINER in os.environ: -- if os.environ[ENV_CONTAINER] in ['docker', 'oci', 'podman']: -- self._in_container = True -- if ENV_HOST_SYSROOT in os.environ: -- self._host_sysroot = os.environ[ENV_HOST_SYSROOT] -- use_sysroot = self._in_container and self._host_sysroot is not None -- if use_sysroot: -- host_tmp_dir = os.path.abspath(self._host_sysroot + self._tmp_dir) -- self._tmp_dir = host_tmp_dir -- return self._host_sysroot if use_sysroot else None -- - def runlevel_by_service(self, name): - from subprocess import Popen, PIPE - ret = [] -@@ -183,10 +162,6 @@ def get_tmp_dir(self, opt_tmp_dir): - return opt_tmp_dir - - --# Container environment variables on Red Hat systems. --ENV_CONTAINER = 'container' --ENV_HOST_SYSROOT = 'HOST' -- - # Legal disclaimer text for Red Hat products - disclaimer_text = """ - Any information provided to %(vendor)s will be treated in \ -diff --git a/sos/policies/init_systems/__init__.py b/sos/policies/init_systems/__init__.py -index dd663e6522..beac44cee3 100644 ---- a/sos/policies/init_systems/__init__.py -+++ b/sos/policies/init_systems/__init__.py -@@ -29,9 +29,14 @@ class InitSystem(): - status of services - :type query_cmd: ``str`` - -+ :param chroot: Location to chroot to for any command execution, i.e. the -+ sysroot if we're running in a container -+ :type chroot: ``str`` or ``None`` -+ - """ - -- def __init__(self, init_cmd=None, list_cmd=None, query_cmd=None): -+ def __init__(self, init_cmd=None, list_cmd=None, query_cmd=None, -+ chroot=None): - """Initialize a new InitSystem()""" - - self.services = {} -@@ -39,6 +44,7 @@ def __init__(self, init_cmd=None, list_cmd=None, query_cmd=None): - self.init_cmd = init_cmd - self.list_cmd = "%s %s" % (self.init_cmd, list_cmd) or None - self.query_cmd = "%s %s" % (self.init_cmd, query_cmd) or None -+ self.chroot = chroot - - def is_enabled(self, name): - """Check if given service name is enabled -@@ -108,7 +114,10 @@ def _query_service(self, name): - """Query an individual service""" - if self.query_cmd: - try: -- return sos_get_command_output("%s %s" % (self.query_cmd, name)) -+ return sos_get_command_output( -+ "%s %s" % (self.query_cmd, name), -+ chroot=self.chroot -+ ) - except Exception: - return None - return None -diff --git a/sos/policies/init_systems/systemd.py b/sos/policies/init_systems/systemd.py -index 1b138f97b3..76dc57e27f 100644 ---- a/sos/policies/init_systems/systemd.py -+++ b/sos/policies/init_systems/systemd.py -@@ -15,11 +15,12 @@ - class SystemdInit(InitSystem): - """InitSystem abstraction for SystemD systems""" - -- def __init__(self): -+ def __init__(self, chroot=None): - super(SystemdInit, self).__init__( - init_cmd='systemctl', - list_cmd='list-unit-files --type=service', -- query_cmd='status' -+ query_cmd='status', -+ chroot=chroot - ) - self.load_all_services() - -@@ -30,7 +31,7 @@ def parse_query(self, output): - return 'unknown' - - def load_all_services(self): -- svcs = shell_out(self.list_cmd).splitlines()[1:] -+ svcs = shell_out(self.list_cmd, chroot=self.chroot).splitlines()[1:] - for line in svcs: - try: - name = line.split('.service')[0] -From e869bc84c714bfc2249bbcb84e14908049ee42c4 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Mon, 27 Sep 2021 12:07:08 -0400 -Subject: [PATCH] [Plugin,utilities] Add sysroot wrapper for os.path.join - -Adds a wrapper for `os.path.join()` which accounts for non-/ sysroots, -like we have done previously for other `os.path` methods. Further -updates `Plugin()` to use this wrapper where appropriate. - -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/__init__.py | 43 +++++++++++++++++----------------- - sos/utilities.py | 6 +++++ - 2 files changed, 28 insertions(+), 21 deletions(-) - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index c635b8de9..1f84bca49 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -13,7 +13,7 @@ - from sos.utilities import (sos_get_command_output, import_module, grep, - fileobj, tail, is_executable, TIMEOUT_DEFAULT, - path_exists, path_isdir, path_isfile, path_islink, -- listdir) -+ listdir, path_join) - - import os - import glob -@@ -708,19 +708,6 @@ def _log_info(self, msg): - def _log_debug(self, msg): - self.soslog.debug(self._format_msg(msg)) - -- def join_sysroot(self, path): -- """Join a given path with the configured sysroot -- -- :param path: The filesystem path that needs to be joined -- :type path: ``str`` -- -- :returns: The joined filesystem path -- :rtype: ``str`` -- """ -- if path[0] == os.sep: -- path = path[1:] -- return os.path.join(self.sysroot, path) -- - def strip_sysroot(self, path): - """Remove the configured sysroot from a filesystem path - -@@ -1176,7 +1163,7 @@ def _copy_dir(self, srcpath): - - def _get_dest_for_srcpath(self, srcpath): - if self.use_sysroot(): -- srcpath = self.join_sysroot(srcpath) -+ srcpath = self.path_join(srcpath) - for copied in self.copied_files: - if srcpath == copied["srcpath"]: - return copied["dstpath"] -@@ -1284,7 +1271,7 @@ def add_forbidden_path(self, forbidden, recursive=False): - forbidden = [forbidden] - - if self.use_sysroot(): -- forbidden = [self.join_sysroot(f) for f in forbidden] -+ forbidden = [self.path_join(f) for f in forbidden] - - for forbid in forbidden: - self._log_info("adding forbidden path '%s'" % forbid) -@@ -1438,7 +1425,7 @@ def add_copy_spec(self, copyspecs, sizelimit=None, maxage=None, - since = self.get_option('since') - - logarchive_pattern = re.compile(r'.*((\.(zip|gz|bz2|xz))|[-.][\d]+)$') -- configfile_pattern = re.compile(r"^%s/*" % self.join_sysroot("etc")) -+ configfile_pattern = re.compile(r"^%s/*" % self.path_join("etc")) - - if not self.test_predicate(pred=pred): - self._log_info("skipped copy spec '%s' due to predicate (%s)" % -@@ -1468,7 +1455,7 @@ def add_copy_spec(self, copyspecs, sizelimit=None, maxage=None, - return False - - if self.use_sysroot(): -- copyspec = self.join_sysroot(copyspec) -+ copyspec = self.path_join(copyspec) - - files = self._expand_copy_spec(copyspec) - -@@ -1683,7 +1670,7 @@ def _add_device_cmd(self, cmds, devices, timeout=None, sizelimit=None, - if not _dev_ok: - continue - if prepend_path: -- device = os.path.join(prepend_path, device) -+ device = self.path_join(prepend_path, device) - _cmd = cmd % {'dev': device} - self._add_cmd_output(cmd=_cmd, timeout=timeout, - sizelimit=sizelimit, chroot=chroot, -@@ -2592,7 +2579,7 @@ def __expand(paths): - if self.path_isfile(path) or self.path_islink(path): - found_paths.append(path) - elif self.path_isdir(path) and self.listdir(path): -- found_paths.extend(__expand(os.path.join(path, '*'))) -+ found_paths.extend(__expand(self.path_join(path, '*'))) - else: - found_paths.append(path) - except PermissionError: -@@ -2608,7 +2595,7 @@ def __expand(paths): - if (os.access(copyspec, os.R_OK) and self.path_isdir(copyspec) and - self.listdir(copyspec)): - # the directory exists and is non-empty, recurse through it -- copyspec = os.path.join(copyspec, '*') -+ copyspec = self.path_join(copyspec, '*') - expanded = glob.glob(copyspec, recursive=True) - recursed_files = [] - for _path in expanded: -@@ -2877,6 +2864,20 @@ def listdir(self, path): - """ - return listdir(path, self.commons['cmdlineopts'].sysroot) - -+ def path_join(self, path, *p): -+ """Helper to call the sos.utilities wrapper that allows the -+ corresponding `os` call to account for sysroot -+ -+ :param path: The leading path passed to os.path.join() -+ :type path: ``str`` -+ -+ :param p: Following path section(s) to be joined with ``path``, -+ an empty parameter will result in a path that ends with -+ a separator -+ :type p: ``str`` -+ """ -+ return path_join(path, *p, sysroot=self.sysroot) -+ - def postproc(self): - """Perform any postprocessing. To be replaced by a plugin if required. - """ -diff --git a/sos/utilities.py b/sos/utilities.py -index c940e066d..b75751539 100644 ---- a/sos/utilities.py -+++ b/sos/utilities.py -@@ -242,6 +242,12 @@ def listdir(path, sysroot): - return _os_wrapper(path, sysroot, 'listdir', os) - - -+def path_join(path, *p, sysroot=os.sep): -+ if not path.startswith(sysroot): -+ path = os.path.join(sysroot, path.lstrip(os.sep)) -+ return os.path.join(path, *p) -+ -+ - class AsyncReader(threading.Thread): - """Used to limit command output to a given size without deadlocking - sos. -From 9596473d1779b9c48e9923c220aaf2b8d9b3bebf Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 18 Nov 2021 13:17:14 -0500 -Subject: [PATCH] [global] Align sysroot determination and usage across sos - -The determination of sysroot - being automatic, user-specified, or -controlled via environment variables in a container - has gotten muddied -over time. This has resulted in different parts of the project; -`Policy`, `Plugin`, `SoSComponent`, etc... to not always be in sync when -sysroot is not `/`, thus causing varying and unexpected/unintended -behavior. - -Fix this by only determining sysroot within `Policy()` initialization, -and then using that determination across all aspects of the project that -use or reference sysroot. - -This results in several changes: - -- `PackageManager()` will now (again) correctly reference host package - lists when sos is run in a container. - -- `ContainerRuntime()` is now able to activate when sos is running in a - container. - -- Plugins will now properly use sysroot for _all_ plugin enablement - triggers. - -- Plugins, Policy, and SoSComponents now all reference the - `self.sysroot` variable, rather than changing between `sysroot`. -`_host_sysroot`, and `commons['sysroot']`. `_host_sysroot` has been -removed from `Policy`. - -Signed-off-by: Jake Hunsaker ---- - sos/archive.py | 2 +- - sos/component.py | 2 +- - sos/policies/__init__.py | 11 +---------- - sos/policies/distros/__init__.py | 33 +++++++++++++++++++------------ - sos/policies/distros/debian.py | 2 +- - sos/policies/distros/redhat.py | 3 +-- - sos/policies/runtimes/__init__.py | 15 +++++++++----- - sos/policies/runtimes/docker.py | 4 ++-- - sos/report/__init__.py | 6 ++---- - sos/report/plugins/__init__.py | 22 +++++++++++---------- - sos/report/plugins/unpackaged.py | 7 ++++--- - sos/utilities.py | 13 ++++++++---- - 12 files changed, 64 insertions(+), 56 deletions(-) - -diff --git a/sos/archive.py b/sos/archive.py -index b02b247595..e3c68b7789 100644 ---- a/sos/archive.py -+++ b/sos/archive.py -@@ -153,7 +153,7 @@ def dest_path(self, name): - return (os.path.join(self._archive_root, name)) - - def join_sysroot(self, path): -- if path.startswith(self.sysroot): -+ if not self.sysroot or path.startswith(self.sysroot): - return path - if path[0] == os.sep: - path = path[1:] -diff --git a/sos/component.py b/sos/component.py -index 5ac6e47f4f..dba0aabf2b 100644 ---- a/sos/component.py -+++ b/sos/component.py -@@ -109,7 +109,7 @@ def __init__(self, parser, parsed_args, cmdline_args): - try: - import sos.policies - self.policy = sos.policies.load(sysroot=self.opts.sysroot) -- self.sysroot = self.policy.host_sysroot() -+ self.sysroot = self.policy.sysroot - except KeyboardInterrupt: - self._exit(0) - self._is_root = self.policy.is_root() -diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py -index fb8db1d724..ef9188deb4 100644 ---- a/sos/policies/__init__.py -+++ b/sos/policies/__init__.py -@@ -110,7 +110,6 @@ class Policy(object): - presets = {"": PresetDefaults()} - presets_path = PRESETS_PATH - _in_container = False -- _host_sysroot = '/' - - def __init__(self, sysroot=None, probe_runtime=True): - """Subclasses that choose to override this initializer should call -@@ -124,7 +123,7 @@ def __init__(self, sysroot=None, probe_runtime=True): - self.package_manager = PackageManager() - self.valid_subclasses = [IndependentPlugin] - self.set_exec_path() -- self._host_sysroot = sysroot -+ self.sysroot = sysroot - self.register_presets(GENERIC_PRESETS) - - def check(self, remote=''): -@@ -177,14 +176,6 @@ def in_container(self): - """ - return self._in_container - -- def host_sysroot(self): -- """Get the host's default sysroot -- -- :returns: Host sysroot -- :rtype: ``str`` or ``None`` -- """ -- return self._host_sysroot -- - def dist_version(self): - """ - Return the OS version -diff --git a/sos/policies/distros/__init__.py b/sos/policies/distros/__init__.py -index 7bdc81b852..c69fc1e73c 100644 ---- a/sos/policies/distros/__init__.py -+++ b/sos/policies/distros/__init__.py -@@ -71,19 +71,18 @@ class LinuxPolicy(Policy): - def __init__(self, sysroot=None, init=None, probe_runtime=True): - super(LinuxPolicy, self).__init__(sysroot=sysroot, - probe_runtime=probe_runtime) -- self.init_kernel_modules() - -- # need to set _host_sysroot before PackageManager() - if sysroot: -- self._container_init() -- self._host_sysroot = sysroot -+ self.sysroot = sysroot - else: -- sysroot = self._container_init() -+ self.sysroot = self._container_init() -+ -+ self.init_kernel_modules() - - if init is not None: - self.init_system = init - elif os.path.isdir("/run/systemd/system/"): -- self.init_system = SystemdInit(chroot=sysroot) -+ self.init_system = SystemdInit(chroot=self.sysroot) - else: - self.init_system = InitSystem() - -@@ -149,27 +148,30 @@ def _container_init(self): - if os.environ[ENV_CONTAINER] in ['docker', 'oci', 'podman']: - self._in_container = True - if ENV_HOST_SYSROOT in os.environ: -- self._host_sysroot = os.environ[ENV_HOST_SYSROOT] -- use_sysroot = self._in_container and self._host_sysroot is not None -+ _host_sysroot = os.environ[ENV_HOST_SYSROOT] -+ use_sysroot = self._in_container and _host_sysroot is not None - if use_sysroot: -- host_tmp_dir = os.path.abspath(self._host_sysroot + self._tmp_dir) -+ host_tmp_dir = os.path.abspath(_host_sysroot + self._tmp_dir) - self._tmp_dir = host_tmp_dir -- return self._host_sysroot if use_sysroot else None -+ return _host_sysroot if use_sysroot else None - - def init_kernel_modules(self): - """Obtain a list of loaded kernel modules to reference later for plugin - enablement and SoSPredicate checks - """ - self.kernel_mods = [] -+ release = os.uname().release - - # first load modules from lsmod -- lines = shell_out("lsmod", timeout=0).splitlines() -+ lines = shell_out("lsmod", timeout=0, chroot=self.sysroot).splitlines() - self.kernel_mods.extend([ - line.split()[0].strip() for line in lines[1:] - ]) - - # next, include kernel builtins -- builtins = "/usr/lib/modules/%s/modules.builtin" % os.uname().release -+ builtins = self.join_sysroot( -+ "/usr/lib/modules/%s/modules.builtin" % release -+ ) - try: - with open(builtins, "r") as mfile: - for line in mfile: -@@ -186,7 +188,7 @@ def init_kernel_modules(self): - 'dm_mod': 'CONFIG_BLK_DEV_DM' - } - -- booted_config = "/boot/config-%s" % os.uname().release -+ booted_config = self.join_sysroot("/boot/config-%s" % release) - kconfigs = [] - try: - with open(booted_config, "r") as kfile: -@@ -200,6 +202,11 @@ def init_kernel_modules(self): - if config_strings[builtin] in kconfigs: - self.kernel_mods.append(builtin) - -+ def join_sysroot(self, path): -+ if self.sysroot and self.sysroot != '/': -+ path = os.path.join(self.sysroot, path.lstrip('/')) -+ return path -+ - def pre_work(self): - # this method will be called before the gathering begins - -diff --git a/sos/policies/distros/debian.py b/sos/policies/distros/debian.py -index 95b389a65e..639fd5eba3 100644 ---- a/sos/policies/distros/debian.py -+++ b/sos/policies/distros/debian.py -@@ -27,7 +27,7 @@ def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(DebianPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime) -- self.package_manager = DpkgPackageManager(chroot=sysroot, -+ self.package_manager = DpkgPackageManager(chroot=self.sysroot, - remote_exec=remote_exec) - self.valid_subclasses += [DebianPlugin] - -diff --git a/sos/policies/distros/redhat.py b/sos/policies/distros/redhat.py -index eb44240736..4b14abaf3a 100644 ---- a/sos/policies/distros/redhat.py -+++ b/sos/policies/distros/redhat.py -@@ -42,7 +42,6 @@ class RedHatPolicy(LinuxPolicy): - _redhat_release = '/etc/redhat-release' - _tmp_dir = "/var/tmp" - _in_container = False -- _host_sysroot = '/' - default_scl_prefix = '/opt/rh' - name_pattern = 'friendly' - upload_url = None -@@ -57,7 +56,7 @@ def __init__(self, sysroot=None, init=None, probe_runtime=True, - probe_runtime=probe_runtime) - self.usrmove = False - -- self.package_manager = RpmPackageManager(chroot=sysroot, -+ self.package_manager = RpmPackageManager(chroot=self.sysroot, - remote_exec=remote_exec) - - self.valid_subclasses += [RedHatPlugin] -diff --git a/sos/policies/runtimes/__init__.py b/sos/policies/runtimes/__init__.py -index f28d6a1df3..2e60ad2361 100644 ---- a/sos/policies/runtimes/__init__.py -+++ b/sos/policies/runtimes/__init__.py -@@ -64,7 +64,7 @@ def check_is_active(self): - :returns: ``True`` if the runtime is active, else ``False`` - :rtype: ``bool`` - """ -- if is_executable(self.binary): -+ if is_executable(self.binary, self.policy.sysroot): - self.active = True - return True - return False -@@ -78,7 +78,7 @@ def get_containers(self, get_all=False): - containers = [] - _cmd = "%s ps %s" % (self.binary, '-a' if get_all else '') - if self.active: -- out = sos_get_command_output(_cmd) -+ out = sos_get_command_output(_cmd, chroot=self.policy.sysroot) - if out['status'] == 0: - for ent in out['output'].splitlines()[1:]: - ent = ent.split() -@@ -112,8 +112,10 @@ def get_images(self): - images = [] - fmt = '{{lower .Repository}}:{{lower .Tag}} {{lower .ID}}' - if self.active: -- out = sos_get_command_output("%s images --format '%s'" -- % (self.binary, fmt)) -+ out = sos_get_command_output( -+ "%s images --format '%s'" % (self.binary, fmt), -+ chroot=self.policy.sysroot -+ ) - if out['status'] == 0: - for ent in out['output'].splitlines(): - ent = ent.split() -@@ -129,7 +131,10 @@ def get_volumes(self): - """ - vols = [] - if self.active: -- out = sos_get_command_output("%s volume ls" % self.binary) -+ out = sos_get_command_output( -+ "%s volume ls" % self.binary, -+ chroot=self.policy.sysroot -+ ) - if out['status'] == 0: - for ent in out['output'].splitlines()[1:]: - ent = ent.split() -diff --git a/sos/policies/runtimes/docker.py b/sos/policies/runtimes/docker.py -index 759dfaf6a0..e81f580ec3 100644 ---- a/sos/policies/runtimes/docker.py -+++ b/sos/policies/runtimes/docker.py -@@ -18,9 +18,9 @@ class DockerContainerRuntime(ContainerRuntime): - name = 'docker' - binary = 'docker' - -- def check_is_active(self): -+ def check_is_active(self, sysroot=None): - # the daemon must be running -- if (is_executable('docker') and -+ if (is_executable('docker', sysroot) and - (self.policy.init_system.is_running('docker') or - self.policy.init_system.is_running('snap.docker.dockerd'))): - self.active = True -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index a4c92accd3..a6c72778fc 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -173,14 +173,12 @@ def __init__(self, parser, args, cmdline): - self._set_directories() - - msg = "default" -- host_sysroot = self.policy.host_sysroot() -+ self.sysroot = self.policy.sysroot - # set alternate system root directory - if self.opts.sysroot: - msg = "cmdline" -- self.sysroot = self.opts.sysroot -- elif self.policy.in_container() and host_sysroot != os.sep: -+ elif self.policy.in_container() and self.sysroot != os.sep: - msg = "policy" -- self.sysroot = host_sysroot - self.soslog.debug("set sysroot to '%s' (%s)" % (self.sysroot, msg)) - - if self.opts.chroot not in chroot_modes: -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index 46028bb124..e180ae1727 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -724,7 +724,7 @@ def strip_sysroot(self, path): - """ - if not self.use_sysroot(): - return path -- if path.startswith(self.sysroot): -+ if self.sysroot and path.startswith(self.sysroot): - return path[len(self.sysroot):] - return path - -@@ -743,8 +743,10 @@ def tmp_in_sysroot(self): - ``False`` - :rtype: ``bool`` - """ -- paths = [self.sysroot, self.archive.get_tmp_dir()] -- return os.path.commonprefix(paths) == self.sysroot -+ # if sysroot is still None, that implies '/' -+ _sysroot = self.sysroot or '/' -+ paths = [_sysroot, self.archive.get_tmp_dir()] -+ return os.path.commonprefix(paths) == _sysroot - - def is_installed(self, package_name): - """Is the package $package_name installed? -@@ -2621,7 +2623,7 @@ def __expand(paths): - return list(set(expanded)) - - def _collect_copy_specs(self): -- for path in self.copy_paths: -+ for path in sorted(self.copy_paths, reverse=True): - self._log_info("collecting path '%s'" % path) - self._do_copy_path(path) - self.generate_copyspec_tags() -@@ -2749,7 +2751,7 @@ def _check_plugin_triggers(self, files, packages, commands, services, - - return ((any(self.path_exists(fname) for fname in files) or - any(self.is_installed(pkg) for pkg in packages) or -- any(is_executable(cmd) for cmd in commands) or -+ any(is_executable(cmd, self.sysroot) for cmd in commands) or - any(self.is_module_loaded(mod) for mod in self.kernel_mods) or - any(self.is_service(svc) for svc in services) or - any(self.container_exists(cntr) for cntr in containers)) and -@@ -2817,7 +2819,7 @@ def path_exists(self, path): - :returns: True if the path exists in sysroot, else False - :rtype: ``bool`` - """ -- return path_exists(path, self.commons['cmdlineopts'].sysroot) -+ return path_exists(path, self.sysroot) - - def path_isdir(self, path): - """Helper to call the sos.utilities wrapper that allows the -@@ -2830,7 +2832,7 @@ def path_isdir(self, path): - :returns: True if the path is a dir, else False - :rtype: ``bool`` - """ -- return path_isdir(path, self.commons['cmdlineopts'].sysroot) -+ return path_isdir(path, self.sysroot) - - def path_isfile(self, path): - """Helper to call the sos.utilities wrapper that allows the -@@ -2843,7 +2845,7 @@ def path_isfile(self, path): - :returns: True if the path is a file, else False - :rtype: ``bool`` - """ -- return path_isfile(path, self.commons['cmdlineopts'].sysroot) -+ return path_isfile(path, self.sysroot) - - def path_islink(self, path): - """Helper to call the sos.utilities wrapper that allows the -@@ -2856,7 +2858,7 @@ def path_islink(self, path): - :returns: True if the path is a link, else False - :rtype: ``bool`` - """ -- return path_islink(path, self.commons['cmdlineopts'].sysroot) -+ return path_islink(path, self.sysroot) - - def listdir(self, path): - """Helper to call the sos.utilities wrapper that allows the -@@ -2869,7 +2871,7 @@ def listdir(self, path): - :returns: Contents of path, if it is a directory - :rtype: ``list`` - """ -- return listdir(path, self.commons['cmdlineopts'].sysroot) -+ return listdir(path, self.sysroot) - - def path_join(self, path, *p): - """Helper to call the sos.utilities wrapper that allows the -diff --git a/sos/report/plugins/unpackaged.py b/sos/report/plugins/unpackaged.py -index 772b1d1fbb..24203c4b13 100644 ---- a/sos/report/plugins/unpackaged.py -+++ b/sos/report/plugins/unpackaged.py -@@ -58,10 +58,11 @@ def format_output(files): - """ - expanded = [] - for f in files: -- if self.path_islink(f): -- expanded.append("{} -> {}".format(f, os.readlink(f))) -+ fp = self.path_join(f) -+ if self.path_islink(fp): -+ expanded.append("{} -> {}".format(fp, os.readlink(fp))) - else: -- expanded.append(f) -+ expanded.append(fp) - return expanded - - # Check command predicate to avoid costly processing -diff --git a/sos/utilities.py b/sos/utilities.py -index b757515397..d66309334b 100644 ---- a/sos/utilities.py -+++ b/sos/utilities.py -@@ -96,11 +96,15 @@ def grep(pattern, *files_or_paths): - return matches - - --def is_executable(command): -+def is_executable(command, sysroot=None): - """Returns if a command matches an executable on the PATH""" - - paths = os.environ.get("PATH", "").split(os.path.pathsep) - candidates = [command] + [os.path.join(p, command) for p in paths] -+ if sysroot: -+ candidates += [ -+ os.path.join(sysroot, c.lstrip('/')) for c in candidates -+ ] - return any(os.access(path, os.X_OK) for path in candidates) - - -@@ -216,8 +220,9 @@ def get_human_readable(size, precision=2): - - - def _os_wrapper(path, sysroot, method, module=os.path): -- if sysroot not in [None, '/']: -- path = os.path.join(sysroot, path.lstrip('/')) -+ if sysroot and sysroot != os.sep: -+ if not path.startswith(sysroot): -+ path = os.path.join(sysroot, path.lstrip('/')) - _meth = getattr(module, method) - return _meth(path) - -@@ -243,7 +248,7 @@ def listdir(path, sysroot): - - - def path_join(path, *p, sysroot=os.sep): -- if not path.startswith(sysroot): -+ if sysroot and not path.startswith(sysroot): - path = os.path.join(sysroot, path.lstrip(os.sep)) - return os.path.join(path, *p) - -From a43124e1f6217107838eed4d70339d100cbbc77a Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Wed, 9 Feb 2022 19:45:27 +0100 -Subject: [PATCH] [policies] Set fallback to None sysroot - -9596473 commit added a regression allowing to set sysroot to None -when running sos report on a regular system (outside a container). In -such a case, we need to fallback to '/' sysroot. - -Resolves: #2846 - -Signed-off-by: Pavel Moravec ---- - sos/policies/distros/__init__.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/policies/distros/__init__.py b/sos/policies/distros/__init__.py -index f3c1de11..9048f1c4 100644 ---- a/sos/policies/distros/__init__.py -+++ b/sos/policies/distros/__init__.py -@@ -78,7 +78,7 @@ class LinuxPolicy(Policy): - if sysroot: - self.sysroot = sysroot - else: -- self.sysroot = self._container_init() -+ self.sysroot = self._container_init() or '/' - - self.init_kernel_modules() - --- -2.34.1 - diff --git a/SOURCES/sos-bz2011538-iptables-save-under-nf_tables-kmod.patch b/SOURCES/sos-bz2011538-iptables-save-under-nf_tables-kmod.patch deleted file mode 100644 index e234bc6..0000000 --- a/SOURCES/sos-bz2011538-iptables-save-under-nf_tables-kmod.patch +++ /dev/null @@ -1,73 +0,0 @@ -From 7d5157aa5071e3620246e2d4aa80acb2d3ed30f0 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 28 Sep 2021 22:44:52 +0200 -Subject: [PATCH] [networking] prevent iptables-save commands to load nf_tables - kmod - -If iptables has built-in nf_tables kmod, then -'ip netns iptables-save' command requires the kmod which must -be guarded by predicate. - -Analogously for ip6tables. - -Resolves: #2703 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/networking.py | 29 ++++++++++++++++++++++++----- - 1 file changed, 24 insertions(+), 5 deletions(-) - -diff --git a/sos/report/plugins/networking.py b/sos/report/plugins/networking.py -index c80ae719..1237f629 100644 ---- a/sos/report/plugins/networking.py -+++ b/sos/report/plugins/networking.py -@@ -182,22 +182,41 @@ class Networking(Plugin): - # per-namespace. - self.add_cmd_output("ip netns") - cmd_prefix = "ip netns exec " -- for namespace in self.get_network_namespaces( -- self.get_option("namespace_pattern"), -- self.get_option("namespaces")): -+ namespaces = self.get_network_namespaces( -+ self.get_option("namespace_pattern"), -+ self.get_option("namespaces")) -+ if (namespaces): -+ # 'ip netns exec iptables-save' must be guarded by nf_tables -+ # kmod, if 'iptables -V' output contains 'nf_tables' -+ # analogously for ip6tables -+ co = {'cmd': 'iptables -V', 'output': 'nf_tables'} -+ co6 = {'cmd': 'ip6tables -V', 'output': 'nf_tables'} -+ iptables_with_nft = (SoSPredicate(self, kmods=['nf_tables']) -+ if self.test_predicate(self, -+ pred=SoSPredicate(self, cmd_outputs=co)) -+ else None) -+ ip6tables_with_nft = (SoSPredicate(self, kmods=['nf_tables']) -+ if self.test_predicate(self, -+ pred=SoSPredicate(self, cmd_outputs=co6)) -+ else None) -+ for namespace in namespaces: - ns_cmd_prefix = cmd_prefix + namespace + " " - self.add_cmd_output([ - ns_cmd_prefix + "ip address show", - ns_cmd_prefix + "ip route show table all", - ns_cmd_prefix + "ip -s -s neigh show", - ns_cmd_prefix + "ip rule list", -- ns_cmd_prefix + "iptables-save", -- ns_cmd_prefix + "ip6tables-save", - ns_cmd_prefix + "netstat %s -neopa" % self.ns_wide, - ns_cmd_prefix + "netstat -s", - ns_cmd_prefix + "netstat %s -agn" % self.ns_wide, - ns_cmd_prefix + "nstat -zas", - ], priority=50) -+ self.add_cmd_output([ns_cmd_prefix + "iptables-save"], -+ pred=iptables_with_nft, -+ priority=50) -+ self.add_cmd_output([ns_cmd_prefix + "ip6tables-save"], -+ pred=ip6tables_with_nft, -+ priority=50) - - ss_cmd = ns_cmd_prefix + "ss -peaonmi" - # --allow-system-changes is handled directly in predicate --- -2.31.1 - diff --git a/SOURCES/sos-bz2012858-dryrun-uncaught-exception.patch b/SOURCES/sos-bz2012858-dryrun-uncaught-exception.patch deleted file mode 100644 index 619d538..0000000 --- a/SOURCES/sos-bz2012858-dryrun-uncaught-exception.patch +++ /dev/null @@ -1,33 +0,0 @@ -From e56b3ea999731b831ebba80cf367e43e65c12b62 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 4 Oct 2021 14:43:08 +0200 -Subject: [PATCH] [report] Overwrite pred=None before refering predicate - attributes - -During a dry run, add_journal method sets pred=None whilst log_skipped_cmd -refers to predicate attributes. In that case, replace None predicate -by a default / empty predicate. - -Resolves: #2711 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/__init__.py | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index 3c2b64d9..c635b8de 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -1693,6 +1693,8 @@ class Plugin(): - def _add_cmd_output(self, **kwargs): - """Internal helper to add a single command to the collection list.""" - pred = kwargs.pop('pred') if 'pred' in kwargs else SoSPredicate(self) -+ if pred is None: -+ pred = SoSPredicate(self) - if 'priority' not in kwargs: - kwargs['priority'] = 10 - if 'changes' not in kwargs: --- -2.31.1 - diff --git a/SOURCES/sos-bz2012859-plugin-timeout-unhandled-exception.patch b/SOURCES/sos-bz2012859-plugin-timeout-unhandled-exception.patch deleted file mode 100644 index e977fb5..0000000 --- a/SOURCES/sos-bz2012859-plugin-timeout-unhandled-exception.patch +++ /dev/null @@ -1,31 +0,0 @@ -From a93e118a9c88df52fd2c701d2276185f877d565c Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Wed, 3 Nov 2021 16:07:15 +0100 -Subject: [PATCH] [report] shutdown threads for timeouted plugins - -Wait for shutting down threads of timeouted plugins, to prevent -them in writing to moved auxiliary files like sos_logs/sos.log - -Resolves: #2722 -Closes: #2746 - -Signed-off-by: Pavel Moravec ---- - sos/report/__init__.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index 1b5bc97d..ef86b28d 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -1046,6 +1046,7 @@ class SoSReport(SoSComponent): - self.ui_log.error("\n Plugin %s timed out\n" % plugin[1]) - self.running_plugs.remove(plugin[1]) - self.loaded_plugins[plugin[0]-1][1].set_timeout_hit() -+ pool.shutdown(wait=True) - pool._threads.clear() - if self.opts.estimate_only: - from pathlib import Path --- -2.31.1 - diff --git a/SOURCES/sos-bz2019697-openvswitch-offline-analysis.patch b/SOURCES/sos-bz2019697-openvswitch-offline-analysis.patch deleted file mode 100644 index 8bd4adb..0000000 --- a/SOURCES/sos-bz2019697-openvswitch-offline-analysis.patch +++ /dev/null @@ -1,151 +0,0 @@ -From 3f0ec3e55e7dcec89dd7fad10084ea7f16178608 Mon Sep 17 00:00:00 2001 -From: Salvatore Daniele -Date: Tue, 7 Sep 2021 13:48:22 -0400 -Subject: [PATCH 1/2] [openvswitch] add ovs default OpenFlow protocols - -ovs-vsctl list bridge can return an empty 'protocol' column even when -there are OpenFlow protocols in place by default. - -ovs-ofctl --version will return the range of supported ofp and should -also be used to ensure flow information for relevant protocol versions -is collected. - -OpenFlow default versions: -https://docs.openvswitch.org/en/latest/faq/openflow/ - -Signed-off-by: Salvatore Daniele ---- - sos/report/plugins/openvswitch.py | 26 ++++++++++++++++++++++++++ - 1 file changed, 26 insertions(+) - -diff --git a/sos/report/plugins/openvswitch.py b/sos/report/plugins/openvswitch.py -index cd897db2..92cc7259 100644 ---- a/sos/report/plugins/openvswitch.py -+++ b/sos/report/plugins/openvswitch.py -@@ -206,6 +206,7 @@ class OpenVSwitch(Plugin): - - # Gather additional output for each OVS bridge on the host. - br_list_result = self.collect_cmd_output("ovs-vsctl -t 5 list-br") -+ ofp_ver_result = self.collect_cmd_output("ovs-ofctl -t 5 --version") - if br_list_result['status'] == 0: - for br in br_list_result['output'].splitlines(): - self.add_cmd_output([ -@@ -232,6 +233,16 @@ class OpenVSwitch(Plugin): - "OpenFlow15" - ] - -+ # Flow protocol hex identifiers -+ ofp_versions = { -+ 0x01: "OpenFlow10", -+ 0x02: "OpenFlow11", -+ 0x03: "OpenFlow12", -+ 0x04: "OpenFlow13", -+ 0x05: "OpenFlow14", -+ 0x06: "OpenFlow15", -+ } -+ - # List protocols currently in use, if any - ovs_list_bridge_cmd = "ovs-vsctl -t 5 list bridge %s" % br - br_info = self.collect_cmd_output(ovs_list_bridge_cmd) -@@ -242,6 +253,21 @@ class OpenVSwitch(Plugin): - br_protos_ln = line[line.find("[")+1:line.find("]")] - br_protos = br_protos_ln.replace('"', '').split(", ") - -+ # If 'list bridge' yeilded no protocols, use the range of -+ # protocols enabled by default on this version of ovs. -+ if br_protos == [''] and ofp_ver_result['output']: -+ ofp_version_range = ofp_ver_result['output'].splitlines() -+ ver_range = [] -+ -+ for line in ofp_version_range: -+ if "OpenFlow versions" in line: -+ v = line.split("OpenFlow versions ")[1].split(":") -+ ver_range = range(int(v[0], 16), int(v[1], 16)+1) -+ -+ for protocol in ver_range: -+ if protocol in ofp_versions: -+ br_protos.append(ofp_versions[protocol]) -+ - # Collect flow information for relevant protocol versions only - for flow in flow_versions: - if flow in br_protos: --- -2.31.1 - - -From 5a006024f730213a726c70e82c5ecd2daf685b2b Mon Sep 17 00:00:00 2001 -From: Salvatore Daniele -Date: Tue, 7 Sep 2021 14:17:19 -0400 -Subject: [PATCH 2/2] [openvswitch] add commands for offline analysis - -Replicas of ovs-vswitchd and ovsdb-server can be recreated offline -using flow, group, and tlv dumps, and ovs conf.db. This allows for -offline anaylsis and the use of tools such as ovs-appctl -ofproto/trace and ovs-ofctl for debugging. - -This patch ensures this information is available in the sos report. -The db is copied rather than collected using ovsdb-client list dump -for two reasons: - -ovsdb-client requires interacting with the ovsdb-server which could -take it 'down' for some time, and impact large, busy clusters. - -The list-dump is not in a format that can be used to restore the db -offline. All of the information in the list dump is available and more -by copying the db. - -Signed-off-by: Salvatore Daniele ---- - sos/report/plugins/openvswitch.py | 12 ++++++++++-- - sos/report/plugins/ovn_central.py | 1 + - 2 files changed, 11 insertions(+), 2 deletions(-) - -diff --git a/sos/report/plugins/openvswitch.py b/sos/report/plugins/openvswitch.py -index 92cc7259..003596c6 100644 ---- a/sos/report/plugins/openvswitch.py -+++ b/sos/report/plugins/openvswitch.py -@@ -75,12 +75,19 @@ class OpenVSwitch(Plugin): - "/run/openvswitch/ovs-monitor-ipsec.pid" - ]) - -+ self.add_copy_spec([ -+ path_join('/usr/local/etc/openvswitch', 'conf.db'), -+ path_join('/etc/openvswitch', 'conf.db'), -+ path_join('/var/lib/openvswitch', 'conf.db'), -+ ]) -+ ovs_dbdir = environ.get('OVS_DBDIR') -+ if ovs_dbdir: -+ self.add_copy_spec(path_join(ovs_dbdir, 'conf.db')) -+ - self.add_cmd_output([ - # The '-t 5' adds an upper bound on how long to wait to connect - # to the Open vSwitch server, avoiding hangs when running sos. - "ovs-vsctl -t 5 show", -- # Gather the database. -- "ovsdb-client -f list dump", - # List the contents of important runtime directories - "ls -laZ /run/openvswitch", - "ls -laZ /dev/hugepages/", -@@ -276,6 +283,7 @@ class OpenVSwitch(Plugin): - "ovs-ofctl -O %s dump-groups %s" % (flow, br), - "ovs-ofctl -O %s dump-group-stats %s" % (flow, br), - "ovs-ofctl -O %s dump-flows %s" % (flow, br), -+ "ovs-ofctl -O %s dump-tlv-map %s" % (flow, br), - "ovs-ofctl -O %s dump-ports-desc %s" % (flow, br) - ]) - -diff --git a/sos/report/plugins/ovn_central.py b/sos/report/plugins/ovn_central.py -index a4c483a9..d6647aad 100644 ---- a/sos/report/plugins/ovn_central.py -+++ b/sos/report/plugins/ovn_central.py -@@ -138,6 +138,7 @@ class OVNCentral(Plugin): - os.path.join('/usr/local/etc/openvswitch', dbfile), - os.path.join('/etc/openvswitch', dbfile), - os.path.join('/var/lib/openvswitch', dbfile), -+ os.path.join('/var/lib/ovn/etc', dbfile), - ]) - if ovs_dbdir: - self.add_copy_spec(os.path.join(ovs_dbdir, dbfile)) --- -2.31.1 - diff --git a/SOURCES/sos-bz2020778-filter-namespace-per-pattern.patch b/SOURCES/sos-bz2020778-filter-namespace-per-pattern.patch deleted file mode 100644 index 5b0afdb..0000000 --- a/SOURCES/sos-bz2020778-filter-namespace-per-pattern.patch +++ /dev/null @@ -1,54 +0,0 @@ -From 568eb2fbcf74ecad00d5c06989f55f8a6a9e3516 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Thu, 4 Nov 2021 23:14:21 +0100 -Subject: [PATCH] [report] fix filter_namespace per pattern - -Curently, -k networking.namespace_pattern=.. is broken as the R.E. test -forgets to add the namespace in case of positive match. - -Also ensure both plugopts namespace_pattern and namespaces work -together. - -Resolves: #2748 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/__init__.py | 15 +++++++-------- - 1 file changed, 7 insertions(+), 8 deletions(-) - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index 3e717993..a0d4e95d 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -2953,21 +2953,20 @@ class Plugin(): - ) - for ns in ns_list: - # if ns_pattern defined, skip namespaces not matching the pattern -- if ns_pattern: -- if not bool(re.match(pattern, ns)): -- continue -+ if ns_pattern and not bool(re.match(pattern, ns)): -+ continue -+ out_ns.append(ns) - -- # if ns_max is defined at all, limit returned list to that number -+ # if ns_max is defined at all, break the loop when the limit is -+ # reached - # this allows the use of both '0' and `None` to mean unlimited -- elif ns_max: -- out_ns.append(ns) -+ if ns_max: - if len(out_ns) == ns_max: - self._log_warn("Limiting namespace iteration " - "to first %s namespaces found" - % ns_max) - break -- else: -- out_ns.append(ns) -+ - return out_ns - - --- -2.31.1 - diff --git a/SOURCES/sos-bz2023481-plugin-timeouts-proper-handling.patch b/SOURCES/sos-bz2023481-plugin-timeouts-proper-handling.patch deleted file mode 100644 index 9fc7c3d..0000000 --- a/SOURCES/sos-bz2023481-plugin-timeouts-proper-handling.patch +++ /dev/null @@ -1,91 +0,0 @@ -From 3fea9a564c4112d04f6324df0d8b212e78feb5b3 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 3 Nov 2021 11:02:54 -0400 -Subject: [PATCH] [Plugin] Ensure specific plugin timeouts are only set for - that plugin - -It was discovered that setting a specific plugin timeout via the `-k -$plugin.timeout` option could influence the timeout setting for other -plugins that are not also having their timeout explicitly set. Fix this -by moving the default plugin opts into `Plugin.__init__()` so that each -plugin is ensured a private copy of these default plugin options. - -Additionally, add more timeout data to plugin manifest entries to allow -for better tracking of this setting. - -Adds a test case for this scenario. - -Closes: #2744 - -Signed-off-by: Jake Hunsaker ---- - sos/report/__init__.py | 2 +- - sos/report/plugins/__init__.py | 28 +++++++++++++------ - tests/vendor_tests/redhat/rhbz2018033.py | 35 ++++++++++++++++++++++++ - 3 files changed, 55 insertions(+), 10 deletions(-) - create mode 100644 tests/vendor_tests/redhat/rhbz2018033.py - -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index ef86b28d..c95e6300 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -766,7 +766,7 @@ class SoSReport(SoSComponent): - if self.all_options: - self.ui_log.info(_("The following options are available for ALL " - "plugins:")) -- for opt in self.all_options[0][0]._default_plug_opts: -+ for opt in self.all_options[0][0].get_default_plugin_opts(): - val = opt[3] - if val == -1: - val = TIMEOUT_DEFAULT -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index 49f1af27..3e717993 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -474,12 +474,6 @@ class Plugin(object): - # Default predicates - predicate = None - cmd_predicate = None -- _default_plug_opts = [ -- ('timeout', 'Timeout in seconds for plugin to finish', 'fast', -1), -- ('cmd-timeout', 'Timeout in seconds for a command', 'fast', -1), -- ('postproc', 'Enable post-processing collected plugin data', 'fast', -- True) -- ] - - def __init__(self, commons): - -@@ -506,7 +500,7 @@ class Plugin(object): - else logging.getLogger('sos') - - # add the default plugin opts -- self.option_list.extend(self._default_plug_opts) -+ self.option_list.extend(self.get_default_plugin_opts()) - - # get the option list into a dictionary - for opt in self.option_list: -@@ -591,6 +583,14 @@ class Plugin(): - # Initialise the default --dry-run predicate - self.set_predicate(SoSPredicate(self)) - -+ def get_default_plugin_opts(self): -+ return [ -+ ('timeout', 'Timeout in seconds for plugin to finish', 'fast', -1), -+ ('cmd-timeout', 'Timeout in seconds for a command', 'fast', -1), -+ ('postproc', 'Enable post-processing collected plugin data', 'fast', -+ True) -+ ] -+ - def set_plugin_manifest(self, manifest): - """Pass in a manifest object to the plugin to write to - -@@ -547,7 +541,9 @@ class Plugin(object): - self.manifest.add_field('setup_start', '') - self.manifest.add_field('setup_end', '') - self.manifest.add_field('setup_time', '') -+ self.manifest.add_field('timeout', self.timeout) - self.manifest.add_field('timeout_hit', False) -+ self.manifest.add_field('command_timeout', self.cmdtimeout) - self.manifest.add_list('commands', []) - self.manifest.add_list('files', []) - diff --git a/SOURCES/sos-bz2024893-cleaner-hostnames-improvements.patch b/SOURCES/sos-bz2024893-cleaner-hostnames-improvements.patch deleted file mode 100644 index b129f9e..0000000 --- a/SOURCES/sos-bz2024893-cleaner-hostnames-improvements.patch +++ /dev/null @@ -1,1829 +0,0 @@ -From decd39b7799a0579ea085b0da0728b6eabd49b38 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 1 Sep 2021 00:28:58 -0400 -Subject: [PATCH] [clean] Provide archive abstractions to obfuscate more than - sos archives - -This commit removes the restriction imposed on `sos clean` since its -introduction in sos-4.0 to only work against known sos report archives -or build directories. This is because there has been interest in using -the obfuscation bits of sos in other data-collector projects. - -The `SoSObfuscationArchive()` class has been revamped to now be an -abstraction for different types of archives, and the cleaner logic has -been updated to leverage this new abstraction rather than assuming we're -working on an sos archive. - -Abstractions are added for our own native use cases - that being `sos -report` and `sos collect` for at-runtime obfuscation, as well as -standalone archives previously generated. Further generic abstractions -are available for plain directories and tarballs however these will not -provide the same level of coverage as fully supported archive types, as -is noted in the manpage for sos-clean. - -Signed-off-by: Jake Hunsaker ---- - man/en/sos-clean.1 | 25 ++ - sos/cleaner/__init__.py | 308 +++++++++--------- - .../__init__.py} | 80 ++++- - sos/cleaner/archives/generic.py | 52 +++ - sos/cleaner/archives/sos.py | 106 ++++++ - sos/cleaner/parsers/__init__.py | 6 - - sos/cleaner/parsers/hostname_parser.py | 1 - - sos/cleaner/parsers/ip_parser.py | 1 - - sos/cleaner/parsers/keyword_parser.py | 1 - - sos/cleaner/parsers/mac_parser.py | 1 - - sos/cleaner/parsers/username_parser.py | 8 - - tests/cleaner_tests/existing_archive.py | 7 + - tests/cleaner_tests/full_report_run.py | 3 + - tests/cleaner_tests/report_with_mask.py | 3 + - 14 files changed, 423 insertions(+), 179 deletions(-) - rename sos/cleaner/{obfuscation_archive.py => archives/__init__.py} (81%) - create mode 100644 sos/cleaner/archives/generic.py - create mode 100644 sos/cleaner/archives/sos.py - -diff --git a/man/en/sos-clean.1 b/man/en/sos-clean.1 -index b77bc63c..54026713 100644 ---- a/man/en/sos-clean.1 -+++ b/man/en/sos-clean.1 -@@ -10,6 +10,7 @@ sos clean - Obfuscate sensitive data from one or more sosreports - [\-\-jobs] - [\-\-no-update] - [\-\-keep-binary-files] -+ [\-\-archive-type] - - .SH DESCRIPTION - \fBsos clean\fR or \fBsos mask\fR is an sos subcommand used to obfuscate sensitive information from -@@ -88,6 +89,30 @@ Users should review any archive that keeps binary files in place before sending - a third party. - - Default: False (remove encountered binary files) -+.TP -+.B \-\-archive-type TYPE -+Specify the type of archive that TARGET was generated as. -+When sos inspects a TARGET archive, it tries to identify what type of archive it is. -+For example, it may be a report generated by \fBsos report\fR, or a collection of those -+reports generated by \fBsos collect\fR, which require separate approaches. -+ -+This option may be useful if a given TARGET archive is known to be of a specific type, -+but due to unknown reasons or some malformed/missing information in the archive directly, -+that is not properly identified by sos. -+ -+The following are accepted values for this option: -+ -+ \fBauto\fR Automatically detect the archive type -+ \fBreport\fR An archive generated by \fBsos report\fR -+ \fBcollect\fR An archive generated by \fBsos collect\fR -+ -+The following may also be used, however note that these do not attempt to pre-load -+any information from the archives into the parsers. This means that, among other limitations, -+items like host and domain names may not be obfuscated unless an obfuscated mapping already exists -+on the system from a previous execution. -+ -+ \fBdata-dir\fR A plain directory on the filesystem. -+ \fBtarball\fR A generic tar archive not associated with any known tool - - .SH SEE ALSO - .BR sos (1) -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index 6aadfe79..6d2eb483 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -12,9 +12,7 @@ import hashlib - import json - import logging - import os --import re - import shutil --import tarfile - import tempfile - - from concurrent.futures import ThreadPoolExecutor -@@ -27,7 +25,10 @@ from sos.cleaner.parsers.mac_parser import SoSMacParser - from sos.cleaner.parsers.hostname_parser import SoSHostnameParser - from sos.cleaner.parsers.keyword_parser import SoSKeywordParser - from sos.cleaner.parsers.username_parser import SoSUsernameParser --from sos.cleaner.obfuscation_archive import SoSObfuscationArchive -+from sos.cleaner.archives.sos import (SoSReportArchive, SoSReportDirectory, -+ SoSCollectorArchive, -+ SoSCollectorDirectory) -+from sos.cleaner.archives.generic import DataDirArchive, TarballArchive - from sos.utilities import get_human_readable - from textwrap import fill - -@@ -41,6 +42,7 @@ class SoSCleaner(SoSComponent): - desc = "Obfuscate sensitive networking information in a report" - - arg_defaults = { -+ 'archive_type': 'auto', - 'domains': [], - 'jobs': 4, - 'keywords': [], -@@ -70,6 +72,7 @@ class SoSCleaner(SoSComponent): - self.from_cmdline = False - if not hasattr(self.opts, 'jobs'): - self.opts.jobs = 4 -+ self.opts.archive_type = 'auto' - self.soslog = logging.getLogger('sos') - self.ui_log = logging.getLogger('sos_ui') - # create the tmp subdir here to avoid a potential race condition -@@ -92,6 +95,17 @@ class SoSCleaner(SoSComponent): - SoSUsernameParser(self.cleaner_mapping, self.opts.usernames) - ] - -+ self.archive_types = [ -+ SoSReportDirectory, -+ SoSReportArchive, -+ SoSCollectorDirectory, -+ SoSCollectorArchive, -+ # make sure these two are always last as they are fallbacks -+ DataDirArchive, -+ TarballArchive -+ ] -+ self.nested_archive = None -+ - self.log_info("Cleaner initialized. From cmdline: %s" - % self.from_cmdline) - -@@ -178,6 +192,11 @@ third party. - ) - clean_grp.add_argument('target', metavar='TARGET', - help='The directory or archive to obfuscate') -+ clean_grp.add_argument('--archive-type', default='auto', -+ choices=['auto', 'report', 'collect', -+ 'data-dir', 'tarball'], -+ help=('Specify what kind of archive the target ' -+ 'was generated as')) - clean_grp.add_argument('--domains', action='extend', default=[], - help='List of domain names to obfuscate') - clean_grp.add_argument('-j', '--jobs', default=4, type=int, -@@ -218,59 +237,28 @@ third party. - - In the event the target path is not an archive, abort. - """ -- if not tarfile.is_tarfile(self.opts.target): -- self.ui_log.error( -- "Invalid target: must be directory or tar archive" -- ) -- self._exit(1) -- -- archive = tarfile.open(self.opts.target) -- self.arc_name = self.opts.target.split('/')[-1].split('.')[:-2][0] -- -- try: -- archive.getmember(os.path.join(self.arc_name, 'sos_logs')) -- except Exception: -- # this is not an sos archive -- self.ui_log.error("Invalid target: not an sos archive") -- self._exit(1) -- -- # see if there are archives within this archive -- nested_archives = [] -- for _file in archive.getmembers(): -- if (re.match('sosreport-.*.tar', _file.name.split('/')[-1]) and not -- (_file.name.endswith(('.md5', '.sha256')))): -- nested_archives.append(_file.name.split('/')[-1]) -- -- if nested_archives: -- self.log_info("Found nested archive(s), extracting top level") -- nested_path = self.extract_archive(archive) -- for arc_file in os.listdir(nested_path): -- if re.match('sosreport.*.tar.*', arc_file): -- if arc_file.endswith(('.md5', '.sha256')): -- continue -- self.report_paths.append(os.path.join(nested_path, -- arc_file)) -- # add the toplevel extracted archive -- self.report_paths.append(nested_path) -+ _arc = None -+ if self.opts.archive_type != 'auto': -+ check_type = self.opts.archive_type.replace('-', '_') -+ for archive in self.archive_types: -+ if archive.type_name == check_type: -+ _arc = archive(self.opts.target, self.tmpdir) - else: -- self.report_paths.append(self.opts.target) -- -- archive.close() -- -- def extract_archive(self, archive): -- """Extract an archive into our tmpdir so that we may inspect it or -- iterate through its contents for obfuscation -- -- Positional arguments: -- -- :param archive: An open TarFile object for the archive -- -- """ -- if not isinstance(archive, tarfile.TarFile): -- archive = tarfile.open(archive) -- path = os.path.join(self.tmpdir, 'cleaner') -- archive.extractall(path) -- return os.path.join(path, archive.name.split('/')[-1].split('.tar')[0]) -+ for arc in self.archive_types: -+ if arc.check_is_type(self.opts.target): -+ _arc = arc(self.opts.target, self.tmpdir) -+ break -+ if not _arc: -+ return -+ self.report_paths.append(_arc) -+ if _arc.is_nested: -+ self.report_paths.extend(_arc.get_nested_archives()) -+ # We need to preserve the top level archive until all -+ # nested archives are processed -+ self.report_paths.remove(_arc) -+ self.nested_archive = _arc -+ if self.nested_archive: -+ self.nested_archive.ui_name = self.nested_archive.description - - def execute(self): - """SoSCleaner will begin by inspecting the TARGET option to determine -@@ -283,6 +271,7 @@ third party. - be unpacked, cleaned, and repacked and the final top-level archive will - then be repacked as well. - """ -+ self.arc_name = self.opts.target.split('/')[-1].split('.tar')[0] - if self.from_cmdline: - self.print_disclaimer() - self.report_paths = [] -@@ -290,23 +279,11 @@ third party. - self.ui_log.error("Invalid target: no such file or directory %s" - % self.opts.target) - self._exit(1) -- if os.path.isdir(self.opts.target): -- self.arc_name = self.opts.target.split('/')[-1] -- for _file in os.listdir(self.opts.target): -- if _file == 'sos_logs': -- self.report_paths.append(self.opts.target) -- if (_file.startswith('sosreport') and -- (_file.endswith(".tar.gz") or _file.endswith(".tar.xz"))): -- self.report_paths.append(os.path.join(self.opts.target, -- _file)) -- if not self.report_paths: -- self.ui_log.error("Invalid target: not an sos directory") -- self._exit(1) -- else: -- self.inspect_target_archive() -+ -+ self.inspect_target_archive() - - if not self.report_paths: -- self.ui_log.error("No valid sos archives or directories found\n") -+ self.ui_log.error("No valid archives or directories found\n") - self._exit(1) - - # we have at least one valid target to obfuscate -@@ -334,33 +311,7 @@ third party. - - final_path = None - if len(self.completed_reports) > 1: -- # we have an archive of archives, so repack the obfuscated tarball -- arc_name = self.arc_name + '-obfuscated' -- self.setup_archive(name=arc_name) -- for arc in self.completed_reports: -- if arc.is_tarfile: -- arc_dest = self.obfuscate_string( -- arc.final_archive_path.split('/')[-1] -- ) -- self.archive.add_file(arc.final_archive_path, -- dest=arc_dest) -- checksum = self.get_new_checksum(arc.final_archive_path) -- if checksum is not None: -- dname = self.obfuscate_string( -- "checksums/%s.%s" % (arc_dest, self.hash_name) -- ) -- self.archive.add_string(checksum, dest=dname) -- else: -- for dirname, dirs, files in os.walk(arc.archive_path): -- for filename in files: -- if filename.startswith('sosreport'): -- continue -- fname = os.path.join(dirname, filename) -- dnm = self.obfuscate_string( -- fname.split(arc.archive_name)[-1].lstrip('/') -- ) -- self.archive.add_file(fname, dest=dnm) -- arc_path = self.archive.finalize(self.opts.compression_type) -+ arc_path = self.rebuild_nested_archive() - else: - arc = self.completed_reports[0] - arc_path = arc.final_archive_path -@@ -371,8 +322,7 @@ third party. - ) - with open(os.path.join(self.sys_tmp, chksum_name), 'w') as cf: - cf.write(checksum) -- -- self.write_cleaner_log() -+ self.write_cleaner_log() - - final_path = self.obfuscate_string( - os.path.join(self.sys_tmp, arc_path.split('/')[-1]) -@@ -393,6 +343,30 @@ third party. - - self.cleanup() - -+ def rebuild_nested_archive(self): -+ """Handles repacking the nested tarball, now containing only obfuscated -+ copies of the reports, log files, manifest, etc... -+ """ -+ # we have an archive of archives, so repack the obfuscated tarball -+ arc_name = self.arc_name + '-obfuscated' -+ self.setup_archive(name=arc_name) -+ for archive in self.completed_reports: -+ arc_dest = archive.final_archive_path.split('/')[-1] -+ checksum = self.get_new_checksum(archive.final_archive_path) -+ if checksum is not None: -+ dname = "checksums/%s.%s" % (arc_dest, self.hash_name) -+ self.archive.add_string(checksum, dest=dname) -+ for dirn, dirs, files in os.walk(self.nested_archive.extracted_path): -+ for filename in files: -+ fname = os.path.join(dirn, filename) -+ dname = fname.split(self.nested_archive.extracted_path)[-1] -+ dname = dname.lstrip('/') -+ self.archive.add_file(fname, dest=dname) -+ # remove it now so we don't balloon our fs space needs -+ os.remove(fname) -+ self.write_cleaner_log(archive=True) -+ return self.archive.finalize(self.opts.compression_type) -+ - def compile_mapping_dict(self): - """Build a dict that contains each parser's map as a key, with the - contents as that key's value. This will then be written to disk in the -@@ -441,7 +415,7 @@ third party. - self.log_error("Could not update mapping config file: %s" - % err) - -- def write_cleaner_log(self): -+ def write_cleaner_log(self, archive=False): - """When invoked via the command line, the logging from SoSCleaner will - not be added to the archive(s) it processes, so we need to write it - separately to disk -@@ -454,6 +428,10 @@ third party. - for line in self.sos_log_file.readlines(): - logfile.write(line) - -+ if archive: -+ self.obfuscate_file(log_name) -+ self.archive.add_file(log_name, dest="sos_logs/cleaner.log") -+ - def get_new_checksum(self, archive_path): - """Calculate a new checksum for the obfuscated archive, as the previous - checksum will no longer be valid -@@ -481,11 +459,11 @@ third party. - be obfuscated concurrently. - """ - try: -- if len(self.report_paths) > 1: -- msg = ("Found %s total reports to obfuscate, processing up to " -- "%s concurrently\n" -- % (len(self.report_paths), self.opts.jobs)) -- self.ui_log.info(msg) -+ msg = ( -+ "Found %s total reports to obfuscate, processing up to %s " -+ "concurrently\n" % (len(self.report_paths), self.opts.jobs) -+ ) -+ self.ui_log.info(msg) - if self.opts.keep_binary_files: - self.ui_log.warning( - "WARNING: binary files that potentially contain sensitive " -@@ -494,53 +472,67 @@ third party. - pool = ThreadPoolExecutor(self.opts.jobs) - pool.map(self.obfuscate_report, self.report_paths, chunksize=1) - pool.shutdown(wait=True) -+ # finally, obfuscate the nested archive if one exists -+ if self.nested_archive: -+ self._replace_obfuscated_archives() -+ self.obfuscate_report(self.nested_archive) - except KeyboardInterrupt: - self.ui_log.info("Exiting on user cancel") - os._exit(130) - -+ def _replace_obfuscated_archives(self): -+ """When we have a nested archive, we need to rebuild the original -+ archive, which entails replacing the existing archives with their -+ obfuscated counterparts -+ """ -+ for archive in self.completed_reports: -+ os.remove(archive.archive_path) -+ dest = self.nested_archive.extracted_path -+ tarball = archive.final_archive_path.split('/')[-1] -+ dest_name = os.path.join(dest, tarball) -+ shutil.move(archive.final_archive_path, dest) -+ archive.final_archive_path = dest_name -+ - def preload_all_archives_into_maps(self): - """Before doing the actual obfuscation, if we have multiple archives - to obfuscate then we need to preload each of them into the mappings - to ensure that node1 is obfuscated in node2 as well as node2 being - obfuscated in node1's archive. - """ -- self.log_info("Pre-loading multiple archives into obfuscation maps") -+ self.log_info("Pre-loading all archives into obfuscation maps") - for _arc in self.report_paths: -- is_dir = os.path.isdir(_arc) -- if is_dir: -- _arc_name = _arc -- else: -- archive = tarfile.open(_arc) -- _arc_name = _arc.split('/')[-1].split('.tar')[0] -- # for each parser, load the map_prep_file into memory, and then -- # send that for obfuscation. We don't actually obfuscate the file -- # here, do that in the normal archive loop - for _parser in self.parsers: -- if not _parser.prep_map_file: -+ try: -+ pfile = _arc.prep_files[_parser.name.lower().split()[0]] -+ if not pfile: -+ continue -+ except (IndexError, KeyError): - continue -- if isinstance(_parser.prep_map_file, str): -- _parser.prep_map_file = [_parser.prep_map_file] -- for parse_file in _parser.prep_map_file: -- _arc_path = os.path.join(_arc_name, parse_file) -+ if isinstance(pfile, str): -+ pfile = [pfile] -+ for parse_file in pfile: -+ self.log_debug("Attempting to load %s" % parse_file) - try: -- if is_dir: -- _pfile = open(_arc_path, 'r') -- content = _pfile.read() -- else: -- _pfile = archive.extractfile(_arc_path) -- content = _pfile.read().decode('utf-8') -- _pfile.close() -+ content = _arc.get_file_content(parse_file) -+ if not content: -+ continue - if isinstance(_parser, SoSUsernameParser): - _parser.load_usernames_into_map(content) -- for line in content.splitlines(): -- if isinstance(_parser, SoSHostnameParser): -- _parser.load_hostname_into_map(line) -- self.obfuscate_line(line) -+ elif isinstance(_parser, SoSHostnameParser): -+ _parser.load_hostname_into_map( -+ content.splitlines()[0] -+ ) -+ else: -+ for line in content.splitlines(): -+ self.obfuscate_line(line) - except Exception as err: -- self.log_debug("Could not prep %s: %s" -- % (_arc_path, err)) -+ self.log_info( -+ "Could not prepare %s from %s (archive: %s): %s" -+ % (_parser.name, parse_file, _arc.archive_name, -+ err) -+ ) - -- def obfuscate_report(self, report): -+ def obfuscate_report(self, archive): - """Individually handle each archive or directory we've discovered by - running through each file therein. - -@@ -549,17 +541,12 @@ third party. - :param report str: Filepath to the directory or archive - """ - try: -- if not os.access(report, os.W_OK): -- msg = "Insufficient permissions on %s" % report -- self.log_info(msg) -- self.ui_log.error(msg) -- return -- -- archive = SoSObfuscationArchive(report, self.tmpdir) - arc_md = self.cleaner_md.add_section(archive.archive_name) - start_time = datetime.now() - arc_md.add_field('start_time', start_time) -- archive.extract() -+ # don't double extract nested archives -+ if not archive.is_extracted: -+ archive.extract() - archive.report_msg("Beginning obfuscation...") - - file_list = archive.get_file_list() -@@ -586,27 +573,28 @@ third party. - caller=archive.archive_name) - - # if the archive was already a tarball, repack it -- method = archive.get_compression() -- if method: -- archive.report_msg("Re-compressing...") -- try: -- archive.rename_top_dir( -- self.obfuscate_string(archive.archive_name) -- ) -- archive.compress(method) -- except Exception as err: -- self.log_debug("Archive %s failed to compress: %s" -- % (archive.archive_name, err)) -- archive.report_msg("Failed to re-compress archive: %s" -- % err) -- return -+ if not archive.is_nested: -+ method = archive.get_compression() -+ if method: -+ archive.report_msg("Re-compressing...") -+ try: -+ archive.rename_top_dir( -+ self.obfuscate_string(archive.archive_name) -+ ) -+ archive.compress(method) -+ except Exception as err: -+ self.log_debug("Archive %s failed to compress: %s" -+ % (archive.archive_name, err)) -+ archive.report_msg("Failed to re-compress archive: %s" -+ % err) -+ return -+ self.completed_reports.append(archive) - - end_time = datetime.now() - arc_md.add_field('end_time', end_time) - arc_md.add_field('run_time', end_time - start_time) - arc_md.add_field('files_obfuscated', len(archive.file_sub_list)) - arc_md.add_field('total_substitutions', archive.total_sub_count) -- self.completed_reports.append(archive) - rmsg = '' - if archive.removed_file_count: - rmsg = " [removed %s unprocessable files]" -@@ -615,7 +603,7 @@ third party. - - except Exception as err: - self.ui_log.info("Exception while processing %s: %s" -- % (report, err)) -+ % (archive.archive_name, err)) - - def obfuscate_file(self, filename, short_name=None, arc_name=None): - """Obfuscate and individual file, line by line. -@@ -635,6 +623,8 @@ third party. - # the requested file doesn't exist in the archive - return - subs = 0 -+ if not short_name: -+ short_name = filename.split('/')[-1] - if not os.path.islink(filename): - # don't run the obfuscation on the link, but on the actual file - # at some other point. -@@ -745,3 +735,5 @@ third party. - for parser in self.parsers: - _sec = parse_sec.add_section(parser.name.replace(' ', '_').lower()) - _sec.add_field('entries', len(parser.mapping.dataset.keys())) -+ -+# vim: set et ts=4 sw=4 : -diff --git a/sos/cleaner/obfuscation_archive.py b/sos/cleaner/archives/__init__.py -similarity index 81% -rename from sos/cleaner/obfuscation_archive.py -rename to sos/cleaner/archives/__init__.py -index ea0b7012..795c5a78 100644 ---- a/sos/cleaner/obfuscation_archive.py -+++ b/sos/cleaner/archives/__init__.py -@@ -40,6 +40,10 @@ class SoSObfuscationArchive(): - file_sub_list = [] - total_sub_count = 0 - removed_file_count = 0 -+ type_name = 'undetermined' -+ description = 'undetermined' -+ is_nested = False -+ prep_files = {} - - def __init__(self, archive_path, tmpdir): - self.archive_path = archive_path -@@ -50,7 +54,43 @@ class SoSObfuscationArchive(): - self.soslog = logging.getLogger('sos') - self.ui_log = logging.getLogger('sos_ui') - self.skip_list = self._load_skip_list() -- self.log_info("Loaded %s as an archive" % self.archive_path) -+ self.is_extracted = False -+ self._load_self() -+ self.archive_root = '' -+ self.log_info( -+ "Loaded %s as type %s" -+ % (self.archive_path, self.description) -+ ) -+ -+ @classmethod -+ def check_is_type(cls, arc_path): -+ """Check if the archive is a well-known type we directly support""" -+ return False -+ -+ def _load_self(self): -+ if self.is_tarfile: -+ self.tarobj = tarfile.open(self.archive_path) -+ -+ def get_nested_archives(self): -+ """Return a list of ObfuscationArchives that represent additional -+ archives found within the target archive. For example, an archive from -+ `sos collect` will return a list of ``SoSReportArchive`` objects. -+ -+ This should be overridden by individual types of ObfuscationArchive's -+ """ -+ return [] -+ -+ def get_archive_root(self): -+ """Set the root path for the archive that should be prepended to any -+ filenames given to methods in this class. -+ """ -+ if self.is_tarfile: -+ toplevel = self.tarobj.firstmember -+ if toplevel.isdir(): -+ return toplevel.name -+ else: -+ return os.sep -+ return os.path.abspath(self.archive_path) - - def report_msg(self, msg): - """Helper to easily format ui messages on a per-report basis""" -@@ -96,10 +136,42 @@ class SoSObfuscationArchive(): - os.remove(full_fname) - self.removed_file_count += 1 - -- def extract(self): -+ def format_file_name(self, fname): -+ """Based on the type of archive we're dealing with, do whatever that -+ archive requires to a provided **relative** filepath to be able to -+ access it within the archive -+ """ -+ if not self.is_extracted: -+ if not self.archive_root: -+ self.archive_root = self.get_archive_root() -+ return os.path.join(self.archive_root, fname) -+ else: -+ return os.path.join(self.extracted_path, fname) -+ -+ def get_file_content(self, fname): -+ """Return the content from the specified fname. Particularly useful for -+ tarball-type archives so we can retrieve prep file contents prior to -+ extracting the entire archive -+ """ -+ if self.is_extracted is False and self.is_tarfile: -+ filename = self.format_file_name(fname) -+ try: -+ return self.tarobj.extractfile(filename).read().decode('utf-8') -+ except KeyError: -+ self.log_debug( -+ "Unable to retrieve %s: no such file in archive" % fname -+ ) -+ return '' -+ else: -+ with open(self.format_file_name(fname), 'r') as to_read: -+ return to_read.read() -+ -+ def extract(self, quiet=False): - if self.is_tarfile: -- self.report_msg("Extracting...") -+ if not quiet: -+ self.report_msg("Extracting...") - self.extracted_path = self.extract_self() -+ self.is_extracted = True - else: - self.extracted_path = self.archive_path - # if we're running as non-root (e.g. collector), then we can have a -@@ -317,3 +389,5 @@ class SoSObfuscationArchive(): - return False - except UnicodeDecodeError: - return True -+ -+# vim: set et ts=4 sw=4 : -diff --git a/sos/cleaner/archives/generic.py b/sos/cleaner/archives/generic.py -new file mode 100644 -index 00000000..2ce6f09b ---- /dev/null -+++ b/sos/cleaner/archives/generic.py -@@ -0,0 +1,52 @@ -+# Copyright 2020 Red Hat, Inc. Jake Hunsaker -+ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+ -+from sos.cleaner.archives import SoSObfuscationArchive -+ -+import os -+import tarfile -+ -+ -+class DataDirArchive(SoSObfuscationArchive): -+ """A plain directory on the filesystem that is not directly associated with -+ any known or supported collection utility -+ """ -+ -+ type_name = 'data_dir' -+ description = 'unassociated directory' -+ -+ @classmethod -+ def check_is_type(cls, arc_path): -+ return os.path.isdir(arc_path) -+ -+ def set_archive_root(self): -+ return os.path.abspath(self.archive_path) -+ -+ -+class TarballArchive(SoSObfuscationArchive): -+ """A generic tar archive that is not associated with any known or supported -+ collection utility -+ """ -+ -+ type_name = 'tarball' -+ description = 'unassociated tarball' -+ -+ @classmethod -+ def check_is_type(cls, arc_path): -+ try: -+ return tarfile.is_tarfile(arc_path) -+ except Exception: -+ return False -+ -+ def set_archive_root(self): -+ if self.tarobj.firstmember.isdir(): -+ return self.tarobj.firstmember.name -+ return '' -diff --git a/sos/cleaner/archives/sos.py b/sos/cleaner/archives/sos.py -new file mode 100644 -index 00000000..4401d710 ---- /dev/null -+++ b/sos/cleaner/archives/sos.py -@@ -0,0 +1,106 @@ -+# Copyright 2021 Red Hat, Inc. Jake Hunsaker -+ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+ -+from sos.cleaner.archives import SoSObfuscationArchive -+ -+import os -+import tarfile -+ -+ -+class SoSReportArchive(SoSObfuscationArchive): -+ """This is the class representing an sos report, or in other words the -+ type the archive the SoS project natively generates -+ """ -+ -+ type_name = 'report' -+ description = 'sos report archive' -+ prep_files = { -+ 'hostname': 'sos_commands/host/hostname', -+ 'ip': 'sos_commands/networking/ip_-o_addr', -+ 'mac': 'sos_commands/networking/ip_-d_address', -+ 'username': [ -+ 'sos_commands/login/lastlog_-u_1000-60000', -+ 'sos_commands/login/lastlog_-u_60001-65536', -+ 'sos_commands/login/lastlog_-u_65537-4294967295', -+ # AD users will be reported here, but favor the lastlog files since -+ # those will include local users who have not logged in -+ 'sos_commands/login/last' -+ ] -+ } -+ -+ @classmethod -+ def check_is_type(cls, arc_path): -+ try: -+ return tarfile.is_tarfile(arc_path) and 'sosreport-' in arc_path -+ except Exception: -+ return False -+ -+ -+class SoSReportDirectory(SoSReportArchive): -+ """This is the archive class representing a build directory, or in other -+ words what `sos report --clean` will end up using for in-line obfuscation -+ """ -+ -+ type_name = 'report_dir' -+ description = 'sos report directory' -+ -+ @classmethod -+ def check_is_type(cls, arc_path): -+ if os.path.isdir(arc_path): -+ return 'sos_logs' in os.listdir(arc_path) -+ return False -+ -+ -+class SoSCollectorArchive(SoSObfuscationArchive): -+ """Archive class representing the tarball created by ``sos collect``. It -+ will not provide prep files on its own, however it will provide a list -+ of SoSReportArchive's which will then be used to prep the parsers -+ """ -+ -+ type_name = 'collect' -+ description = 'sos collect tarball' -+ is_nested = True -+ -+ @classmethod -+ def check_is_type(cls, arc_path): -+ try: -+ return (tarfile.is_tarfile(arc_path) and 'sos-collect' in arc_path) -+ except Exception: -+ return False -+ -+ def get_nested_archives(self): -+ self.extract(quiet=True) -+ _path = self.extracted_path -+ archives = [] -+ for fname in os.listdir(_path): -+ arc_name = os.path.join(_path, fname) -+ if 'sosreport-' in fname and tarfile.is_tarfile(arc_name): -+ archives.append(SoSReportArchive(arc_name, self.tmpdir)) -+ return archives -+ -+ -+class SoSCollectorDirectory(SoSCollectorArchive): -+ """The archive class representing the temp directory used by ``sos -+ collect`` when ``--clean`` is used during runtime. -+ """ -+ -+ type_name = 'collect_dir' -+ description = 'sos collect directory' -+ -+ @classmethod -+ def check_is_type(cls, arc_path): -+ if os.path.isdir(arc_path): -+ for fname in os.listdir(arc_path): -+ if 'sos-collector-' in fname: -+ return True -+ return False -+ -+# vim: set et ts=4 sw=4 : -diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py -index af6e375e..e62fd938 100644 ---- a/sos/cleaner/parsers/__init__.py -+++ b/sos/cleaner/parsers/__init__.py -@@ -37,11 +37,6 @@ class SoSCleanerParser(): - :cvar map_file_key: The key in the ``map_file`` to read when loading - previous obfuscation matches - :vartype map_file_key: ``str`` -- -- -- :cvar prep_map_file: File to read from an archive to pre-seed the map with -- matches. E.G. ip_addr for loading IP addresses -- :vartype prep_map_fie: ``str`` - """ - - name = 'Undefined Parser' -@@ -49,7 +44,6 @@ class SoSCleanerParser(): - skip_line_patterns = [] - skip_files = [] - map_file_key = 'unset' -- prep_map_file = [] - - def __init__(self, config={}): - if self.map_file_key in config: -diff --git a/sos/cleaner/parsers/hostname_parser.py b/sos/cleaner/parsers/hostname_parser.py -index 71e13d3f..daa76a62 100644 ---- a/sos/cleaner/parsers/hostname_parser.py -+++ b/sos/cleaner/parsers/hostname_parser.py -@@ -16,7 +16,6 @@ class SoSHostnameParser(SoSCleanerParser): - - name = 'Hostname Parser' - map_file_key = 'hostname_map' -- prep_map_file = 'sos_commands/host/hostname' - regex_patterns = [ - r'(((\b|_)[a-zA-Z0-9-\.]{1,200}\.[a-zA-Z]{1,63}(\b|_)))' - ] -diff --git a/sos/cleaner/parsers/ip_parser.py b/sos/cleaner/parsers/ip_parser.py -index 525139e8..71d38be8 100644 ---- a/sos/cleaner/parsers/ip_parser.py -+++ b/sos/cleaner/parsers/ip_parser.py -@@ -41,7 +41,6 @@ class SoSIPParser(SoSCleanerParser): - ] - - map_file_key = 'ip_map' -- prep_map_file = 'sos_commands/networking/ip_-o_addr' - - def __init__(self, config): - self.mapping = SoSIPMap() -diff --git a/sos/cleaner/parsers/keyword_parser.py b/sos/cleaner/parsers/keyword_parser.py -index 68de3727..694c6073 100644 ---- a/sos/cleaner/parsers/keyword_parser.py -+++ b/sos/cleaner/parsers/keyword_parser.py -@@ -20,7 +20,6 @@ class SoSKeywordParser(SoSCleanerParser): - - name = 'Keyword Parser' - map_file_key = 'keyword_map' -- prep_map_file = '' - - def __init__(self, config, keywords=None, keyword_file=None): - self.mapping = SoSKeywordMap() -diff --git a/sos/cleaner/parsers/mac_parser.py b/sos/cleaner/parsers/mac_parser.py -index 7ca80b8d..c74288cf 100644 ---- a/sos/cleaner/parsers/mac_parser.py -+++ b/sos/cleaner/parsers/mac_parser.py -@@ -30,7 +30,6 @@ class SoSMacParser(SoSCleanerParser): - '534f:53' - ) - map_file_key = 'mac_map' -- prep_map_file = 'sos_commands/networking/ip_-d_address' - - def __init__(self, config): - self.mapping = SoSMacMap() -diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py -index b142e371..35377a31 100644 ---- a/sos/cleaner/parsers/username_parser.py -+++ b/sos/cleaner/parsers/username_parser.py -@@ -25,14 +25,6 @@ class SoSUsernameParser(SoSCleanerParser): - - name = 'Username Parser' - map_file_key = 'username_map' -- prep_map_file = [ -- 'sos_commands/login/lastlog_-u_1000-60000', -- 'sos_commands/login/lastlog_-u_60001-65536', -- 'sos_commands/login/lastlog_-u_65537-4294967295', -- # AD users will be reported here, but favor the lastlog files since -- # those will include local users who have not logged in -- 'sos_commands/login/last' -- ] - regex_patterns = [] - skip_list = [ - 'core', -diff --git a/tests/cleaner_tests/existing_archive.py b/tests/cleaner_tests/existing_archive.py -index 0eaf6c8d..e13d1cae 100644 ---- a/tests/cleaner_tests/existing_archive.py -+++ b/tests/cleaner_tests/existing_archive.py -@@ -28,6 +28,13 @@ class ExistingArchiveCleanTest(StageTwoReportTest): - def test_obfuscation_log_created(self): - self.assertFileExists(os.path.join(self.tmpdir, '%s-obfuscation.log' % ARCHIVE)) - -+ def test_archive_type_correct(self): -+ with open(os.path.join(self.tmpdir, '%s-obfuscation.log' % ARCHIVE), 'r') as log: -+ for line in log: -+ if "Loaded %s" % ARCHIVE in line: -+ assert 'as type sos report archive' in line, "Incorrect archive type detected: %s" % line -+ break -+ - def test_from_cmdline_logged(self): - with open(os.path.join(self.tmpdir, '%s-obfuscation.log' % ARCHIVE), 'r') as log: - for line in log: -diff --git a/tests/cleaner_tests/full_report_run.py b/tests/cleaner_tests/full_report_run.py -index 3b28e7a2..2de54946 100644 ---- a/tests/cleaner_tests/full_report_run.py -+++ b/tests/cleaner_tests/full_report_run.py -@@ -35,6 +35,9 @@ class FullCleanTest(StageTwoReportTest): - def test_tarball_named_obfuscated(self): - self.assertTrue('obfuscated' in self.archive) - -+ def test_archive_type_correct(self): -+ self.assertSosLogContains('Loaded .* as type sos report directory') -+ - def test_hostname_not_in_any_file(self): - host = self.sysinfo['pre']['networking']['hostname'] - # much faster to just use grep here -diff --git a/tests/cleaner_tests/report_with_mask.py b/tests/cleaner_tests/report_with_mask.py -index 4f94ba33..08e873d4 100644 ---- a/tests/cleaner_tests/report_with_mask.py -+++ b/tests/cleaner_tests/report_with_mask.py -@@ -31,6 +31,9 @@ class ReportWithMask(StageOneReportTest): - def test_tarball_named_obfuscated(self): - self.assertTrue('obfuscated' in self.archive) - -+ def test_archive_type_correct(self): -+ self.assertSosLogContains('Loaded .* as type sos report directory') -+ - def test_localhost_was_obfuscated(self): - self.assertFileHasContent('/etc/hostname', 'host0') - --- -2.31.1 - -From 9b119f860eaec089f7ef884ff39c42589a662994 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 1 Sep 2021 00:34:04 -0400 -Subject: [PATCH] [hostname_map] Add a catch for single-character hostnames - -If a log file was truncated at a specific boundary in a string of the -FQDN of the host such that we only get a couple characters before the -rest of the domain, we would previously bodly replace all instances of -that character with the obfuscated short name; not very helpful. - -Instead, don't sanitize the short name if this happens and instead -obfuscate the whole FQDN as 'unknown.example.com'. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/mappings/hostname_map.py | 9 ++++++++- - 1 file changed, 8 insertions(+), 1 deletion(-) - -diff --git a/sos/cleaner/mappings/hostname_map.py b/sos/cleaner/mappings/hostname_map.py -index d4b2c88e..e70a5530 100644 ---- a/sos/cleaner/mappings/hostname_map.py -+++ b/sos/cleaner/mappings/hostname_map.py -@@ -184,7 +184,14 @@ class SoSHostnameMap(SoSMap): - hostname = host[0] - domain = host[1:] - # obfuscate the short name -- ob_hostname = self.sanitize_short_name(hostname) -+ if len(hostname) > 2: -+ ob_hostname = self.sanitize_short_name(hostname) -+ else: -+ # by best practice it appears the host part of the fqdn was cut -+ # off due to some form of truncating, as such don't obfuscate -+ # short strings that are likely to throw off obfuscation of -+ # unrelated bits and paths -+ ob_hostname = 'unknown' - ob_domain = self.sanitize_domain(domain) - self.dataset[item] = ob_domain - return '.'.join([ob_hostname, ob_domain]) --- -2.31.1 - -From f3f3e763d7c31b7b7cafdf8dd4dab87056fb7696 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 1 Sep 2021 15:54:55 -0400 -Subject: [PATCH] [cleaner] Add support for Insights client archives - -Adds a new type of `SoSObfuscationArchive` to add support for -obfuscating archives generated by the Insights project. - -Signed-off-by: Jake Hunsaker ---- - man/en/sos-clean.1 | 1 + - sos/cleaner/__init__.py | 4 ++- - sos/cleaner/archives/insights.py | 42 ++++++++++++++++++++++++++++++++ - 3 files changed, 46 insertions(+), 1 deletion(-) - create mode 100644 sos/cleaner/archives/insights.py - -diff --git a/man/en/sos-clean.1 b/man/en/sos-clean.1 -index 54026713..358ec0cb 100644 ---- a/man/en/sos-clean.1 -+++ b/man/en/sos-clean.1 -@@ -105,6 +105,7 @@ The following are accepted values for this option: - \fBauto\fR Automatically detect the archive type - \fBreport\fR An archive generated by \fBsos report\fR - \fBcollect\fR An archive generated by \fBsos collect\fR -+ \fBinsights\fR An archive generated by the \fBinsights-client\fR package - - The following may also be used, however note that these do not attempt to pre-load - any information from the archives into the parsers. This means that, among other limitations, -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index 6d2eb483..3e08aa28 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -29,6 +29,7 @@ from sos.cleaner.archives.sos import (SoSReportArchive, SoSReportDirectory, - SoSCollectorArchive, - SoSCollectorDirectory) - from sos.cleaner.archives.generic import DataDirArchive, TarballArchive -+from sos.cleaner.archives.insights import InsightsArchive - from sos.utilities import get_human_readable - from textwrap import fill - -@@ -100,6 +101,7 @@ class SoSCleaner(SoSComponent): - SoSReportArchive, - SoSCollectorDirectory, - SoSCollectorArchive, -+ InsightsArchive, - # make sure these two are always last as they are fallbacks - DataDirArchive, - TarballArchive -@@ -194,7 +196,7 @@ third party. - help='The directory or archive to obfuscate') - clean_grp.add_argument('--archive-type', default='auto', - choices=['auto', 'report', 'collect', -- 'data-dir', 'tarball'], -+ 'insights', 'data-dir', 'tarball'], - help=('Specify what kind of archive the target ' - 'was generated as')) - clean_grp.add_argument('--domains', action='extend', default=[], -diff --git a/sos/cleaner/archives/insights.py b/sos/cleaner/archives/insights.py -new file mode 100644 -index 00000000..dab48b16 ---- /dev/null -+++ b/sos/cleaner/archives/insights.py -@@ -0,0 +1,42 @@ -+# Copyright 2021 Red Hat, Inc. Jake Hunsaker -+ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+ -+from sos.cleaner.archives import SoSObfuscationArchive -+ -+import tarfile -+ -+ -+class InsightsArchive(SoSObfuscationArchive): -+ """This class represents archives generated by the insights-client utility -+ for RHEL systems. -+ """ -+ -+ type_name = 'insights' -+ description = 'insights-client archive' -+ -+ prep_files = { -+ 'hostname': 'data/insights_commands/hostname_-f', -+ 'ip': 'data/insights_commands/ip_addr', -+ 'mac': 'data/insights_commands/ip_addr' -+ } -+ -+ @classmethod -+ def check_is_type(cls, arc_path): -+ try: -+ return tarfile.is_tarfile(arc_path) and 'insights-' in arc_path -+ except Exception: -+ return False -+ -+ def get_archive_root(self): -+ top = self.archive_path.split('/')[-1].split('.tar')[0] -+ if self.tarobj.firstmember.name == '.': -+ top = './' + top -+ return top --- -2.31.1 - -From 9639dc3d240076b55f2a1d04b43ea42bebd09215 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 16 Nov 2021 17:50:42 -0500 -Subject: [PATCH] [clean,hostname_parser] Source /etc/hosts for obfuscation - -Up until now, our sourcing of hostnames/domains for obfuscation has been -dependent upon the output of the `hostname` command. However, some -scenarios have come up where sourcing `/etc/hosts` is advantageous for -several reasons: - -First, if `hostname` output is unavailable, this provides a fallback -measure. - -Second, `/etc/hosts` is a common place to have short names defined which -would otherwise not be detected (or at the very least would result in a -race condition based on where/if the short name was elsewhere able to be -gleaned from an FQDN), thus leaving the potential for unobfuscated data -in an archive. - -Due to both the nature of hostname obfuscation and the malleable syntax -of `/etc/hosts`, the parsing of this file needs special handling not -covered by our more generic parsing and obfuscation methods. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/__init__.py | 11 ++++++++--- - sos/cleaner/archives/sos.py | 5 ++++- - sos/cleaner/parsers/hostname_parser.py | 19 +++++++++++++++++++ - 3 files changed, 31 insertions(+), 4 deletions(-) - -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index ed461a8f..3f530d44 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -523,9 +523,14 @@ third party. - if isinstance(_parser, SoSUsernameParser): - _parser.load_usernames_into_map(content) - elif isinstance(_parser, SoSHostnameParser): -- _parser.load_hostname_into_map( -- content.splitlines()[0] -- ) -+ if 'hostname' in parse_file: -+ _parser.load_hostname_into_map( -+ content.splitlines()[0] -+ ) -+ elif 'etc/hosts' in parse_file: -+ _parser.load_hostname_from_etc_hosts( -+ content -+ ) - else: - for line in content.splitlines(): - self.obfuscate_line(line) -diff --git a/sos/cleaner/archives/sos.py b/sos/cleaner/archives/sos.py -index 4401d710..f8720c88 100644 ---- a/sos/cleaner/archives/sos.py -+++ b/sos/cleaner/archives/sos.py -@@ -23,7 +23,10 @@ class SoSReportArchive(SoSObfuscationArchive): - type_name = 'report' - description = 'sos report archive' - prep_files = { -- 'hostname': 'sos_commands/host/hostname', -+ 'hostname': [ -+ 'sos_commands/host/hostname', -+ 'etc/hosts' -+ ], - 'ip': 'sos_commands/networking/ip_-o_addr', - 'mac': 'sos_commands/networking/ip_-d_address', - 'username': [ -diff --git a/sos/cleaner/parsers/hostname_parser.py b/sos/cleaner/parsers/hostname_parser.py -index daa76a62..0a733bee 100644 ---- a/sos/cleaner/parsers/hostname_parser.py -+++ b/sos/cleaner/parsers/hostname_parser.py -@@ -61,6 +61,25 @@ class SoSHostnameParser(SoSCleanerParser): - self.mapping.add(high_domain) - self.mapping.add(hostname_string) - -+ def load_hostname_from_etc_hosts(self, content): -+ """Parse an archive's copy of /etc/hosts, which requires handling that -+ is separate from the output of the `hostname` command. Just like -+ load_hostname_into_map(), this has to be done explicitly and we -+ cannot rely upon the more generic methods to do this reliably. -+ """ -+ lines = content.splitlines() -+ for line in lines: -+ if line.startswith('#') or 'localhost' in line: -+ continue -+ hostln = line.split()[1:] -+ for host in hostln: -+ if len(host.split('.')) == 1: -+ # only generate a mapping for fqdns but still record the -+ # short name here for later obfuscation with parse_line() -+ self.short_names.append(host) -+ else: -+ self.mapping.add(host) -+ - def parse_line(self, line): - """Override the default parse_line() method to also check for the - shortname of the host derived from the hostname. --- -2.31.1 - -From c1680226b53452b18f27f2e76c3e0e03e521f935 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 17 Nov 2021 13:11:33 -0500 -Subject: [PATCH] [clean, hostname] Fix unintentionally case sensitive - shortname handling - -It was discovered that our extra handling for shortnames was -unintentionally case sensitive. Fix this to ensure that shortnames are -obfuscated regardless of case in all collected text. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/mappings/hostname_map.py | 6 +++--- - sos/cleaner/parsers/hostname_parser.py | 8 +++++--- - tests/cleaner_tests/full_report_run.py | 21 ++++++++++++++++++++- - 3 files changed, 28 insertions(+), 7 deletions(-) - -diff --git a/sos/cleaner/mappings/hostname_map.py b/sos/cleaner/mappings/hostname_map.py -index e70a5530..0fe78fb1 100644 ---- a/sos/cleaner/mappings/hostname_map.py -+++ b/sos/cleaner/mappings/hostname_map.py -@@ -169,13 +169,13 @@ class SoSHostnameMap(SoSMap): - - def sanitize_item(self, item): - host = item.split('.') -- if all([h.isupper() for h in host]): -+ if len(host) > 1 and all([h.isupper() for h in host]): - # by convention we have just a domain - _host = [h.lower() for h in host] - return self.sanitize_domain(_host).upper() - if len(host) == 1: - # we have a shortname for a host -- return self.sanitize_short_name(host[0]) -+ return self.sanitize_short_name(host[0].lower()) - if len(host) == 2: - # we have just a domain name, e.g. example.com - return self.sanitize_domain(host) -@@ -185,7 +185,7 @@ class SoSHostnameMap(SoSMap): - domain = host[1:] - # obfuscate the short name - if len(hostname) > 2: -- ob_hostname = self.sanitize_short_name(hostname) -+ ob_hostname = self.sanitize_short_name(hostname.lower()) - else: - # by best practice it appears the host part of the fqdn was cut - # off due to some form of truncating, as such don't obfuscate -diff --git a/sos/cleaner/parsers/hostname_parser.py b/sos/cleaner/parsers/hostname_parser.py -index 0a733bee..7fd0e698 100644 ---- a/sos/cleaner/parsers/hostname_parser.py -+++ b/sos/cleaner/parsers/hostname_parser.py -@@ -8,6 +8,8 @@ - # - # See the LICENSE file in the source distribution for further information. - -+import re -+ - from sos.cleaner.parsers import SoSCleanerParser - from sos.cleaner.mappings.hostname_map import SoSHostnameMap - -@@ -91,9 +93,9 @@ class SoSHostnameParser(SoSCleanerParser): - """ - if search in self.mapping.skip_keys: - return ln, count -- if search in ln: -- count += ln.count(search) -- ln = ln.replace(search, self.mapping.get(repl or search)) -+ _reg = re.compile(search, re.I) -+ if _reg.search(ln): -+ return _reg.subn(self.mapping.get(repl or search), ln) - return ln, count - - count = 0 -diff --git a/tests/cleaner_tests/full_report_run.py b/tests/cleaner_tests/full_report_run.py -index 2de54946..0b23acaf 100644 ---- a/tests/cleaner_tests/full_report_run.py -+++ b/tests/cleaner_tests/full_report_run.py -@@ -26,6 +26,24 @@ class FullCleanTest(StageTwoReportTest): - # replace with an empty placeholder, make sure that this test case is not - # influenced by previous clean runs - files = ['/etc/sos/cleaner/default_mapping'] -+ packages = { -+ 'rhel': ['python3-systemd'], -+ 'ubuntu': ['python3-systemd'] -+ } -+ -+ def pre_sos_setup(self): -+ # ensure that case-insensitive matching of FQDNs and shortnames work -+ from systemd import journal -+ from socket import gethostname -+ host = gethostname() -+ short = host.split('.')[0] -+ sosfd = journal.stream('sos-testing') -+ sosfd.write( -+ "This is a test line from sos clean testing. The hostname %s " -+ "should not appear, nor should %s in an obfuscated archive. The " -+ "shortnames of %s and %s should also not appear." -+ % (host.lower(), host.upper(), short.lower(), short.upper()) -+ ) - - def test_private_map_was_generated(self): - self.assertOutputContains('A mapping of obfuscated elements is available at') -@@ -40,8 +58,9 @@ class FullCleanTest(StageTwoReportTest): - - def test_hostname_not_in_any_file(self): - host = self.sysinfo['pre']['networking']['hostname'] -+ short = host.split('.')[0] - # much faster to just use grep here -- content = self.grep_for_content(host) -+ content = self.grep_for_content(host) + self.grep_for_content(short) - if not content: - assert True - else: --- -2.31.1 - -From aaeb8cb57ed55598ab744b96d4f127aedebcb292 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 21 Sep 2021 15:23:20 -0400 -Subject: [PATCH] [build] Add archives to setup.py packages - -Adds the newly abstracted `sos.cleaner.archives` package to `setup.py` -so that manual builds will properly include it. - -Signed-off-by: Jake Hunsaker ---- - setup.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/setup.py b/setup.py -index 1e8d8e2dc5..7653b59de3 100644 ---- a/setup.py -+++ b/setup.py -@@ -102,7 +102,7 @@ def copy_file (self, filename, dirname): - 'sos.policies.package_managers', 'sos.policies.init_systems', - 'sos.report', 'sos.report.plugins', 'sos.collector', - 'sos.collector.clusters', 'sos.cleaner', 'sos.cleaner.mappings', -- 'sos.cleaner.parsers' -+ 'sos.cleaner.parsers', 'sos.cleaner.archives' - ], - cmdclass=cmdclass, - command_options=command_options, --- -2.31.1 - -From ba3528230256429a4394f155a9ca1fdb91cf3560 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 30 Nov 2021 12:46:34 -0500 -Subject: [PATCH 1/2] [hostname] Simplify case matching for domains - -Instead of special handling all uppercase domain conventions, use our -normal flow for obfuscation and just match the casing at the end of the -sanitization routine. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/mappings/hostname_map.py | 14 ++++++++------ - 1 file changed, 8 insertions(+), 6 deletions(-) - -diff --git a/sos/cleaner/mappings/hostname_map.py b/sos/cleaner/mappings/hostname_map.py -index 0fe78fb1..5cd8e985 100644 ---- a/sos/cleaner/mappings/hostname_map.py -+++ b/sos/cleaner/mappings/hostname_map.py -@@ -169,16 +169,15 @@ class SoSHostnameMap(SoSMap): - - def sanitize_item(self, item): - host = item.split('.') -- if len(host) > 1 and all([h.isupper() for h in host]): -- # by convention we have just a domain -- _host = [h.lower() for h in host] -- return self.sanitize_domain(_host).upper() - if len(host) == 1: - # we have a shortname for a host - return self.sanitize_short_name(host[0].lower()) - if len(host) == 2: - # we have just a domain name, e.g. example.com -- return self.sanitize_domain(host) -+ dname = self.sanitize_domain(host) -+ if all([h.isupper() for h in host]): -+ dname = dname.upper() -+ return dname - if len(host) > 2: - # we have an FQDN, e.g. foo.example.com - hostname = host[0] -@@ -194,7 +193,10 @@ class SoSHostnameMap(SoSMap): - ob_hostname = 'unknown' - ob_domain = self.sanitize_domain(domain) - self.dataset[item] = ob_domain -- return '.'.join([ob_hostname, ob_domain]) -+ _fqdn = '.'.join([ob_hostname, ob_domain]) -+ if all([h.isupper() for h in host]): -+ _fqdn = _fqdn.upper() -+ return _fqdn - - def sanitize_short_name(self, hostname): - """Obfuscate the short name of the host with an incremented counter --- -2.31.1 - - -From 189586728de22dd55122c1f7e06b19590f9a788f Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 30 Nov 2021 12:47:58 -0500 -Subject: [PATCH 2/2] [username] Improve username sourcing and remove case - sensitivity - -First, don't skip the first line of `last` output, and instead add the -header from lastlog to the skip list. Additionally, add -`/etc/cron.allow` and `/etc/cron.deny` as sources for usernames that -might not appear in other locations in certain environments. - -Also, make matching and replacement case insensitive. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/archives/sos.py | 4 +++- - sos/cleaner/mappings/username_map.py | 2 +- - sos/cleaner/parsers/username_parser.py | 14 +++++++++----- - 3 files changed, 13 insertions(+), 7 deletions(-) - -diff --git a/sos/cleaner/archives/sos.py b/sos/cleaner/archives/sos.py -index f8720c88..12766496 100644 ---- a/sos/cleaner/archives/sos.py -+++ b/sos/cleaner/archives/sos.py -@@ -35,7 +35,9 @@ class SoSReportArchive(SoSObfuscationArchive): - 'sos_commands/login/lastlog_-u_65537-4294967295', - # AD users will be reported here, but favor the lastlog files since - # those will include local users who have not logged in -- 'sos_commands/login/last' -+ 'sos_commands/login/last', -+ 'etc/cron.allow', -+ 'etc/cron.deny' - ] - } - -diff --git a/sos/cleaner/mappings/username_map.py b/sos/cleaner/mappings/username_map.py -index cdbf36fe..7ecccd7b 100644 ---- a/sos/cleaner/mappings/username_map.py -+++ b/sos/cleaner/mappings/username_map.py -@@ -33,5 +33,5 @@ class SoSUsernameMap(SoSMap): - ob_name = "obfuscateduser%s" % self.name_count - self.name_count += 1 - if ob_name in self.dataset.values(): -- return self.sanitize_item(username) -+ return self.sanitize_item(username.lower()) - return ob_name -diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py -index 35377a31..229c7de4 100644 ---- a/sos/cleaner/parsers/username_parser.py -+++ b/sos/cleaner/parsers/username_parser.py -@@ -8,6 +8,7 @@ - # - # See the LICENSE file in the source distribution for further information. - -+import re - - from sos.cleaner.parsers import SoSCleanerParser - from sos.cleaner.mappings.username_map import SoSUsernameMap -@@ -34,6 +35,7 @@ class SoSUsernameParser(SoSCleanerParser): - 'reboot', - 'root', - 'ubuntu', -+ 'username', - 'wtmp' - ] - -@@ -47,12 +49,12 @@ class SoSUsernameParser(SoSCleanerParser): - this parser, we need to override the initial parser prepping here. - """ - users = set() -- for line in content.splitlines()[1:]: -+ for line in content.splitlines(): - try: - user = line.split()[0] - except Exception: - continue -- if user in self.skip_list: -+ if user.lower() in self.skip_list: - continue - users.add(user) - for each in users: -@@ -61,7 +63,9 @@ class SoSUsernameParser(SoSCleanerParser): - def parse_line(self, line): - count = 0 - for username in sorted(self.mapping.dataset.keys(), reverse=True): -- if username in line: -- count = line.count(username) -- line = line.replace(username, self.mapping.get(username)) -+ _reg = re.compile(username, re.I) -+ if _reg.search(line): -+ line, count = _reg.subn( -+ self.mapping.get(username.lower()), line -+ ) - return line, count --- -2.31.1 - -From cafd0f3a52436a3966576e7db21e5dd17c06f0cc Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Sun, 12 Dec 2021 11:10:46 -0500 -Subject: [PATCH] [hostname] Fix edge case for new hosts in a known subdomain - -Fixes an edge case that would cause us to at first not recognize that a -given hostname string is a new host in a known subdomain, but then on -the obfuscation attempt properly recognize it as such and result in an -incomplete obfuscation. - -This was mostly triggered by specific patterns for build hosts within -`sos_commands/rpm/package-data`. With this refined check, these types of -matches are properly obfuscated. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/mappings/hostname_map.py | 9 +++++---- - 1 file changed, 5 insertions(+), 4 deletions(-) - -diff --git a/sos/cleaner/mappings/hostname_map.py b/sos/cleaner/mappings/hostname_map.py -index 5cd8e9857..33b0e6c80 100644 ---- a/sos/cleaner/mappings/hostname_map.py -+++ b/sos/cleaner/mappings/hostname_map.py -@@ -129,7 +129,7 @@ def get(self, item): - item = item[0:-1] - if not self.domain_name_in_loaded_domains(item.lower()): - return item -- if item.endswith(('.yaml', '.yml', '.crt', '.key', '.pem')): -+ if item.endswith(('.yaml', '.yml', '.crt', '.key', '.pem', '.log')): - ext = '.' + item.split('.')[-1] - item = item.replace(ext, '') - suffix += ext -@@ -148,7 +148,8 @@ def get(self, item): - if len(_test) == 1 or not _test[0]: - # does not match existing obfuscation - continue -- elif _test[0].endswith('.') and not _host_substr: -+ elif not _host_substr and (_test[0].endswith('.') or -+ item.endswith(_existing)): - # new hostname in known domain - final = super(SoSHostnameMap, self).get(item) - break -@@ -219,8 +220,8 @@ def sanitize_domain(self, domain): - # don't obfuscate vendor domains - if re.match(_skip, '.'.join(domain)): - return '.'.join(domain) -- top_domain = domain[-1] -- dname = '.'.join(domain[0:-1]) -+ top_domain = domain[-1].lower() -+ dname = '.'.join(domain[0:-1]).lower() - ob_domain = self._new_obfuscated_domain(dname) - ob_domain = '.'.join([ob_domain, top_domain]) - self.dataset['.'.join(domain)] = ob_domain -From f5e1298162a9393ea2d9f5c4df40dfece50f5f88 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 6 Jan 2022 13:15:15 -0500 -Subject: [PATCH 1/3] [hostname] Fix loading and detection of long base domains - -Our domain matching has up to now assumed that users would be providing -'base' domains such as 'example.com' whereby something like -'foo.bar.example.com' is a subdomain (or host) within that base domain. - -However, the use case exists to provide 'foo.bar.example.com' as the -base domain, without wanting to obfuscate 'example.com' directly. - -This commit fixes our handling of both loading these longer domains and -doing the 'domain is part of a domain we want to obfuscate' check. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/mappings/hostname_map.py | 9 ++++++++- - 1 file changed, 8 insertions(+), 1 deletion(-) - -diff --git a/sos/cleaner/mappings/hostname_map.py b/sos/cleaner/mappings/hostname_map.py -index 33b0e6c8..7a7cf6b8 100644 ---- a/sos/cleaner/mappings/hostname_map.py -+++ b/sos/cleaner/mappings/hostname_map.py -@@ -50,10 +50,14 @@ class SoSHostnameMap(SoSMap): - in this parser, we need to re-inject entries from the map_file into - these dicts and not just the underlying 'dataset' dict - """ -- for domain in self.dataset: -+ for domain, ob_pair in self.dataset.items(): - if len(domain.split('.')) == 1: - self.hosts[domain.split('.')[0]] = self.dataset[domain] - else: -+ if ob_pair.startswith('obfuscateddomain'): -+ # directly exact domain matches -+ self._domains[domain] = ob_pair.split('.')[0] -+ continue - # strip the host name and trailing top-level domain so that - # we in inject the domain properly for later string matching - -@@ -102,9 +106,12 @@ class SoSHostnameMap(SoSMap): - and should be obfuscated - """ - host = domain.split('.') -+ no_tld = '.'.join(domain.split('.')[0:-1]) - if len(host) == 1: - # don't block on host's shortname - return host[0] in self.hosts.keys() -+ elif any([no_tld.endswith(_d) for _d in self._domains]): -+ return True - else: - domain = host[0:-1] - for known_domain in self._domains: --- -2.31.1 - - -From e241cf33a14ecd4e848a5fd857c5d3d7d07fbd71 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 6 Jan 2022 13:18:44 -0500 -Subject: [PATCH 2/3] [cleaner] Improve parser-specific file skipping - -This commit improves our handling of skipping files on a per-parser -basis, by first filtering the list of parsers that `obfuscate_line()` -will iterate over by the parser's `skip_file` class attr, rather than -relying on higher-level checks. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/__init__.py | 17 ++++++++++++++--- - 1 file changed, 14 insertions(+), 3 deletions(-) - -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index 3f530d44..5686e213 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -12,6 +12,7 @@ import hashlib - import json - import logging - import os -+import re - import shutil - import tempfile - -@@ -640,10 +641,16 @@ third party. - self.log_debug("Obfuscating %s" % short_name or filename, - caller=arc_name) - tfile = tempfile.NamedTemporaryFile(mode='w', dir=self.tmpdir) -+ _parsers = [ -+ _p for _p in self.parsers if not -+ any([ -+ re.match(p, short_name) for p in _p.skip_files -+ ]) -+ ] - with open(filename, 'r') as fname: - for line in fname: - try: -- line, count = self.obfuscate_line(line) -+ line, count = self.obfuscate_line(line, _parsers) - subs += count - tfile.write(line) - except Exception as err: -@@ -713,7 +720,7 @@ third party. - pass - return string_data - -- def obfuscate_line(self, line): -+ def obfuscate_line(self, line, parsers=None): - """Run a line through each of the obfuscation parsers, keeping a - cumulative total of substitutions done on that particular line. - -@@ -721,6 +728,8 @@ third party. - - :param line str: The raw line as read from the file being - processed -+ :param parsers: A list of parser objects to obfuscate -+ with. If None, use all. - - Returns the fully obfuscated line and the number of substitutions made - """ -@@ -729,7 +738,9 @@ third party. - count = 0 - if not line.strip(): - return line, count -- for parser in self.parsers: -+ if parsers is None: -+ parsers = self.parsers -+ for parser in parsers: - try: - line, _count = parser.parse_line(line) - count += _count --- -2.31.1 - - -From 96c9a833e77639a853b7d3d6f1df68bbbbe5e9cb Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 6 Jan 2022 13:20:32 -0500 -Subject: [PATCH 3/3] [cleaner] Add skips for known files and usernames - -Adds skips for `/proc/kallsyms` which should never be obfuscated, as -well as any packaging-related log file for the IP parser. Further, do -not obfuscate the `stack` users, as that is a well-known user for many -configurations that, if obfuscated, could result in undesired string -substitutions in normal logging. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/archives/__init__.py | 2 ++ - sos/cleaner/parsers/ip_parser.py | 3 ++- - sos/cleaner/parsers/username_parser.py | 1 + - 3 files changed, 5 insertions(+), 1 deletion(-) - -diff --git a/sos/cleaner/archives/__init__.py b/sos/cleaner/archives/__init__.py -index 795c5a78..cbf1f809 100644 ---- a/sos/cleaner/archives/__init__.py -+++ b/sos/cleaner/archives/__init__.py -@@ -43,6 +43,7 @@ class SoSObfuscationArchive(): - type_name = 'undetermined' - description = 'undetermined' - is_nested = False -+ skip_files = [] - prep_files = {} - - def __init__(self, archive_path, tmpdir): -@@ -111,6 +112,7 @@ class SoSObfuscationArchive(): - Returns: list of files and file regexes - """ - return [ -+ 'proc/kallsyms', - 'sosreport-', - 'sys/firmware', - 'sys/fs', -diff --git a/sos/cleaner/parsers/ip_parser.py b/sos/cleaner/parsers/ip_parser.py -index 71d38be8..b007368c 100644 ---- a/sos/cleaner/parsers/ip_parser.py -+++ b/sos/cleaner/parsers/ip_parser.py -@@ -37,7 +37,8 @@ class SoSIPParser(SoSCleanerParser): - 'sos_commands/snappy/snap_list_--all', - 'sos_commands/snappy/snap_--version', - 'sos_commands/vulkan/vulkaninfo', -- 'var/log/.*dnf.*' -+ 'var/log/.*dnf.*', -+ 'var/log/.*packag.*' # get 'packages' and 'packaging' logs - ] - - map_file_key = 'ip_map' -diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py -index 229c7de4..3208a655 100644 ---- a/sos/cleaner/parsers/username_parser.py -+++ b/sos/cleaner/parsers/username_parser.py -@@ -32,6 +32,7 @@ class SoSUsernameParser(SoSCleanerParser): - 'nobody', - 'nfsnobody', - 'shutdown', -+ 'stack', - 'reboot', - 'root', - 'ubuntu', --- -2.31.1 - -From 7ebb2ce0bcd13c1b3aada648aceb20b5aff636d9 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 15 Feb 2022 14:18:02 -0500 -Subject: [PATCH] [host] Skip entire /etc/sos/cleaner directory - -While `default_mapping` is typically the only file expected under -`/etc/sos/cleaner/` it is possible for other mapping files (such as -backups) to appear there. - -Make the `add_forbidden_path()` spec here target the entire cleaner -directory to avoid ever capturing these map files. - -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/host.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/report/plugins/host.py b/sos/report/plugins/host.py -index 5e21da7b8e..95a3b9cd95 100644 ---- a/sos/report/plugins/host.py -+++ b/sos/report/plugins/host.py -@@ -20,7 +20,7 @@ class Host(Plugin, IndependentPlugin): - - def setup(self): - -- self.add_forbidden_path('/etc/sos/cleaner/default_mapping') -+ self.add_forbidden_path('/etc/sos/cleaner') - - self.add_cmd_output('hostname', root_symlink='hostname') - self.add_cmd_output('uptime', root_symlink='uptime') diff --git a/SOURCES/sos-bz2025611-RHTS-api-change.patch b/SOURCES/sos-bz2025611-RHTS-api-change.patch deleted file mode 100644 index 580117f..0000000 --- a/SOURCES/sos-bz2025611-RHTS-api-change.patch +++ /dev/null @@ -1,224 +0,0 @@ -From 2e8b5e2d4f30854cce93d149fc7d24b9d9cfd02c Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Fri, 19 Nov 2021 16:16:07 +0100 -Subject: [PATCH 1/3] [policies] strip path from SFTP upload filename - -When case_id is not supplied, we ask SFTP server to store the uploaded -file under name /var/tmp/, which is confusing. - -Let remove the path from it also in case_id not supplied. - -Related to: #2764 - -Signed-off-by: Pavel Moravec ---- - sos/policies/distros/redhat.py | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - -diff --git a/sos/policies/distros/redhat.py b/sos/policies/distros/redhat.py -index 3476e21fb..8817fc785 100644 ---- a/sos/policies/distros/redhat.py -+++ b/sos/policies/distros/redhat.py -@@ -269,10 +269,10 @@ def _get_sftp_upload_name(self): - """The RH SFTP server will only automatically connect file uploads to - cases if the filename _starts_ with the case number - """ -+ fname = self.upload_archive_name.split('/')[-1] - if self.case_id: -- return "%s_%s" % (self.case_id, -- self.upload_archive_name.split('/')[-1]) -- return self.upload_archive_name -+ return "%s_%s" % (self.case_id, fname) -+ return fname - - def upload_sftp(self): - """Override the base upload_sftp to allow for setting an on-demand - -From 61023b29a656dd7afaa4a0643368b0a53f1a3779 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Fri, 19 Nov 2021 17:31:31 +0100 -Subject: [PATCH 2/3] [redhat] update SFTP API version to v2 - -Change API version from v1 to v2, which includes: -- change of URL -- different URI -- POST method for token generation instead of GET - -Resolves: #2764 - -Signed-off-by: Pavel Moravec ---- - sos/policies/distros/redhat.py | 10 +++++----- - 1 file changed, 5 insertions(+), 5 deletions(-) - -diff --git a/sos/policies/distros/redhat.py b/sos/policies/distros/redhat.py -index 8817fc785..e4e2b8835 100644 ---- a/sos/policies/distros/redhat.py -+++ b/sos/policies/distros/redhat.py -@@ -175,7 +175,7 @@ def get_tmp_dir(self, opt_tmp_dir): - No changes will be made to system configuration. - """ - --RH_API_HOST = "https://access.redhat.com" -+RH_API_HOST = "https://api.access.redhat.com" - RH_SFTP_HOST = "sftp://sftp.access.redhat.com" - - -@@ -287,12 +287,12 @@ def upload_sftp(self): - " for obtaining SFTP auth token.") - _token = None - _user = None -+ url = RH_API_HOST + '/support/v2/sftp/token' - # we have a username and password, but we need to reset the password - # to be the token returned from the auth endpoint - if self.get_upload_user() and self.get_upload_password(): -- url = RH_API_HOST + '/hydra/rest/v1/sftp/token' - auth = self.get_upload_https_auth() -- ret = requests.get(url, auth=auth, timeout=10) -+ ret = requests.post(url, auth=auth, timeout=10) - if ret.status_code == 200: - # credentials are valid - _user = self.get_upload_user() -@@ -302,8 +302,8 @@ def upload_sftp(self): - "credentials. Will try anonymous.") - # we either do not have a username or password/token, or both - if not _token: -- aurl = RH_API_HOST + '/hydra/rest/v1/sftp/token?isAnonymous=true' -- anon = requests.get(aurl, timeout=10) -+ adata = {"isAnonymous": True} -+ anon = requests.post(url, data=json.dumps(adata), timeout=10) - if anon.status_code == 200: - resp = json.loads(anon.text) - _user = resp['username'] - -From 267da2156ec61f526dd28e760ff6528408a76c3f Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 22 Nov 2021 15:22:32 +0100 -Subject: [PATCH 3/3] [policies] Deal 200 return code as success - -Return code 200 of POST method request must be dealt as success. - -Newly required due to the SFTP API change using POST. - -Related to: #2764 - -Signed-off-by: Pavel Moravec ---- - sos/policies/distros/__init__.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/policies/distros/__init__.py b/sos/policies/distros/__init__.py -index 0906fa779..6f257fdce 100644 ---- a/sos/policies/distros/__init__.py -+++ b/sos/policies/distros/__init__.py -@@ -551,7 +551,7 @@ def upload_https(self): - r = self._upload_https_put(arc, verify) - else: - r = self._upload_https_post(arc, verify) -- if r.status_code != 201: -+ if r.status_code != 200 and r.status_code != 201: - if r.status_code == 401: - raise Exception( - "Authentication failed: invalid user credentials" -From 8da1b14246226792c160dd04e5c7c75dd4e8d44b Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 22 Nov 2021 10:44:09 +0100 -Subject: [PATCH] [collect] fix moved get_upload_url under Policy class - -SoSCollector does not further declare get_upload_url method -as that was moved under Policy class(es). - -Resolves: #2766 - -Signed-off-by: Pavel Moravec ---- - sos/collector/__init__.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index 50183e873..42a7731d6 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -1219,7 +1219,7 @@ this utility or remote systems that it c - msg = 'No sosreports were collected, nothing to archive...' - self.exit(msg, 1) - -- if self.opts.upload and self.get_upload_url(): -+ if self.opts.upload and self.policy.get_upload_url(): - try: - self.policy.upload_archive(arc_name) - self.ui_log.info("Uploaded archive successfully") -From abb2fc65bd14760021c61699ad3113cab3bd4c64 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 30 Nov 2021 11:37:02 +0100 -Subject: [PATCH 1/2] [redhat] Fix broken URI to upload to customer portal - -Revert back the unwanted change in URI of uploading tarball to the -Red Hat Customer portal. - -Related: #2772 - -Signed-off-by: Pavel Moravec ---- - sos/policies/distros/redhat.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/policies/distros/redhat.py b/sos/policies/distros/redhat.py -index e4e2b883..eb442407 100644 ---- a/sos/policies/distros/redhat.py -+++ b/sos/policies/distros/redhat.py -@@ -250,7 +250,7 @@ support representative. - elif self.commons['cmdlineopts'].upload_protocol == 'sftp': - return RH_SFTP_HOST - else: -- rh_case_api = "/hydra/rest/cases/%s/attachments" -+ rh_case_api = "/support/v1/cases/%s/attachments" - return RH_API_HOST + rh_case_api % self.case_id - - def _get_upload_headers(self): --- -2.31.1 - - -From ea4f9e88a412c80a4791396e1bb78ac1e24ece14 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 30 Nov 2021 13:00:26 +0100 -Subject: [PATCH 2/2] [policy] Add error message when FTP upload write failure - -When (S)FTP upload fails to write the destination file, -our "expect" code should detect it sooner than after timeout happens -and write appropriate error message. - -Resolves: #2772 - -Signed-off-by: Pavel Moravec ---- - sos/policies/distros/__init__.py | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/sos/policies/distros/__init__.py b/sos/policies/distros/__init__.py -index 6f257fdc..7bdc81b8 100644 ---- a/sos/policies/distros/__init__.py -+++ b/sos/policies/distros/__init__.py -@@ -473,7 +473,8 @@ class LinuxPolicy(Policy): - put_expects = [ - u'100%', - pexpect.TIMEOUT, -- pexpect.EOF -+ pexpect.EOF, -+ u'No such file or directory' - ] - - put_success = ret.expect(put_expects, timeout=180) -@@ -485,6 +486,8 @@ class LinuxPolicy(Policy): - raise Exception("Timeout expired while uploading") - elif put_success == 2: - raise Exception("Unknown error during upload: %s" % ret.before) -+ elif put_success == 3: -+ raise Exception("Unable to write archive to destination") - else: - raise Exception("Unexpected response from server: %s" % ret.before) - --- -2.31.1 - diff --git a/SOURCES/sos-bz2031777-rhui-logs.patch b/SOURCES/sos-bz2031777-rhui-logs.patch deleted file mode 100644 index dcfbc89..0000000 --- a/SOURCES/sos-bz2031777-rhui-logs.patch +++ /dev/null @@ -1,24 +0,0 @@ -From aa2887f71c779448b22e4de67ae68dbaf218b7b9 Mon Sep 17 00:00:00 2001 -From: Taft Sanders -Date: Fri, 10 Dec 2021 09:34:59 -0500 -Subject: [PATCH] [rhui] New log folder - -Included new log folder per Bugzilla 2030741 - -Signed-off-by: Taft Sanders ---- - sos/report/plugins/rhui.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/sos/report/plugins/rhui.py b/sos/report/plugins/rhui.py -index 52065fb44..add024613 100644 ---- a/sos/report/plugins/rhui.py -+++ b/sos/report/plugins/rhui.py -@@ -27,6 +27,7 @@ def setup(self): - "/var/log/rhui-subscription-sync.log", - "/var/cache/rhui/*", - "/root/.rhui/*", -+ "/var/log/rhui/*", - ]) - # skip collecting certificate keys - self.add_forbidden_path("/etc/pki/rhui/**/*.key", recursive=True) diff --git a/SOURCES/sos-bz2034001-nvidia-GPU-info.patch b/SOURCES/sos-bz2034001-nvidia-GPU-info.patch deleted file mode 100644 index 30fbb53..0000000 --- a/SOURCES/sos-bz2034001-nvidia-GPU-info.patch +++ /dev/null @@ -1,46 +0,0 @@ -From f2cc67750f55a71edff0c527a1bfc14fde8132c3 Mon Sep 17 00:00:00 2001 -From: Mamatha Inamdar -Date: Mon, 8 Nov 2021 10:50:03 +0530 -Subject: [PATCH] [nvidia]:Patch to update nvidia plugin for GPU info - -This patch is to update nvidia plugin to collect -logs for Nvidia GPUs - -Signed-off-by: Mamatha Inamdar -Reported-by: Borislav Stoymirski -Reported-by: Yesenia Jimenez ---- - sos/report/plugins/nvidia.py | 15 +++++++++++++-- - 1 file changed, 13 insertions(+), 2 deletions(-) - -diff --git a/sos/report/plugins/nvidia.py b/sos/report/plugins/nvidia.py -index 09aaf586b..9e21b478e 100644 ---- a/sos/report/plugins/nvidia.py -+++ b/sos/report/plugins/nvidia.py -@@ -23,13 +23,24 @@ def setup(self): - '--list-gpus', - '-q -d PERFORMANCE', - '-q -d SUPPORTED_CLOCKS', -- '-q -d PAGE_RETIREMENT' -+ '-q -d PAGE_RETIREMENT', -+ '-q', -+ '-q -d ECC', -+ 'nvlink -s', -+ 'nvlink -e' - ] - - self.add_cmd_output(["nvidia-smi %s" % cmd for cmd in subcmds]) - - query = ('gpu_name,gpu_bus_id,vbios_version,temperature.gpu,' -- 'utilization.gpu,memory.total,memory.free,memory.used') -+ 'utilization.gpu,memory.total,memory.free,memory.used,' -+ 'clocks.applications.graphics,clocks.applications.memory') -+ querypages = ('timestamp,gpu_bus_id,gpu_serial,gpu_uuid,' -+ 'retired_pages.address,retired_pages.cause') - self.add_cmd_output("nvidia-smi --query-gpu=%s --format=csv" % query) -+ self.add_cmd_output( -+ "nvidia-smi --query-retired-pages=%s --format=csv" % querypages -+ ) -+ self.add_journal(boot=0, identifier='nvidia-persistenced') - - # vim: set et ts=4 sw=4 : diff --git a/SOURCES/sos-bz2037350-ocp-backports.patch b/SOURCES/sos-bz2037350-ocp-backports.patch deleted file mode 100644 index 3e53e93..0000000 --- a/SOURCES/sos-bz2037350-ocp-backports.patch +++ /dev/null @@ -1,5145 +0,0 @@ -From 676dfca09d9c783311a51a1c53fa0f7ecd95bd28 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Fri, 10 Sep 2021 13:38:19 -0400 -Subject: [PATCH] [collect] Abstract transport protocol from SoSNode - -Since its addition to sos, collect has assumed the use of a system -installation of SSH in order to connect to the nodes identified for -collection. However, there may be use cases and desires to use other -transport protocols. - -As such, provide an abstraction for these protocols in the form of the -new `RemoteTransport` class that `SoSNode` will now leverage. So far an -abstraction for the currently used SSH ControlPersist function is -provided, along with a psuedo abstraction for local execution so that -SoSNode does not directly need to make more "if local then foo" checks -than are absolutely necessary. - -Related: #2668 - -Signed-off-by: Jake Hunsaker ---- - setup.py | 4 +- - sos/collector/__init__.py | 54 +-- - sos/collector/clusters/__init__.py | 4 +- - sos/collector/clusters/jbon.py | 2 + - sos/collector/clusters/kubernetes.py | 4 +- - sos/collector/clusters/ocp.py | 6 +- - sos/collector/clusters/ovirt.py | 10 +- - sos/collector/clusters/pacemaker.py | 8 +- - sos/collector/clusters/satellite.py | 4 +- - sos/collector/sosnode.py | 388 +++++--------------- - sos/collector/transports/__init__.py | 317 ++++++++++++++++ - sos/collector/transports/control_persist.py | 199 ++++++++++ - sos/collector/transports/local.py | 49 +++ - 13 files changed, 705 insertions(+), 344 deletions(-) - create mode 100644 sos/collector/transports/__init__.py - create mode 100644 sos/collector/transports/control_persist.py - create mode 100644 sos/collector/transports/local.py - -diff --git a/setup.py b/setup.py -index 7653b59d..25e87a71 100644 ---- a/setup.py -+++ b/setup.py -@@ -101,8 +101,8 @@ setup( - 'sos.policies.distros', 'sos.policies.runtimes', - 'sos.policies.package_managers', 'sos.policies.init_systems', - 'sos.report', 'sos.report.plugins', 'sos.collector', -- 'sos.collector.clusters', 'sos.cleaner', 'sos.cleaner.mappings', -- 'sos.cleaner.parsers', 'sos.cleaner.archives' -+ 'sos.collector.clusters', 'sos.collector.transports', 'sos.cleaner', -+ 'sos.cleaner.mappings', 'sos.cleaner.parsers', 'sos.cleaner.archives' - ], - cmdclass=cmdclass, - command_options=command_options, -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index b2a07f37..da912655 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -17,7 +17,6 @@ import re - import string - import socket - import shutil --import subprocess - import sys - - from datetime import datetime -@@ -28,7 +27,6 @@ from pipes import quote - from textwrap import fill - from sos.cleaner import SoSCleaner - from sos.collector.sosnode import SosNode --from sos.collector.exceptions import ControlPersistUnsupportedException - from sos.options import ClusterOption - from sos.component import SoSComponent - from sos import __version__ -@@ -154,7 +152,6 @@ class SoSCollector(SoSComponent): - try: - self.parse_node_strings() - self.parse_cluster_options() -- self._check_for_control_persist() - self.log_debug('Executing %s' % ' '.join(s for s in sys.argv)) - self.log_debug("Found cluster profiles: %s" - % self.clusters.keys()) -@@ -437,33 +434,6 @@ class SoSCollector(SoSComponent): - action='extend', - help='List of usernames to obfuscate') - -- def _check_for_control_persist(self): -- """Checks to see if the local system supported SSH ControlPersist. -- -- ControlPersist allows OpenSSH to keep a single open connection to a -- remote host rather than building a new session each time. This is the -- same feature that Ansible uses in place of paramiko, which we have a -- need to drop in sos-collector. -- -- This check relies on feedback from the ssh binary. The command being -- run should always generate stderr output, but depending on what that -- output reads we can determine if ControlPersist is supported or not. -- -- For our purposes, a host that does not support ControlPersist is not -- able to run sos-collector. -- -- Returns -- True if ControlPersist is supported, else raise Exception. -- """ -- ssh_cmd = ['ssh', '-o', 'ControlPersist'] -- cmd = subprocess.Popen(ssh_cmd, stdout=subprocess.PIPE, -- stderr=subprocess.PIPE) -- out, err = cmd.communicate() -- err = err.decode('utf-8') -- if 'Bad configuration option' in err or 'Usage:' in err: -- raise ControlPersistUnsupportedException -- return True -- - def exit(self, msg, error=1): - """Used to safely terminate if sos-collector encounters an error""" - self.log_error(msg) -@@ -455,7 +455,7 @@ class SoSCollector(SoSComponent): - 'cmdlineopts': self.opts, - 'need_sudo': True if self.opts.ssh_user != 'root' else False, - 'tmpdir': self.tmpdir, -- 'hostlen': len(self.opts.master) or len(self.hostname), -+ 'hostlen': max(len(self.opts.primary), len(self.hostname)), - 'policy': self.policy - } - -@@ -1020,9 +1020,10 @@ class SoSCollector(SoSComponent): - self.node_list.append(self.hostname) - self.reduce_node_list() - try: -- self.commons['hostlen'] = len(max(self.node_list, key=len)) -+ _node_max = len(max(self.node_list, key=len)) -+ self.commons['hostlen'] = max(_node_max, self.commons['hostlen']) - except (TypeError, ValueError): -- self.commons['hostlen'] = len(self.opts.master) -+ pass - - def _connect_to_node(self, node): - """Try to connect to the node, and if we can add to the client list to -@@ -1068,7 +1039,7 @@ class SoSCollector(SoSComponent): - client.set_node_manifest(getattr(self.collect_md.nodes, - node[0])) - else: -- client.close_ssh_session() -+ client.disconnect() - except Exception: - pass - -@@ -1077,12 +1048,11 @@ class SoSCollector(SoSComponent): - provided on the command line - """ - disclaimer = ("""\ --This utility is used to collect sosreports from multiple \ --nodes simultaneously. It uses OpenSSH's ControlPersist feature \ --to connect to nodes and run commands remotely. If your system \ --installation of OpenSSH is older than 5.6, please upgrade. -+This utility is used to collect sos reports from multiple \ -+nodes simultaneously. Remote connections are made and/or maintained \ -+to those nodes via well-known transport protocols such as SSH. - --An archive of sosreport tarballs collected from the nodes will be \ -+An archive of sos report tarballs collected from the nodes will be \ - generated in %s and may be provided to an appropriate support representative. - - The generated archive may contain data considered sensitive \ -@@ -1230,10 +1200,10 @@ this utility or remote systems that it connects to. - self.log_error("Error running sosreport: %s" % err) - - def close_all_connections(self): -- """Close all ssh sessions for nodes""" -+ """Close all sessions for nodes""" - for client in self.client_list: -- self.log_debug('Closing SSH connection to %s' % client.address) -- client.close_ssh_session() -+ self.log_debug('Closing connection to %s' % client.address) -+ client.disconnect() - - def create_cluster_archive(self): - """Calls for creation of tar archive then cleans up the temporary -diff --git a/sos/collector/clusters/__init__.py b/sos/collector/clusters/__init__.py -index 2b5d7018..64ac2a44 100644 ---- a/sos/collector/clusters/__init__.py -+++ b/sos/collector/clusters/__init__.py -@@ -188,8 +188,8 @@ class Cluster(): - :rtype: ``dict`` - """ - res = self.master.run_command(cmd, get_pty=True, need_root=need_root) -- if res['stdout']: -- res['stdout'] = res['stdout'].replace('Password:', '') -+ if res['output']: -+ res['output'] = res['output'].replace('Password:', '') - return res - - def setup(self): -diff --git a/sos/collector/clusters/jbon.py b/sos/collector/clusters/jbon.py -index 488fbd16..8f083ac6 100644 ---- a/sos/collector/clusters/jbon.py -+++ b/sos/collector/clusters/jbon.py -@@ -28,3 +28,5 @@ class jbon(Cluster): - # This should never be called, but as insurance explicitly never - # allow this to be enabled via the determine_cluster() path - return False -+ -+# vim: set et ts=4 sw=4 : -diff --git a/sos/collector/clusters/kubernetes.py b/sos/collector/clusters/kubernetes.py -index cdbf8861..99f788dc 100644 ---- a/sos/collector/clusters/kubernetes.py -+++ b/sos/collector/clusters/kubernetes.py -@@ -34,7 +34,7 @@ class kubernetes(Cluster): - if res['status'] == 0: - nodes = [] - roles = [x for x in self.get_option('role').split(',') if x] -- for nodeln in res['stdout'].splitlines()[1:]: -+ for nodeln in res['output'].splitlines()[1:]: - node = nodeln.split() - if not roles: - nodes.append(node[0]) -@@ -44,3 +44,5 @@ class kubernetes(Cluster): - return nodes - else: - raise Exception('Node enumeration did not return usable output') -+ -+# vim: set et ts=4 sw=4 : -diff --git a/sos/collector/clusters/ocp.py b/sos/collector/clusters/ocp.py -index 5479417d..ad97587f 100644 ---- a/sos/collector/clusters/ocp.py -+++ b/sos/collector/clusters/ocp.py -@@ -93,7 +93,7 @@ class ocp(Cluster): - res = self.exec_master_cmd(self.fmt_oc_cmd(cmd)) - if res['status'] == 0: - roles = [r for r in self.get_option('role').split(':')] -- self.node_dict = self._build_dict(res['stdout'].splitlines()) -+ self.node_dict = self._build_dict(res['output'].splitlines()) - for node in self.node_dict: - if roles: - for role in roles: -@@ -103,7 +103,7 @@ class ocp(Cluster): - nodes.append(node) - else: - msg = "'oc' command failed" -- if 'Missing or incomplete' in res['stdout']: -+ if 'Missing or incomplete' in res['output']: - msg = ("'oc' failed due to missing kubeconfig on master node." - " Specify one via '-c ocp.kubeconfig='") - raise Exception(msg) -@@ -168,3 +168,5 @@ class ocp(Cluster): - def set_node_options(self, node): - # don't attempt OC API collections on non-primary nodes - node.plugin_options.append('openshift.no-oc=on') -+ -+# vim: set et ts=4 sw=4 : -diff --git a/sos/collector/clusters/ovirt.py b/sos/collector/clusters/ovirt.py -index 079a122e..bd2d0c74 100644 ---- a/sos/collector/clusters/ovirt.py -+++ b/sos/collector/clusters/ovirt.py -@@ -98,7 +98,7 @@ class ovirt(Cluster): - return [] - res = self._run_db_query(self.dbquery) - if res['status'] == 0: -- nodes = res['stdout'].splitlines()[2:-1] -+ nodes = res['output'].splitlines()[2:-1] - return [n.split('(')[0].strip() for n in nodes] - else: - raise Exception('database query failed, return code: %s' -@@ -114,7 +114,7 @@ class ovirt(Cluster): - engconf = '/etc/ovirt-engine/engine.conf.d/10-setup-database.conf' - res = self.exec_primary_cmd('cat %s' % engconf, need_root=True) - if res['status'] == 0: -- config = res['stdout'].splitlines() -+ config = res['output'].splitlines() - for line in config: - try: - k = str(line.split('=')[0]) -@@ -141,7 +141,7 @@ class ovirt(Cluster): - '--batch -o postgresql {}' - ).format(self.conf['ENGINE_DB_PASSWORD'], sos_opt) - db_sos = self.exec_primary_cmd(cmd, need_root=True) -- for line in db_sos['stdout'].splitlines(): -+ for line in db_sos['output'].splitlines(): - if fnmatch.fnmatch(line, '*sosreport-*tar*'): - _pg_dump = line.strip() - self.master.manifest.add_field('postgresql_dump', -@@ -180,5 +180,7 @@ class rhhi_virt(rhv): - ret = self._run_db_query('SELECT count(server_id) FROM gluster_server') - if ret['status'] == 0: - # if there are any entries in this table, RHHI-V is in use -- return ret['stdout'].splitlines()[2].strip() != '0' -+ return ret['output'].splitlines()[2].strip() != '0' - return False -+ -+# vim: set et ts=4 sw=4 : -diff --git a/sos/collector/clusters/pacemaker.py b/sos/collector/clusters/pacemaker.py -index 034f3f3e..55024314 100644 ---- a/sos/collector/clusters/pacemaker.py -+++ b/sos/collector/clusters/pacemaker.py -@@ -27,7 +27,7 @@ class pacemaker(Cluster): - self.log_error('Cluster status could not be determined. Is the ' - 'cluster running on this node?') - return [] -- if 'node names do not match' in self.res['stdout']: -+ if 'node names do not match' in self.res['output']: - self.log_warn('Warning: node name mismatch reported. Attempts to ' - 'connect to some nodes may fail.\n') - return self.parse_pcs_output() -@@ -41,17 +41,19 @@ class pacemaker(Cluster): - return nodes - - def get_online_nodes(self): -- for line in self.res['stdout'].splitlines(): -+ for line in self.res['output'].splitlines(): - if line.startswith('Online:'): - nodes = line.split('[')[1].split(']')[0] - return [n for n in nodes.split(' ') if n] - - def get_offline_nodes(self): - offline = [] -- for line in self.res['stdout'].splitlines(): -+ for line in self.res['output'].splitlines(): - if line.startswith('Node') and line.endswith('(offline)'): - offline.append(line.split()[1].replace(':', '')) - if line.startswith('OFFLINE:'): - nodes = line.split('[')[1].split(']')[0] - offline.extend([n for n in nodes.split(' ') if n]) - return offline -+ -+# vim: set et ts=4 sw=4 : -diff --git a/sos/collector/clusters/satellite.py b/sos/collector/clusters/satellite.py -index e123c8a3..7c21e553 100644 ---- a/sos/collector/clusters/satellite.py -+++ b/sos/collector/clusters/satellite.py -@@ -28,7 +28,7 @@ class satellite(Cluster): - res = self.exec_primary_cmd(cmd, need_root=True) - if res['status'] == 0: - nodes = [ -- n.strip() for n in res['stdout'].splitlines() -+ n.strip() for n in res['output'].splitlines() - if 'could not change directory' not in n - ] - return nodes -@@ -38,3 +38,5 @@ class satellite(Cluster): - if node.address == self.master.address: - return 'satellite' - return 'capsule' -+ -+# vim: set et ts=4 sw=4 : -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 4b1ee109..f79bd5ff 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -12,22 +12,16 @@ import fnmatch - import inspect - import logging - import os --import pexpect - import re --import shutil - - from distutils.version import LooseVersion - from pipes import quote - from sos.policies import load - from sos.policies.init_systems import InitSystem --from sos.collector.exceptions import (InvalidPasswordException, -- TimeoutPasswordAuthException, -- PasswordRequestException, -- AuthPermissionDeniedException, -+from sos.collector.transports.control_persist import SSHControlPersist -+from sos.collector.transports.local import LocalTransport -+from sos.collector.exceptions import (CommandTimeoutException, - ConnectionException, -- CommandTimeoutException, -- ConnectionTimeoutException, -- ControlSocketMissingException, - UnsupportedHostException) - - -@@ -61,34 +61,25 @@ class SosNode(): - 'sos_cmd': commons['sos_cmd'] - } - self.sos_bin = 'sosreport' -- filt = ['localhost', '127.0.0.1'] - self.soslog = logging.getLogger('sos') - self.ui_log = logging.getLogger('sos_ui') -- self.control_path = ("%s/.sos-collector-%s" -- % (self.tmpdir, self.address)) -- self.ssh_cmd = self._create_ssh_command() -- if self.address not in filt: -- try: -- self.connected = self._create_ssh_session() -- except Exception as err: -- self.log_error('Unable to open SSH session: %s' % err) -- raise -- else: -- self.connected = True -- self.local = True -- self.need_sudo = os.getuid() != 0 -+ self._transport = self._load_remote_transport(commons) -+ try: -+ self._transport.connect(self._password) -+ except Exception as err: -+ self.log_error('Unable to open remote session: %s' % err) -+ raise - # load the host policy now, even if we don't want to load further - # host information. This is necessary if we're running locally on the - # cluster master but do not want a local report as we still need to do - # package checks in that instance - self.host = self.determine_host_policy() -- self.get_hostname() -+ self.hostname = self._transport.hostname - if self.local and self.opts.no_local: - load_facts = False - if self.connected and load_facts: - if not self.host: -- self.connected = False -- self.close_ssh_session() -+ self._transport.disconnect() - return None - if self.local: - if self.check_in_container(): -@@ -103,11 +88,26 @@ class SosNode(): - self.create_sos_container() - self._load_sos_info() - -- def _create_ssh_command(self): -- """Build the complete ssh command for this node""" -- cmd = "ssh -oControlPath=%s " % self.control_path -- cmd += "%s@%s " % (self.opts.ssh_user, self.address) -- return cmd -+ @property -+ def connected(self): -+ if self._transport: -+ return self._transport.connected -+ # if no transport, we're running locally -+ return True -+ -+ def disconnect(self): -+ """Wrapper to close the remote session via our transport agent -+ """ -+ self._transport.disconnect() -+ -+ def _load_remote_transport(self, commons): -+ """Determine the type of remote transport to load for this node, then -+ return an instantiated instance of that transport -+ """ -+ if self.address in ['localhost', '127.0.0.1']: -+ self.local = True -+ return LocalTransport(self.address, commons) -+ return SSHControlPersist(self.address, commons) - - def _fmt_msg(self, msg): - return '{:<{}} : {}'.format(self._hostname, self.hostlen + 1, msg) -@@ -135,6 +135,7 @@ class SosNode(): - self.manifest.add_field('policy', self.host.distro) - self.manifest.add_field('sos_version', self.sos_info['version']) - self.manifest.add_field('final_sos_command', '') -+ self.manifest.add_field('transport', self._transport.name) - - def check_in_container(self): - """ -@@ -160,13 +161,13 @@ class SosNode(): - res = self.run_command(cmd, need_root=True) - if res['status'] in [0, 125]: - if res['status'] == 125: -- if 'unable to retrieve auth token' in res['stdout']: -+ if 'unable to retrieve auth token' in res['output']: - self.log_error( - "Could not pull image. Provide either a username " - "and password or authfile" - ) - raise Exception -- elif 'unknown: Not found' in res['stdout']: -+ elif 'unknown: Not found' in res['output']: - self.log_error('Specified image not found on registry') - raise Exception - # 'name exists' with code 125 means the container was -@@ -181,11 +182,11 @@ class SosNode(): - return True - else: - self.log_error("Could not start container after create: %s" -- % ret['stdout']) -+ % ret['output']) - raise Exception - else: - self.log_error("Could not create container on host: %s" -- % res['stdout']) -+ % res['output']) - raise Exception - - def get_container_auth(self): -@@ -204,18 +205,11 @@ class SosNode(): - - def file_exists(self, fname, need_root=False): - """Checks for the presence of fname on the remote node""" -- if not self.local: -- try: -- res = self.run_command("stat %s" % fname, need_root=need_root) -- return res['status'] == 0 -- except Exception: -- return False -- else: -- try: -- os.stat(fname) -- return True -- except Exception: -- return False -+ try: -+ res = self.run_command("stat %s" % fname, need_root=need_root) -+ return res['status'] == 0 -+ except Exception: -+ return False - - @property - def _hostname(self): -@@ -223,18 +217,6 @@ class SosNode(): - return self.hostname - return self.address - -- @property -- def control_socket_exists(self): -- """Check if the SSH control socket exists -- -- The control socket is automatically removed by the SSH daemon in the -- event that the last connection to the node was greater than the timeout -- set by the ControlPersist option. This can happen for us if we are -- collecting from a large number of nodes, and the timeout expires before -- we start collection. -- """ -- return os.path.exists(self.control_path) -- - def _sanitize_log_msg(self, msg): - """Attempts to obfuscate sensitive information in log messages such as - passwords""" -@@ -264,12 +246,6 @@ class SosNode(): - msg = '[%s:%s] %s' % (self._hostname, caller, msg) - self.soslog.debug(msg) - -- def get_hostname(self): -- """Get the node's hostname""" -- sout = self.run_command('hostname') -- self.hostname = sout['stdout'].strip() -- self.log_info('Hostname set to %s' % self.hostname) -- - def _format_cmd(self, cmd): - """If we need to provide a sudo or root password to a command, then - here we prefix the command with the correct bits -@@ -280,19 +256,6 @@ class SosNode(): - return "sudo -S %s" % cmd - return cmd - -- def _fmt_output(self, output=None, rc=0): -- """Formats the returned output from a command into a dict""" -- if rc == 0: -- stdout = output -- stderr = '' -- else: -- stdout = '' -- stderr = output -- res = {'status': rc, -- 'stdout': stdout, -- 'stderr': stderr} -- return res -- - def _load_sos_info(self): - """Queries the node for information about the installed version of sos - """ -@@ -306,7 +269,7 @@ class SosNode(): - pkgs = self.run_command(self.host.container_version_command, - use_container=True, need_root=True) - if pkgs['status'] == 0: -- ver = pkgs['stdout'].strip().split('-')[1] -+ ver = pkgs['output'].strip().split('-')[1] - if ver: - self.sos_info['version'] = ver - else: -@@ -321,18 +284,21 @@ class SosNode(): - self.log_error('sos is not installed on this node') - self.connected = False - return False -- cmd = 'sosreport -l' -+ # sos-4.0 changes the binary -+ if self.check_sos_version('4.0'): -+ self.sos_bin = 'sos report' -+ cmd = "%s -l" % self.sos_bin - sosinfo = self.run_command(cmd, use_container=True, need_root=True) - if sosinfo['status'] == 0: -- self._load_sos_plugins(sosinfo['stdout']) -+ self._load_sos_plugins(sosinfo['output']) - if self.check_sos_version('3.6'): - self._load_sos_presets() - - def _load_sos_presets(self): -- cmd = 'sosreport --list-presets' -+ cmd = '%s --list-presets' % self.sos_bin - res = self.run_command(cmd, use_container=True, need_root=True) - if res['status'] == 0: -- for line in res['stdout'].splitlines(): -+ for line in res['output'].splitlines(): - if line.strip().startswith('name:'): - pname = line.split('name:')[1].strip() - self.sos_info['presets'].append(pname) -@@ -372,21 +338,7 @@ class SosNode(): - """Reads the specified file and returns the contents""" - try: - self.log_info("Reading file %s" % to_read) -- if not self.local: -- res = self.run_command("cat %s" % to_read, timeout=5) -- if res['status'] == 0: -- return res['stdout'] -- else: -- if 'No such file' in res['stdout']: -- self.log_debug("File %s does not exist on node" -- % to_read) -- else: -- self.log_error("Error reading %s: %s" % -- (to_read, res['stdout'].split(':')[1:])) -- return '' -- else: -- with open(to_read, 'r') as rfile: -- return rfile.read() -+ return self._transport.read_file(to_read) - except Exception as err: - self.log_error("Exception while reading %s: %s" % (to_read, err)) - return '' -@@ -400,7 +352,8 @@ class SosNode(): - % self.commons['policy'].distro) - return self.commons['policy'] - host = load(cache={}, sysroot=self.opts.sysroot, init=InitSystem(), -- probe_runtime=True, remote_exec=self.ssh_cmd, -+ probe_runtime=True, -+ remote_exec=self._transport.remote_exec, - remote_check=self.read_file('/etc/os-release')) - if host: - self.log_info("loaded policy %s for host" % host.distro) -@@ -422,7 +375,7 @@ class SosNode(): - return self.host.package_manager.pkg_by_name(pkg) is not None - - def run_command(self, cmd, timeout=180, get_pty=False, need_root=False, -- force_local=False, use_container=False, env=None): -+ use_container=False, env=None): - """Runs a given cmd, either via the SSH session or locally - - Arguments: -@@ -433,58 +386,37 @@ class SosNode(): - need_root - if a command requires root privileges, setting this to - True tells sos-collector to format the command with - sudo or su - as appropriate and to input the password -- force_local - force a command to run locally. Mainly used for scp. - use_container - Run this command in a container *IF* the host is - containerized - """ -- if not self.control_socket_exists and not self.local: -- self.log_debug('Control socket does not exist, attempting to ' -- 're-create') -+ if not self.connected and not self.local: -+ self.log_debug('Node is disconnected, attempting to reconnect') - try: -- _sock = self._create_ssh_session() -- if not _sock: -- self.log_debug('Failed to re-create control socket') -- raise ControlSocketMissingException -+ reconnected = self._transport.reconnect(self._password) -+ if not reconnected: -+ self.log_debug('Failed to reconnect to node') -+ raise ConnectionException - except Exception as err: -- self.log_error('Cannot run command: control socket does not ' -- 'exist') -- self.log_debug("Error while trying to create new SSH control " -- "socket: %s" % err) -+ self.log_debug("Error while trying to reconnect: %s" % err) - raise - if use_container and self.host.containerized: - cmd = self.host.format_container_command(cmd) - if need_root: -- get_pty = True - cmd = self._format_cmd(cmd) -- self.log_debug('Running command %s' % cmd) -+ - if 'atomic' in cmd: - get_pty = True -- if not self.local and not force_local: -- cmd = "%s %s" % (self.ssh_cmd, quote(cmd)) -- else: -- if get_pty: -- cmd = "/bin/bash -c %s" % quote(cmd) -+ -+ if get_pty: -+ cmd = "/bin/bash -c %s" % quote(cmd) -+ - if env: - _cmd_env = self.env_vars - env = _cmd_env.update(env) -- res = pexpect.spawn(cmd, encoding='utf-8', env=env) -- if need_root: -- if self.need_sudo: -- res.sendline(self.opts.sudo_pw) -- if self.opts.become_root: -- res.sendline(self.opts.root_password) -- output = res.expect([pexpect.EOF, pexpect.TIMEOUT], -- timeout=timeout) -- if output == 0: -- out = res.before -- res.close() -- rc = res.exitstatus -- return {'status': rc, 'stdout': out} -- elif output == 1: -- raise CommandTimeoutException(cmd) -+ return self._transport.run_command(cmd, timeout, need_root, env) - - def sosreport(self): -- """Run a sosreport on the node, then collect it""" -+ """Run an sos report on the node, then collect it""" - try: - path = self.execute_sos_command() - if path: -@@ -497,109 +429,6 @@ class SosNode(): - pass - self.cleanup() - -- def _create_ssh_session(self): -- """ -- Using ControlPersist, create the initial connection to the node. -- -- This will generate an OpenSSH ControlPersist socket within the tmp -- directory created or specified for sos-collector to use. -- -- At most, we will wait 30 seconds for a connection. This involves a 15 -- second wait for the initial connection attempt, and a subsequent 15 -- second wait for a response when we supply a password. -- -- Since we connect to nodes in parallel (using the --threads value), this -- means that the time between 'Connecting to nodes...' and 'Beginning -- collection of sosreports' that users see can be up to an amount of time -- equal to 30*(num_nodes/threads) seconds. -- -- Returns -- True if session is successfully opened, else raise Exception -- """ -- # Don't use self.ssh_cmd here as we need to add a few additional -- # parameters to establish the initial connection -- self.log_info('Opening SSH session to create control socket') -- connected = False -- ssh_key = '' -- ssh_port = '' -- if self.opts.ssh_port != 22: -- ssh_port = "-p%s " % self.opts.ssh_port -- if self.opts.ssh_key: -- ssh_key = "-i%s" % self.opts.ssh_key -- cmd = ("ssh %s %s -oControlPersist=600 -oControlMaster=auto " -- "-oStrictHostKeyChecking=no -oControlPath=%s %s@%s " -- "\"echo Connected\"" % (ssh_key, -- ssh_port, -- self.control_path, -- self.opts.ssh_user, -- self.address)) -- res = pexpect.spawn(cmd, encoding='utf-8') -- -- connect_expects = [ -- u'Connected', -- u'password:', -- u'.*Permission denied.*', -- u'.* port .*: No route to host', -- u'.*Could not resolve hostname.*', -- pexpect.TIMEOUT -- ] -- -- index = res.expect(connect_expects, timeout=15) -- -- if index == 0: -- connected = True -- elif index == 1: -- if self._password: -- pass_expects = [ -- u'Connected', -- u'Permission denied, please try again.', -- pexpect.TIMEOUT -- ] -- res.sendline(self._password) -- pass_index = res.expect(pass_expects, timeout=15) -- if pass_index == 0: -- connected = True -- elif pass_index == 1: -- # Note that we do not get an exitstatus here, so matching -- # this line means an invalid password will be reported for -- # both invalid passwords and invalid user names -- raise InvalidPasswordException -- elif pass_index == 2: -- raise TimeoutPasswordAuthException -- else: -- raise PasswordRequestException -- elif index == 2: -- raise AuthPermissionDeniedException -- elif index == 3: -- raise ConnectionException(self.address, self.opts.ssh_port) -- elif index == 4: -- raise ConnectionException(self.address) -- elif index == 5: -- raise ConnectionTimeoutException -- else: -- raise Exception("Unknown error, client returned %s" % res.before) -- if connected: -- self.log_debug("Successfully created control socket at %s" -- % self.control_path) -- return True -- return False -- -- def close_ssh_session(self): -- """Remove the control socket to effectively terminate the session""" -- if self.local: -- return True -- try: -- res = self.run_command("rm -f %s" % self.control_path, -- force_local=True) -- if res['status'] == 0: -- return True -- self.log_error("Could not remove ControlPath %s: %s" -- % (self.control_path, res['stdout'])) -- return False -- except Exception as e: -- self.log_error('Error closing SSH session: %s' % e) -- return False -- - def _preset_exists(self, preset): - """Verifies if the given preset exists on the node""" - return preset in self.sos_info['presets'] -@@ -646,8 +475,8 @@ class SosNode(): - self.cluster = cluster - - def update_cmd_from_cluster(self): -- """This is used to modify the sosreport command run on the nodes. -- By default, sosreport is run without any options, using this will -+ """This is used to modify the sos report command run on the nodes. -+ By default, sos report is run without any options, using this will - allow the profile to specify what plugins to run or not and what - options to use. - -@@ -727,10 +556,6 @@ class SosNode(): - if self.opts.since: - sos_opts.append('--since=%s' % quote(self.opts.since)) - -- # sos-4.0 changes the binary -- if self.check_sos_version('4.0'): -- self.sos_bin = 'sos report' -- - if self.check_sos_version('4.1'): - if self.opts.skip_commands: - sos_opts.append( -@@ -811,7 +636,7 @@ class SosNode(): - self.manifest.add_field('final_sos_command', self.sos_cmd) - - def determine_sos_label(self): -- """Determine what, if any, label should be added to the sosreport""" -+ """Determine what, if any, label should be added to the sos report""" - label = '' - label += self.cluster.get_node_label(self) - -@@ -822,7 +647,7 @@ class SosNode(): - if not label: - return None - -- self.log_debug('Label for sosreport set to %s' % label) -+ self.log_debug('Label for sos report set to %s' % label) - if self.check_sos_version('3.6'): - lcmd = '--label' - else: -@@ -844,20 +669,20 @@ class SosNode(): - - def determine_sos_error(self, rc, stdout): - if rc == -1: -- return 'sosreport process received SIGKILL on node' -+ return 'sos report process received SIGKILL on node' - if rc == 1: - if 'sudo' in stdout: - return 'sudo attempt failed' - if rc == 127: -- return 'sosreport terminated unexpectedly. Check disk space' -+ return 'sos report terminated unexpectedly. Check disk space' - if len(stdout) > 0: - return stdout.split('\n')[0:1] - else: - return 'sos exited with code %s' % rc - - def execute_sos_command(self): -- """Run sosreport and capture the resulting file path""" -- self.ui_msg('Generating sosreport...') -+ """Run sos report and capture the resulting file path""" -+ self.ui_msg('Generating sos report...') - try: - path = False - checksum = False -@@ -867,7 +692,7 @@ class SosNode(): - use_container=True, - env=self.sos_env_vars) - if res['status'] == 0: -- for line in res['stdout'].splitlines(): -+ for line in res['output'].splitlines(): - if fnmatch.fnmatch(line, '*sosreport-*tar*'): - path = line.strip() - if line.startswith((" sha256\t", " md5\t")): -@@ -884,44 +709,31 @@ class SosNode(): - else: - self.manifest.add_field('checksum_type', 'unknown') - else: -- err = self.determine_sos_error(res['status'], res['stdout']) -- self.log_debug("Error running sosreport. rc = %s msg = %s" -- % (res['status'], res['stdout'] or -- res['stderr'])) -+ err = self.determine_sos_error(res['status'], res['output']) -+ self.log_debug("Error running sos report. rc = %s msg = %s" -+ % (res['status'], res['output'])) - raise Exception(err) - return path - except CommandTimeoutException: - self.log_error('Timeout exceeded') - raise - except Exception as e: -- self.log_error('Error running sosreport: %s' % e) -+ self.log_error('Error running sos report: %s' % e) - raise - - def retrieve_file(self, path): - """Copies the specified file from the host to our temp dir""" - destdir = self.tmpdir + '/' -- dest = destdir + path.split('/')[-1] -+ dest = os.path.join(destdir, path.split('/')[-1]) - try: -- if not self.local: -- if self.file_exists(path): -- self.log_info("Copying remote %s to local %s" % -- (path, destdir)) -- cmd = "/usr/bin/scp -oControlPath=%s %s@%s:%s %s" % ( -- self.control_path, -- self.opts.ssh_user, -- self.address, -- path, -- destdir -- ) -- res = self.run_command(cmd, force_local=True) -- return res['status'] == 0 -- else: -- self.log_debug("Attempting to copy remote file %s, but it " -- "does not exist on filesystem" % path) -- return False -+ if self.file_exists(path): -+ self.log_info("Copying remote %s to local %s" % -+ (path, destdir)) -+ self._transport.retrieve_file(path, dest) - else: -- self.log_debug("Moving %s to %s" % (path, destdir)) -- shutil.copy(path, dest) -+ self.log_debug("Attempting to copy remote file %s, but it " -+ "does not exist on filesystem" % path) -+ return False - return True - except Exception as err: - self.log_debug("Failed to retrieve %s: %s" % (path, err)) -@@ -933,7 +745,7 @@ class SosNode(): - """ - path = ''.join(path.split()) - try: -- if len(path) <= 2: # ensure we have a non '/' path -+ if len(path.split('/')) <= 2: # ensure we have a non '/' path - self.log_debug("Refusing to remove path %s: appears to be " - "incorrect and possibly dangerous" % path) - return False -@@ -959,14 +771,14 @@ class SosNode(): - except Exception: - self.log_error('Failed to make archive readable') - return False -- self.soslog.info('Retrieving sosreport from %s' % self.address) -- self.ui_msg('Retrieving sosreport...') -+ self.soslog.info('Retrieving sos report from %s' % self.address) -+ self.ui_msg('Retrieving sos report...') - ret = self.retrieve_file(self.sos_path) - if ret: -- self.ui_msg('Successfully collected sosreport') -+ self.ui_msg('Successfully collected sos report') - self.file_list.append(self.sos_path.split('/')[-1]) - else: -- self.log_error('Failed to retrieve sosreport') -+ self.log_error('Failed to retrieve sos report') - raise SystemExit - return True - else: -@@ -976,8 +788,8 @@ class SosNode(): - else: - e = [x.strip() for x in self.stdout.readlines() if x.strip][-1] - self.soslog.error( -- 'Failed to run sosreport on %s: %s' % (self.address, e)) -- self.log_error('Failed to run sosreport. %s' % e) -+ 'Failed to run sos report on %s: %s' % (self.address, e)) -+ self.log_error('Failed to run sos report. %s' % e) - return False - - def remove_sos_archive(self): -@@ -986,20 +798,20 @@ class SosNode(): - if self.sos_path is None: - return - if 'sosreport' not in self.sos_path: -- self.log_debug("Node sosreport path %s looks incorrect. Not " -+ self.log_debug("Node sos report path %s looks incorrect. Not " - "attempting to remove path" % self.sos_path) - return - removed = self.remove_file(self.sos_path) - if not removed: -- self.log_error('Failed to remove sosreport') -+ self.log_error('Failed to remove sos report') - - def cleanup(self): - """Remove the sos archive from the node once we have it locally""" - self.remove_sos_archive() - if self.sos_path: - for ext in ['.sha256', '.md5']: -- if os.path.isfile(self.sos_path + ext): -- self.remove_file(self.sos_path + ext) -+ if self.remove_file(self.sos_path + ext): -+ break - cleanup = self.host.set_cleanup_cmd() - if cleanup: - self.run_command(cleanup, need_root=True) -@@ -1040,3 +852,5 @@ class SosNode(): - msg = "Exception while making %s readable. Return code was %s" - self.log_error(msg % (filepath, res['status'])) - raise Exception -+ -+# vim: set et ts=4 sw=4 : -diff --git a/sos/collector/transports/__init__.py b/sos/collector/transports/__init__.py -new file mode 100644 -index 00000000..5be7dc6d ---- /dev/null -+++ b/sos/collector/transports/__init__.py -@@ -0,0 +1,317 @@ -+# Copyright Red Hat 2021, Jake Hunsaker -+ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+import inspect -+import logging -+import pexpect -+import re -+ -+from pipes import quote -+from sos.collector.exceptions import (ConnectionException, -+ CommandTimeoutException) -+ -+ -+class RemoteTransport(): -+ """The base class used for defining supported remote transports to connect -+ to remote nodes in conjunction with `sos collect`. -+ -+ This abstraction is used to manage the backend connections to nodes so that -+ SoSNode() objects can be leveraged generically to connect to nodes, inspect -+ those nodes, and run commands on them. -+ """ -+ -+ name = 'undefined' -+ -+ def __init__(self, address, commons): -+ self.address = address -+ self.opts = commons['cmdlineopts'] -+ self.tmpdir = commons['tmpdir'] -+ self.need_sudo = commons['need_sudo'] -+ self._hostname = None -+ self.soslog = logging.getLogger('sos') -+ self.ui_log = logging.getLogger('sos_ui') -+ -+ def _sanitize_log_msg(self, msg): -+ """Attempts to obfuscate sensitive information in log messages such as -+ passwords""" -+ reg = r'(?P(pass|key|secret|PASS|KEY|SECRET).*?=)(?P.*?\s)' -+ return re.sub(reg, r'\g****** ', msg) -+ -+ def log_info(self, msg): -+ """Used to print and log info messages""" -+ caller = inspect.stack()[1][3] -+ lmsg = '[%s:%s] %s' % (self.hostname, caller, msg) -+ self.soslog.info(lmsg) -+ -+ def log_error(self, msg): -+ """Used to print and log error messages""" -+ caller = inspect.stack()[1][3] -+ lmsg = '[%s:%s] %s' % (self.hostname, caller, msg) -+ self.soslog.error(lmsg) -+ -+ def log_debug(self, msg): -+ """Used to print and log debug messages""" -+ msg = self._sanitize_log_msg(msg) -+ caller = inspect.stack()[1][3] -+ msg = '[%s:%s] %s' % (self.hostname, caller, msg) -+ self.soslog.debug(msg) -+ -+ @property -+ def hostname(self): -+ if self._hostname and 'localhost' not in self._hostname: -+ return self._hostname -+ return self.address -+ -+ @property -+ def connected(self): -+ """Is the transport __currently__ connected to the node, or otherwise -+ capable of seamlessly running a command or similar on the node? -+ """ -+ return False -+ -+ @property -+ def remote_exec(self): -+ """This is the command string needed to leverage the remote transport -+ when executing commands. For example, for an SSH transport this would -+ be the `ssh ` string prepended to any command so that the -+ command is executed by the ssh binary. -+ -+ This is also referenced by the `remote_exec` parameter for policies -+ when loading a policy for a remote node -+ """ -+ return None -+ -+ def connect(self, password): -+ """Perform the connection steps in order to ensure that we are able to -+ connect to the node for all future operations. Note that this should -+ not provide an interactive shell at this time. -+ """ -+ if self._connect(password): -+ if not self._hostname: -+ self._get_hostname() -+ return True -+ return False -+ -+ def _connect(self, password): -+ """Actually perform the connection requirements. Should be overridden -+ by specific transports that subclass RemoteTransport -+ """ -+ raise NotImplementedError("Transport %s does not define connect" -+ % self.name) -+ -+ def reconnect(self, password): -+ """Attempts to reconnect to the node using the standard connect() -+ but does not do so indefinitely. This imposes a strict number of retry -+ attempts before failing out -+ """ -+ attempts = 1 -+ last_err = 'unknown' -+ while attempts < 5: -+ self.log_debug("Attempting reconnect (#%s) to node" % attempts) -+ try: -+ if self.connect(password): -+ return True -+ except Exception as err: -+ self.log_debug("Attempt #%s exception: %s" % (attempts, err)) -+ last_err = err -+ attempts += 1 -+ self.log_error("Unable to reconnect to node after 5 attempts, " -+ "aborting.") -+ raise ConnectionException("last exception from transport: %s" -+ % last_err) -+ -+ def disconnect(self): -+ """Perform whatever steps are necessary, if any, to terminate any -+ connection to the node -+ """ -+ try: -+ if self._disconnect(): -+ self.log_debug("Successfully disconnected from node") -+ else: -+ self.log_error("Unable to successfully disconnect, see log for" -+ " more details") -+ except Exception as err: -+ self.log_error("Failed to disconnect: %s" % err) -+ -+ def _disconnect(self): -+ raise NotImplementedError("Transport %s does not define disconnect" -+ % self.name) -+ -+ def run_command(self, cmd, timeout=180, need_root=False, env=None): -+ """Run a command on the node, returning its output and exit code. -+ This should return the exit code of the command being executed, not the -+ exit code of whatever mechanism the transport uses to execute that -+ command -+ -+ :param cmd: The command to run -+ :type cmd: ``str`` -+ -+ :param timeout: The maximum time in seconds to allow the cmd to run -+ :type timeout: ``int`` -+ -+ :param get_pty: Does ``cmd`` require a pty? -+ :type get_pty: ``bool`` -+ -+ :param need_root: Does ``cmd`` require root privileges? -+ :type neeed_root: ``bool`` -+ -+ :param env: Specify env vars to be passed to the ``cmd`` -+ :type env: ``dict`` -+ -+ :returns: Output of ``cmd`` and the exit code -+ :rtype: ``dict`` with keys ``output`` and ``status`` -+ """ -+ self.log_debug('Running command %s' % cmd) -+ # currently we only use/support the use of pexpect for handling the -+ # execution of these commands, as opposed to directly invoking -+ # subprocess.Popen() in conjunction with tools like sshpass. -+ # If that changes in the future, we'll add decision making logic here -+ # to route to the appropriate handler, but for now we just go straight -+ # to using pexpect -+ return self._run_command_with_pexpect(cmd, timeout, need_root, env) -+ -+ def _format_cmd_for_exec(self, cmd): -+ """Format the command in the way needed for the remote transport to -+ successfully execute it as one would when manually executing it -+ -+ :param cmd: The command being executed, as formatted by SoSNode -+ :type cmd: ``str`` -+ -+ -+ :returns: The command further formatted as needed by this -+ transport -+ :rtype: ``str`` -+ """ -+ cmd = "%s %s" % (self.remote_exec, quote(cmd)) -+ cmd = cmd.lstrip() -+ return cmd -+ -+ def _run_command_with_pexpect(self, cmd, timeout, need_root, env): -+ """Execute the command using pexpect, which allows us to more easily -+ handle prompts and timeouts compared to directly leveraging the -+ subprocess.Popen() method. -+ -+ :param cmd: The command to execute. This will be automatically -+ formatted to use the transport. -+ :type cmd: ``str`` -+ -+ :param timeout: The maximum time in seconds to run ``cmd`` -+ :type timeout: ``int`` -+ -+ :param need_root: Does ``cmd`` need to run as root or with sudo? -+ :type need_root: ``bool`` -+ -+ :param env: Any env vars that ``cmd`` should be run with -+ :type env: ``dict`` -+ """ -+ cmd = self._format_cmd_for_exec(cmd) -+ result = pexpect.spawn(cmd, encoding='utf-8', env=env) -+ -+ _expects = [pexpect.EOF, pexpect.TIMEOUT] -+ if need_root and self.opts.ssh_user != 'root': -+ _expects.extend([ -+ '\\[sudo\\] password for .*:', -+ 'Password:' -+ ]) -+ -+ index = result.expect(_expects, timeout=timeout) -+ -+ if index in [2, 3]: -+ self._send_pexpect_password(index, result) -+ index = result.expect(_expects, timeout=timeout) -+ -+ if index == 0: -+ out = result.before -+ result.close() -+ return {'status': result.exitstatus, 'output': out} -+ elif index == 1: -+ raise CommandTimeoutException(cmd) -+ -+ def _send_pexpect_password(self, index, result): -+ """Handle password prompts for sudo and su usage for non-root SSH users -+ -+ :param index: The index pexpect.spawn returned to match against -+ either a sudo or su prompt -+ :type index: ``int`` -+ -+ :param result: The spawn running the command -+ :type result: ``pexpect.spawn`` -+ """ -+ if index == 2: -+ if not self.opts.sudo_pw and not self.opt.nopasswd_sudo: -+ msg = ("Unable to run command: sudo password " -+ "required but not provided") -+ self.log_error(msg) -+ raise Exception(msg) -+ result.sendline(self.opts.sudo_pw) -+ elif index == 3: -+ if not self.opts.root_password: -+ msg = ("Unable to run command as root: no root password given") -+ self.log_error(msg) -+ raise Exception(msg) -+ result.sendline(self.opts.root_password) -+ -+ def _get_hostname(self): -+ """Determine the hostname of the node and set that for future reference -+ and logging -+ -+ :returns: The hostname of the system, per the `hostname` command -+ :rtype: ``str`` -+ """ -+ _out = self.run_command('hostname') -+ if _out['status'] == 0: -+ self._hostname = _out['output'].strip() -+ self.log_info("Hostname set to %s" % self._hostname) -+ return self._hostname -+ -+ def retrieve_file(self, fname, dest): -+ """Copy a remote file, fname, to dest on the local node -+ -+ :param fname: The name of the file to retrieve -+ :type fname: ``str`` -+ -+ :param dest: Where to save the file to locally -+ :type dest: ``str`` -+ -+ :returns: True if file was successfully copied from remote, or False -+ :rtype: ``bool`` -+ """ -+ return self._retrieve_file(fname, dest) -+ -+ def _retrieve_file(self, fname, dest): -+ raise NotImplementedError("Transport %s does not support file copying" -+ % self.name) -+ -+ def read_file(self, fname): -+ """Read the given file fname and return its contents -+ -+ :param fname: The name of the file to read -+ :type fname: ``str`` -+ -+ :returns: The content of the file -+ :rtype: ``str`` -+ """ -+ self.log_debug("Reading file %s" % fname) -+ return self._read_file(fname) -+ -+ def _read_file(self, fname): -+ res = self.run_command("cat %s" % fname, timeout=5) -+ if res['status'] == 0: -+ return res['output'] -+ else: -+ if 'No such file' in res['output']: -+ self.log_debug("File %s does not exist on node" -+ % fname) -+ else: -+ self.log_error("Error reading %s: %s" % -+ (fname, res['output'].split(':')[1:])) -+ return '' -+ -+# vim: set et ts=4 sw=4 : -diff --git a/sos/collector/transports/control_persist.py b/sos/collector/transports/control_persist.py -new file mode 100644 -index 00000000..3e848b41 ---- /dev/null -+++ b/sos/collector/transports/control_persist.py -@@ -0,0 +1,199 @@ -+# Copyright Red Hat 2021, Jake Hunsaker -+ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+ -+import os -+import pexpect -+import subprocess -+ -+from sos.collector.transports import RemoteTransport -+from sos.collector.exceptions import (InvalidPasswordException, -+ TimeoutPasswordAuthException, -+ PasswordRequestException, -+ AuthPermissionDeniedException, -+ ConnectionException, -+ ConnectionTimeoutException, -+ ControlSocketMissingException, -+ ControlPersistUnsupportedException) -+from sos.utilities import sos_get_command_output -+ -+ -+class SSHControlPersist(RemoteTransport): -+ """A transport for collect that leverages OpenSSH's Control Persist -+ functionality which uses control sockets to transparently keep a connection -+ open to the remote host without needing to rebuild the SSH connection for -+ each and every command executed on the node -+ """ -+ -+ name = 'control_persist' -+ -+ def _check_for_control_persist(self): -+ """Checks to see if the local system supported SSH ControlPersist. -+ -+ ControlPersist allows OpenSSH to keep a single open connection to a -+ remote host rather than building a new session each time. This is the -+ same feature that Ansible uses in place of paramiko, which we have a -+ need to drop in sos-collector. -+ -+ This check relies on feedback from the ssh binary. The command being -+ run should always generate stderr output, but depending on what that -+ output reads we can determine if ControlPersist is supported or not. -+ -+ For our purposes, a host that does not support ControlPersist is not -+ able to run sos-collector. -+ -+ Returns -+ True if ControlPersist is supported, else raise Exception. -+ """ -+ ssh_cmd = ['ssh', '-o', 'ControlPersist'] -+ cmd = subprocess.Popen(ssh_cmd, stdout=subprocess.PIPE, -+ stderr=subprocess.PIPE) -+ out, err = cmd.communicate() -+ err = err.decode('utf-8') -+ if 'Bad configuration option' in err or 'Usage:' in err: -+ raise ControlPersistUnsupportedException -+ return True -+ -+ def _connect(self, password=''): -+ """ -+ Using ControlPersist, create the initial connection to the node. -+ -+ This will generate an OpenSSH ControlPersist socket within the tmp -+ directory created or specified for sos-collector to use. -+ -+ At most, we will wait 30 seconds for a connection. This involves a 15 -+ second wait for the initial connection attempt, and a subsequent 15 -+ second wait for a response when we supply a password. -+ -+ Since we connect to nodes in parallel (using the --threads value), this -+ means that the time between 'Connecting to nodes...' and 'Beginning -+ collection of sosreports' that users see can be up to an amount of time -+ equal to 30*(num_nodes/threads) seconds. -+ -+ Returns -+ True if session is successfully opened, else raise Exception -+ """ -+ try: -+ self._check_for_control_persist() -+ except ControlPersistUnsupportedException: -+ self.log_error("OpenSSH ControlPersist is not locally supported. " -+ "Please update your OpenSSH installation.") -+ raise -+ self.log_info('Opening SSH session to create control socket') -+ self.control_path = ("%s/.sos-collector-%s" % (self.tmpdir, -+ self.address)) -+ self.ssh_cmd = '' -+ connected = False -+ ssh_key = '' -+ ssh_port = '' -+ if self.opts.ssh_port != 22: -+ ssh_port = "-p%s " % self.opts.ssh_port -+ if self.opts.ssh_key: -+ ssh_key = "-i%s" % self.opts.ssh_key -+ -+ cmd = ("ssh %s %s -oControlPersist=600 -oControlMaster=auto " -+ "-oStrictHostKeyChecking=no -oControlPath=%s %s@%s " -+ "\"echo Connected\"" % (ssh_key, -+ ssh_port, -+ self.control_path, -+ self.opts.ssh_user, -+ self.address)) -+ res = pexpect.spawn(cmd, encoding='utf-8') -+ -+ connect_expects = [ -+ u'Connected', -+ u'password:', -+ u'.*Permission denied.*', -+ u'.* port .*: No route to host', -+ u'.*Could not resolve hostname.*', -+ pexpect.TIMEOUT -+ ] -+ -+ index = res.expect(connect_expects, timeout=15) -+ -+ if index == 0: -+ connected = True -+ elif index == 1: -+ if password: -+ pass_expects = [ -+ u'Connected', -+ u'Permission denied, please try again.', -+ pexpect.TIMEOUT -+ ] -+ res.sendline(password) -+ pass_index = res.expect(pass_expects, timeout=15) -+ if pass_index == 0: -+ connected = True -+ elif pass_index == 1: -+ # Note that we do not get an exitstatus here, so matching -+ # this line means an invalid password will be reported for -+ # both invalid passwords and invalid user names -+ raise InvalidPasswordException -+ elif pass_index == 2: -+ raise TimeoutPasswordAuthException -+ else: -+ raise PasswordRequestException -+ elif index == 2: -+ raise AuthPermissionDeniedException -+ elif index == 3: -+ raise ConnectionException(self.address, self.opts.ssh_port) -+ elif index == 4: -+ raise ConnectionException(self.address) -+ elif index == 5: -+ raise ConnectionTimeoutException -+ else: -+ raise Exception("Unknown error, client returned %s" % res.before) -+ if connected: -+ if not os.path.exists(self.control_path): -+ raise ControlSocketMissingException -+ self.log_debug("Successfully created control socket at %s" -+ % self.control_path) -+ return True -+ return False -+ -+ def _disconnect(self): -+ if os.path.exists(self.control_path): -+ try: -+ os.remove(self.control_path) -+ return True -+ except Exception as err: -+ self.log_debug("Could not disconnect properly: %s" % err) -+ return False -+ self.log_debug("Control socket not present when attempting to " -+ "terminate session") -+ -+ @property -+ def connected(self): -+ """Check if the SSH control socket exists -+ -+ The control socket is automatically removed by the SSH daemon in the -+ event that the last connection to the node was greater than the timeout -+ set by the ControlPersist option. This can happen for us if we are -+ collecting from a large number of nodes, and the timeout expires before -+ we start collection. -+ """ -+ return os.path.exists(self.control_path) -+ -+ @property -+ def remote_exec(self): -+ if not self.ssh_cmd: -+ self.ssh_cmd = "ssh -oControlPath=%s %s@%s" % ( -+ self.control_path, self.opts.ssh_user, self.address -+ ) -+ return self.ssh_cmd -+ -+ def _retrieve_file(self, fname, dest): -+ cmd = "/usr/bin/scp -oControlPath=%s %s@%s:%s %s" % ( -+ self.control_path, self.opts.ssh_user, self.address, fname, dest -+ ) -+ res = sos_get_command_output(cmd) -+ return res['status'] == 0 -+ -+# vim: set et ts=4 sw=4 : -diff --git a/sos/collector/transports/local.py b/sos/collector/transports/local.py -new file mode 100644 -index 00000000..a4897f19 ---- /dev/null -+++ b/sos/collector/transports/local.py -@@ -0,0 +1,49 @@ -+# Copyright Red Hat 2021, Jake Hunsaker -+ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+import os -+import shutil -+ -+from sos.collector.transports import RemoteTransport -+ -+ -+class LocalTransport(RemoteTransport): -+ """A 'transport' to represent a local node. This allows us to more easily -+ extend SoSNode() without having a ton of 'if local' or similar checks in -+ more places than we actually need them -+ """ -+ -+ name = 'local_node' -+ -+ def _connect(self, password): -+ return True -+ -+ def _disconnect(self): -+ return True -+ -+ @property -+ def connected(self): -+ return True -+ -+ def _retrieve_file(self, fname, dest): -+ self.log_debug("Moving %s to %s" % (fname, dest)) -+ shutil.copy(fname, dest) -+ -+ def _format_cmd_for_exec(self, cmd): -+ return cmd -+ -+ def _read_file(self, fname): -+ if os.path.exists(fname): -+ with open(fname, 'r') as rfile: -+ return rfile.read() -+ self.log_debug("No such file: %s" % fname) -+ return '' -+ -+# vim: set et ts=4 sw=4 : --- -2.31.1 - -From 07d96d52ef69b9f8fe1ef32a1b88089d31c33fe8 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Mon, 27 Sep 2021 12:28:27 -0400 -Subject: [PATCH 2/2] [plugins] Update plugins to use new os.path.join wrapper - -Updates plugins to use the new `self.path_join()` wrapper for -`os.path.join()` so that these plugins now account for non-/ sysroots -for their collections. - -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/__init__.py | 2 +- - sos/report/plugins/azure.py | 4 +-- - sos/report/plugins/collectd.py | 2 +- - sos/report/plugins/container_log.py | 2 +- - sos/report/plugins/corosync.py | 2 +- - sos/report/plugins/docker_distribution.py | 5 ++-- - sos/report/plugins/ds.py | 3 +-- - sos/report/plugins/elastic.py | 4 ++- - sos/report/plugins/etcd.py | 2 +- - sos/report/plugins/gluster.py | 3 ++- - sos/report/plugins/jars.py | 2 +- - sos/report/plugins/kdump.py | 4 +-- - sos/report/plugins/libvirt.py | 2 +- - sos/report/plugins/logs.py | 8 +++--- - sos/report/plugins/manageiq.py | 12 ++++----- - sos/report/plugins/numa.py | 9 +++---- - sos/report/plugins/openstack_instack.py | 2 +- - sos/report/plugins/openstack_nova.py | 2 +- - sos/report/plugins/openvswitch.py | 13 ++++----- - sos/report/plugins/origin.py | 28 +++++++++++--------- - sos/report/plugins/ovirt.py | 2 +- - sos/report/plugins/ovirt_engine_backup.py | 5 ++-- - sos/report/plugins/ovn_central.py | 26 +++++++++--------- - sos/report/plugins/ovn_host.py | 4 +-- - sos/report/plugins/pacemaker.py | 4 +-- - sos/report/plugins/pcp.py | 32 +++++++++++------------ - sos/report/plugins/postfix.py | 2 +- - sos/report/plugins/postgresql.py | 2 +- - sos/report/plugins/powerpc.py | 2 +- - sos/report/plugins/processor.py | 3 +-- - sos/report/plugins/python.py | 4 +-- - sos/report/plugins/sar.py | 5 ++-- - sos/report/plugins/sos_extras.py | 2 +- - sos/report/plugins/ssh.py | 7 +++-- - sos/report/plugins/unpackaged.py | 4 +-- - sos/report/plugins/watchdog.py | 13 +++++---- - sos/report/plugins/yum.py | 2 +- - 37 files changed, 115 insertions(+), 115 deletions(-) - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index 1f84bca4..ec138f83 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -2897,7 +2897,7 @@ class Plugin(): - try: - cmd_line_paths = glob.glob(cmd_line_glob) - for path in cmd_line_paths: -- f = open(path, 'r') -+ f = open(self.path_join(path), 'r') - cmd_line = f.read().strip() - if process in cmd_line: - status = True -diff --git a/sos/report/plugins/azure.py b/sos/report/plugins/azure.py -index 45971a61..90999b3f 100644 ---- a/sos/report/plugins/azure.py -+++ b/sos/report/plugins/azure.py -@@ -8,8 +8,8 @@ - # - # See the LICENSE file in the source distribution for further information. - --import os - from sos.report.plugins import Plugin, UbuntuPlugin, RedHatPlugin -+import os - - - class Azure(Plugin, UbuntuPlugin): -@@ -38,7 +38,7 @@ class Azure(Plugin, UbuntuPlugin): - - for path, subdirs, files in os.walk("/var/log/azure"): - for name in files: -- self.add_copy_spec(os.path.join(path, name), sizelimit=limit) -+ self.add_copy_spec(self.path_join(path, name), sizelimit=limit) - - self.add_cmd_output(( - 'curl -s -H Metadata:true ' -diff --git a/sos/report/plugins/collectd.py b/sos/report/plugins/collectd.py -index 80d4b00a..8584adf9 100644 ---- a/sos/report/plugins/collectd.py -+++ b/sos/report/plugins/collectd.py -@@ -33,7 +33,7 @@ class Collectd(Plugin, IndependentPlugin): - - p = re.compile('^LoadPlugin.*') - try: -- with open("/etc/collectd.conf") as f: -+ with open(self.path_join("/etc/collectd.conf"), 'r') as f: - for line in f: - if p.match(line): - self.add_alert("Active Plugin found: %s" % -diff --git a/sos/report/plugins/container_log.py b/sos/report/plugins/container_log.py -index 14e0b7d8..e8dedad2 100644 ---- a/sos/report/plugins/container_log.py -+++ b/sos/report/plugins/container_log.py -@@ -29,6 +29,6 @@ class ContainerLog(Plugin, IndependentPlugin): - """Collect *.log files from subdirs of passed root path - """ - for dirName, _, _ in os.walk(root): -- self.add_copy_spec(os.path.join(dirName, '*.log')) -+ self.add_copy_spec(self.path_join(dirName, '*.log')) - - # vim: set et ts=4 sw=4 : -diff --git a/sos/report/plugins/corosync.py b/sos/report/plugins/corosync.py -index d74086e3..10e096c6 100644 ---- a/sos/report/plugins/corosync.py -+++ b/sos/report/plugins/corosync.py -@@ -47,7 +47,7 @@ class Corosync(Plugin): - # (it isnt precise but sufficient) - pattern = r'^\s*(logging.)?logfile:\s*(\S+)$' - try: -- with open("/etc/corosync/corosync.conf") as f: -+ with open(self.path_join("/etc/corosync/corosync.conf"), 'r') as f: - for line in f: - if re.match(pattern, line): - self.add_copy_spec(re.search(pattern, line).group(2)) -diff --git a/sos/report/plugins/docker_distribution.py b/sos/report/plugins/docker_distribution.py -index 84222ff7..e760f252 100644 ---- a/sos/report/plugins/docker_distribution.py -+++ b/sos/report/plugins/docker_distribution.py -@@ -19,8 +19,9 @@ class DockerDistribution(Plugin): - def setup(self): - self.add_copy_spec('/etc/docker-distribution/') - self.add_journal('docker-distribution') -- if self.path_exists('/etc/docker-distribution/registry/config.yml'): -- with open('/etc/docker-distribution/registry/config.yml') as f: -+ conf = self.path_join('/etc/docker-distribution/registry/config.yml') -+ if self.path_exists(conf): -+ with open(conf) as f: - for line in f: - if 'rootdirectory' in line: - loc = line.split()[1] -diff --git a/sos/report/plugins/ds.py b/sos/report/plugins/ds.py -index addf49e1..43feb21e 100644 ---- a/sos/report/plugins/ds.py -+++ b/sos/report/plugins/ds.py -@@ -11,7 +11,6 @@ - # See the LICENSE file in the source distribution for further information. - - from sos.report.plugins import Plugin, RedHatPlugin --import os - - - class DirectoryServer(Plugin, RedHatPlugin): -@@ -47,7 +46,7 @@ class DirectoryServer(Plugin, RedHatPlugin): - try: - for d in self.listdir("/etc/dirsrv"): - if d[0:5] == 'slapd': -- certpath = os.path.join("/etc/dirsrv", d) -+ certpath = self.path_join("/etc/dirsrv", d) - self.add_cmd_output("certutil -L -d %s" % certpath) - self.add_cmd_output("dsctl %s healthcheck" % d) - except OSError: -diff --git a/sos/report/plugins/elastic.py b/sos/report/plugins/elastic.py -index ad9a06ff..da2662bc 100644 ---- a/sos/report/plugins/elastic.py -+++ b/sos/report/plugins/elastic.py -@@ -39,7 +39,9 @@ class Elastic(Plugin, IndependentPlugin): - return hostname, port - - def setup(self): -- els_config_file = "/etc/elasticsearch/elasticsearch.yml" -+ els_config_file = self.path_join( -+ "/etc/elasticsearch/elasticsearch.yml" -+ ) - self.add_copy_spec(els_config_file) - - if self.get_option("all_logs"): -diff --git a/sos/report/plugins/etcd.py b/sos/report/plugins/etcd.py -index fd4f67eb..fe017e9f 100644 ---- a/sos/report/plugins/etcd.py -+++ b/sos/report/plugins/etcd.py -@@ -62,7 +62,7 @@ class etcd(Plugin, RedHatPlugin): - - def get_etcd_url(self): - try: -- with open('/etc/etcd/etcd.conf', 'r') as ef: -+ with open(self.path_join('/etc/etcd/etcd.conf'), 'r') as ef: - for line in ef: - if line.startswith('ETCD_LISTEN_CLIENT_URLS'): - return line.split('=')[1].replace('"', '').strip() -diff --git a/sos/report/plugins/gluster.py b/sos/report/plugins/gluster.py -index a44ffeb7..e518e3d3 100644 ---- a/sos/report/plugins/gluster.py -+++ b/sos/report/plugins/gluster.py -@@ -35,9 +35,10 @@ class Gluster(Plugin, RedHatPlugin): - ] - for statedump_file in statedump_entries: - statedumps_present = statedumps_present+1 -+ _spath = self.path_join(name_dir, statedump_file) - ret = -1 - while ret == -1: -- with open(name_dir + '/' + statedump_file, 'r') as sfile: -+ with open(_spath, 'r') as sfile: - last_line = sfile.readlines()[-1] - ret = string.count(last_line, 'DUMP_END_TIME') - -diff --git a/sos/report/plugins/jars.py b/sos/report/plugins/jars.py -index 0d3cf37e..4b98684e 100644 ---- a/sos/report/plugins/jars.py -+++ b/sos/report/plugins/jars.py -@@ -63,7 +63,7 @@ class Jars(Plugin, RedHatPlugin): - for location in locations: - for dirpath, _, filenames in os.walk(location): - for filename in filenames: -- path = os.path.join(dirpath, filename) -+ path = self.path_join(dirpath, filename) - if Jars.is_jar(path): - jar_paths.append(path) - -diff --git a/sos/report/plugins/kdump.py b/sos/report/plugins/kdump.py -index 757c2736..66565664 100644 ---- a/sos/report/plugins/kdump.py -+++ b/sos/report/plugins/kdump.py -@@ -40,7 +40,7 @@ class RedHatKDump(KDump, RedHatPlugin): - packages = ('kexec-tools',) - - def fstab_parse_fs(self, device): -- with open('/etc/fstab', 'r') as fp: -+ with open(self.path_join('/etc/fstab'), 'r') as fp: - for line in fp: - if line.startswith((device)): - return line.split()[1].rstrip('/') -@@ -50,7 +50,7 @@ class RedHatKDump(KDump, RedHatPlugin): - fs = "" - path = "/var/crash" - -- with open('/etc/kdump.conf', 'r') as fp: -+ with open(self.path_join('/etc/kdump.conf'), 'r') as fp: - for line in fp: - if line.startswith("path"): - path = line.split()[1] -diff --git a/sos/report/plugins/libvirt.py b/sos/report/plugins/libvirt.py -index be8120ff..5caa5802 100644 ---- a/sos/report/plugins/libvirt.py -+++ b/sos/report/plugins/libvirt.py -@@ -55,7 +55,7 @@ class Libvirt(Plugin, IndependentPlugin): - else: - self.add_copy_spec("/var/log/libvirt") - -- if self.path_exists(self.join_sysroot(libvirt_keytab)): -+ if self.path_exists(self.path_join(libvirt_keytab)): - self.add_cmd_output("klist -ket %s" % libvirt_keytab) - - self.add_cmd_output("ls -lR /var/lib/libvirt/qemu") -diff --git a/sos/report/plugins/logs.py b/sos/report/plugins/logs.py -index ee6bb98d..606e574a 100644 ---- a/sos/report/plugins/logs.py -+++ b/sos/report/plugins/logs.py -@@ -24,15 +24,15 @@ class Logs(Plugin, IndependentPlugin): - since = self.get_option("since") - - if self.path_exists('/etc/rsyslog.conf'): -- with open('/etc/rsyslog.conf', 'r') as conf: -+ with open(self.path_join('/etc/rsyslog.conf'), 'r') as conf: - for line in conf.readlines(): - if line.startswith('$IncludeConfig'): - confs += glob.glob(line.split()[1]) - - for conf in confs: -- if not self.path_exists(conf): -+ if not self.path_exists(self.path_join(conf)): - continue -- config = self.join_sysroot(conf) -+ config = self.path_join(conf) - logs += self.do_regex_find_all(r"^\S+\s+(-?\/.*$)\s+", config) - - for i in logs: -@@ -60,7 +60,7 @@ class Logs(Plugin, IndependentPlugin): - # - there is some data present, either persistent or runtime only - # - systemd-journald service exists - # otherwise fallback to collecting few well known logfiles directly -- journal = any([self.path_exists(p + "/log/journal/") -+ journal = any([self.path_exists(self.path_join(p, "log/journal/")) - for p in ["/var", "/run"]]) - if journal and self.is_service("systemd-journald"): - self.add_journal(since=since, tags='journal_full', priority=100) -diff --git a/sos/report/plugins/manageiq.py b/sos/report/plugins/manageiq.py -index 27ad6ef4..e20c4a2a 100644 ---- a/sos/report/plugins/manageiq.py -+++ b/sos/report/plugins/manageiq.py -@@ -58,7 +58,7 @@ class ManageIQ(Plugin, RedHatPlugin): - # Log files to collect from miq_dir/log/ - miq_log_dir = os.path.join(miq_dir, "log") - -- miq_main_log_files = [ -+ miq_main_logs = [ - 'ansible_tower.log', - 'top_output.log', - 'evm.log', -@@ -81,16 +81,16 @@ class ManageIQ(Plugin, RedHatPlugin): - self.add_copy_spec(list(self.files)) - - self.add_copy_spec([ -- os.path.join(self.miq_conf_dir, x) for x in self.miq_conf_files -+ self.path_join(self.miq_conf_dir, x) for x in self.miq_conf_files - ]) - - # Collect main log files without size limit. - self.add_copy_spec([ -- os.path.join(self.miq_log_dir, x) for x in self.miq_main_log_files -+ self.path_join(self.miq_log_dir, x) for x in self.miq_main_logs - ], sizelimit=0) - - self.add_copy_spec([ -- os.path.join(self.miq_log_dir, x) for x in self.miq_log_files -+ self.path_join(self.miq_log_dir, x) for x in self.miq_log_files - ]) - - self.add_copy_spec([ -@@ -101,8 +101,8 @@ class ManageIQ(Plugin, RedHatPlugin): - if environ.get("APPLIANCE_PG_DATA"): - pg_dir = environ.get("APPLIANCE_PG_DATA") - self.add_copy_spec([ -- os.path.join(pg_dir, 'pg_log'), -- os.path.join(pg_dir, 'postgresql.conf') -+ self.path_join(pg_dir, 'pg_log'), -+ self.path_join(pg_dir, 'postgresql.conf') - ]) - - # vim: set et ts=4 sw=4 : -diff --git a/sos/report/plugins/numa.py b/sos/report/plugins/numa.py -index 0faef8d2..9094baef 100644 ---- a/sos/report/plugins/numa.py -+++ b/sos/report/plugins/numa.py -@@ -9,7 +9,6 @@ - # See the LICENSE file in the source distribution for further information. - - from sos.report.plugins import Plugin, IndependentPlugin --import os.path - - - class Numa(Plugin, IndependentPlugin): -@@ -42,10 +41,10 @@ class Numa(Plugin, IndependentPlugin): - ]) - - self.add_copy_spec([ -- os.path.join(numa_path, "node*/meminfo"), -- os.path.join(numa_path, "node*/cpulist"), -- os.path.join(numa_path, "node*/distance"), -- os.path.join(numa_path, "node*/hugepages/hugepages-*/*") -+ self.path_join(numa_path, "node*/meminfo"), -+ self.path_join(numa_path, "node*/cpulist"), -+ self.path_join(numa_path, "node*/distance"), -+ self.path_join(numa_path, "node*/hugepages/hugepages-*/*") - ]) - - # vim: set et ts=4 sw=4 : -diff --git a/sos/report/plugins/openstack_instack.py b/sos/report/plugins/openstack_instack.py -index 7c56c162..5b4f7d41 100644 ---- a/sos/report/plugins/openstack_instack.py -+++ b/sos/report/plugins/openstack_instack.py -@@ -68,7 +68,7 @@ class OpenStackInstack(Plugin): - p = uc_config.get(opt) - if p: - if not os.path.isabs(p): -- p = os.path.join('/home/stack', p) -+ p = self.path_join('/home/stack', p) - self.add_copy_spec(p) - except Exception: - pass -diff --git a/sos/report/plugins/openstack_nova.py b/sos/report/plugins/openstack_nova.py -index 53210c48..f840081e 100644 ---- a/sos/report/plugins/openstack_nova.py -+++ b/sos/report/plugins/openstack_nova.py -@@ -103,7 +103,7 @@ class OpenStackNova(Plugin): - "nova-scheduler.log*" - ] - for novalog in novalogs: -- self.add_copy_spec(os.path.join(novadir, novalog)) -+ self.add_copy_spec(self.path_join(novadir, novalog)) - - self.add_copy_spec([ - "/etc/nova/", -diff --git a/sos/report/plugins/openvswitch.py b/sos/report/plugins/openvswitch.py -index 003596c6..179d1532 100644 ---- a/sos/report/plugins/openvswitch.py -+++ b/sos/report/plugins/openvswitch.py -@@ -10,7 +10,6 @@ - - from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin - --from os.path import join as path_join - from os import environ - - import re -@@ -65,7 +64,9 @@ class OpenVSwitch(Plugin): - log_dirs.append(environ.get('OVS_LOGDIR')) - - if not all_logs: -- self.add_copy_spec([path_join(ld, '*.log') for ld in log_dirs]) -+ self.add_copy_spec([ -+ self.path_join(ld, '*.log') for ld in log_dirs -+ ]) - else: - self.add_copy_spec(log_dirs) - -@@ -76,13 +77,13 @@ class OpenVSwitch(Plugin): - ]) - - self.add_copy_spec([ -- path_join('/usr/local/etc/openvswitch', 'conf.db'), -- path_join('/etc/openvswitch', 'conf.db'), -- path_join('/var/lib/openvswitch', 'conf.db'), -+ self.path_join('/usr/local/etc/openvswitch', 'conf.db'), -+ self.path_join('/etc/openvswitch', 'conf.db'), -+ self.path_join('/var/lib/openvswitch', 'conf.db'), - ]) - ovs_dbdir = environ.get('OVS_DBDIR') - if ovs_dbdir: -- self.add_copy_spec(path_join(ovs_dbdir, 'conf.db')) -+ self.add_copy_spec(self.path_join(ovs_dbdir, 'conf.db')) - - self.add_cmd_output([ - # The '-t 5' adds an upper bound on how long to wait to connect -diff --git a/sos/report/plugins/origin.py b/sos/report/plugins/origin.py -index f9cc32c1..7df9c019 100644 ---- a/sos/report/plugins/origin.py -+++ b/sos/report/plugins/origin.py -@@ -69,20 +69,21 @@ class OpenShiftOrigin(Plugin): - - def is_static_etcd(self): - """Determine if we are on a node running etcd""" -- return self.path_exists(os.path.join(self.static_pod_dir, "etcd.yaml")) -+ return self.path_exists(self.path_join(self.static_pod_dir, -+ "etcd.yaml")) - - def is_static_pod_compatible(self): - """Determine if a node is running static pods""" - return self.path_exists(self.static_pod_dir) - - def setup(self): -- bstrap_node_cfg = os.path.join(self.node_base_dir, -- "bootstrap-" + self.node_cfg_file) -- bstrap_kubeconfig = os.path.join(self.node_base_dir, -- "bootstrap.kubeconfig") -- node_certs = os.path.join(self.node_base_dir, "certs", "*") -- node_client_ca = os.path.join(self.node_base_dir, "client-ca.crt") -- admin_cfg = os.path.join(self.master_base_dir, "admin.kubeconfig") -+ bstrap_node_cfg = self.path_join(self.node_base_dir, -+ "bootstrap-" + self.node_cfg_file) -+ bstrap_kubeconfig = self.path_join(self.node_base_dir, -+ "bootstrap.kubeconfig") -+ node_certs = self.path_join(self.node_base_dir, "certs", "*") -+ node_client_ca = self.path_join(self.node_base_dir, "client-ca.crt") -+ admin_cfg = self.path_join(self.master_base_dir, "admin.kubeconfig") - oc_cmd_admin = "%s --config=%s" % ("oc", admin_cfg) - static_pod_logs_cmd = "master-logs" - -@@ -92,11 +93,12 @@ class OpenShiftOrigin(Plugin): - self.add_copy_spec([ - self.master_cfg, - self.master_env, -- os.path.join(self.master_base_dir, "*.crt"), -+ self.path_join(self.master_base_dir, "*.crt"), - ]) - - if self.is_static_pod_compatible(): -- self.add_copy_spec(os.path.join(self.static_pod_dir, "*.yaml")) -+ self.add_copy_spec(self.path_join(self.static_pod_dir, -+ "*.yaml")) - self.add_cmd_output([ - "%s api api" % static_pod_logs_cmd, - "%s controllers controllers" % static_pod_logs_cmd, -@@ -177,9 +179,9 @@ class OpenShiftOrigin(Plugin): - node_client_ca, - bstrap_node_cfg, - bstrap_kubeconfig, -- os.path.join(self.node_base_dir, "*.crt"), -- os.path.join(self.node_base_dir, "resolv.conf"), -- os.path.join(self.node_base_dir, "node-dnsmasq.conf"), -+ self.path_join(self.node_base_dir, "*.crt"), -+ self.path_join(self.node_base_dir, "resolv.conf"), -+ self.path_join(self.node_base_dir, "node-dnsmasq.conf"), - ]) - - self.add_journal(units="atomic-openshift-node") -diff --git a/sos/report/plugins/ovirt.py b/sos/report/plugins/ovirt.py -index 1de606be..09647bf1 100644 ---- a/sos/report/plugins/ovirt.py -+++ b/sos/report/plugins/ovirt.py -@@ -216,7 +216,7 @@ class Ovirt(Plugin, RedHatPlugin): - "isouploader.conf" - ] - for conf_file in passwd_files: -- conf_path = os.path.join("/etc/ovirt-engine", conf_file) -+ conf_path = self.path_join("/etc/ovirt-engine", conf_file) - self.do_file_sub( - conf_path, - r"passwd=(.*)", -diff --git a/sos/report/plugins/ovirt_engine_backup.py b/sos/report/plugins/ovirt_engine_backup.py -index 676e419e..7fb6a5c7 100644 ---- a/sos/report/plugins/ovirt_engine_backup.py -+++ b/sos/report/plugins/ovirt_engine_backup.py -@@ -8,7 +8,6 @@ - # - # See the LICENSE file in the source distribution for further information. - --import os - from sos.report.plugins import (Plugin, RedHatPlugin) - from datetime import datetime - -@@ -29,11 +28,11 @@ class oVirtEngineBackup(Plugin, RedHatPlugin): - - def setup(self): - now = datetime.now().strftime("%Y%m%d%H%M%S") -- backup_filename = os.path.join( -+ backup_filename = self.path_join( - self.get_option("backupdir"), - "engine-db-backup-%s.tar.gz" % (now) - ) -- log_filename = os.path.join( -+ log_filename = self.path_join( - self.get_option("backupdir"), - "engine-db-backup-%s.log" % (now) - ) -diff --git a/sos/report/plugins/ovn_central.py b/sos/report/plugins/ovn_central.py -index d6647aad..914eda60 100644 ---- a/sos/report/plugins/ovn_central.py -+++ b/sos/report/plugins/ovn_central.py -@@ -42,7 +42,7 @@ class OVNCentral(Plugin): - return - else: - try: -- with open(filename, 'r') as f: -+ with open(self.path_join(filename), 'r') as f: - try: - db = json.load(f) - except Exception: -@@ -71,13 +71,13 @@ class OVNCentral(Plugin): - ovs_rundir = os.environ.get('OVS_RUNDIR') - for pidfile in ['ovnnb_db.pid', 'ovnsb_db.pid', 'ovn-northd.pid']: - self.add_copy_spec([ -- os.path.join('/var/lib/openvswitch/ovn', pidfile), -- os.path.join('/usr/local/var/run/openvswitch', pidfile), -- os.path.join('/run/openvswitch/', pidfile), -+ self.path_join('/var/lib/openvswitch/ovn', pidfile), -+ self.path_join('/usr/local/var/run/openvswitch', pidfile), -+ self.path_join('/run/openvswitch/', pidfile), - ]) - - if ovs_rundir: -- self.add_copy_spec(os.path.join(ovs_rundir, pidfile)) -+ self.add_copy_spec(self.path_join(ovs_rundir, pidfile)) - - if self.get_option("all_logs"): - self.add_copy_spec("/var/log/ovn/") -@@ -104,7 +104,7 @@ class OVNCentral(Plugin): - - schema_dir = '/usr/share/openvswitch' - -- nb_tables = self.get_tables_from_schema(os.path.join( -+ nb_tables = self.get_tables_from_schema(self.path_join( - schema_dir, 'ovn-nb.ovsschema')) - - self.add_database_output(nb_tables, nbctl_cmds, 'ovn-nbctl') -@@ -116,7 +116,7 @@ class OVNCentral(Plugin): - format(self.ovn_sbdb_sock_path), - "output": "Leader: self"} - if self.test_predicate(self, pred=SoSPredicate(self, cmd_outputs=co)): -- sb_tables = self.get_tables_from_schema(os.path.join( -+ sb_tables = self.get_tables_from_schema(self.path_join( - schema_dir, 'ovn-sb.ovsschema'), ['Logical_Flow']) - self.add_database_output(sb_tables, sbctl_cmds, 'ovn-sbctl') - cmds += sbctl_cmds -@@ -134,14 +134,14 @@ class OVNCentral(Plugin): - ovs_dbdir = os.environ.get('OVS_DBDIR') - for dbfile in ['ovnnb_db.db', 'ovnsb_db.db']: - self.add_copy_spec([ -- os.path.join('/var/lib/openvswitch/ovn', dbfile), -- os.path.join('/usr/local/etc/openvswitch', dbfile), -- os.path.join('/etc/openvswitch', dbfile), -- os.path.join('/var/lib/openvswitch', dbfile), -- os.path.join('/var/lib/ovn/etc', dbfile), -+ self.path_join('/var/lib/openvswitch/ovn', dbfile), -+ self.path_join('/usr/local/etc/openvswitch', dbfile), -+ self.path_join('/etc/openvswitch', dbfile), -+ self.path_join('/var/lib/openvswitch', dbfile), -+ self.path_join('/var/lib/ovn/etc', dbfile) - ]) - if ovs_dbdir: -- self.add_copy_spec(os.path.join(ovs_dbdir, dbfile)) -+ self.add_copy_spec(self.path_join(ovs_dbdir, dbfile)) - - self.add_journal(units="ovn-northd") - -diff --git a/sos/report/plugins/ovn_host.py b/sos/report/plugins/ovn_host.py -index 3742c49f..78604a15 100644 ---- a/sos/report/plugins/ovn_host.py -+++ b/sos/report/plugins/ovn_host.py -@@ -35,7 +35,7 @@ class OVNHost(Plugin): - else: - self.add_copy_spec("/var/log/ovn/*.log") - -- self.add_copy_spec([os.path.join(pp, pidfile) for pp in pid_paths]) -+ self.add_copy_spec([self.path_join(pp, pidfile) for pp in pid_paths]) - - self.add_copy_spec('/etc/sysconfig/ovn-controller') - -@@ -49,7 +49,7 @@ class OVNHost(Plugin): - - def check_enabled(self): - return (any([self.path_isfile( -- os.path.join(pp, pidfile)) for pp in pid_paths]) or -+ self.path_join(pp, pidfile)) for pp in pid_paths]) or - super(OVNHost, self).check_enabled()) - - -diff --git a/sos/report/plugins/pacemaker.py b/sos/report/plugins/pacemaker.py -index 497807ff..6ce80881 100644 ---- a/sos/report/plugins/pacemaker.py -+++ b/sos/report/plugins/pacemaker.py -@@ -129,7 +129,7 @@ class Pacemaker(Plugin): - - class DebianPacemaker(Pacemaker, DebianPlugin, UbuntuPlugin): - def setup(self): -- self.envfile = "/etc/default/pacemaker" -+ self.envfile = self.path_join("/etc/default/pacemaker") - self.setup_crm_shell() - self.setup_pcs() - super(DebianPacemaker, self).setup() -@@ -141,7 +141,7 @@ class DebianPacemaker(Pacemaker, DebianPlugin, UbuntuPlugin): - - class RedHatPacemaker(Pacemaker, RedHatPlugin): - def setup(self): -- self.envfile = "/etc/sysconfig/pacemaker" -+ self.envfile = self.path_join("/etc/sysconfig/pacemaker") - self.setup_pcs() - self.add_copy_spec("/etc/sysconfig/sbd") - super(RedHatPacemaker, self).setup() -diff --git a/sos/report/plugins/pcp.py b/sos/report/plugins/pcp.py -index 9707d7a9..ad902332 100644 ---- a/sos/report/plugins/pcp.py -+++ b/sos/report/plugins/pcp.py -@@ -41,7 +41,7 @@ class Pcp(Plugin, RedHatPlugin, DebianPlugin): - total_size = 0 - for dirpath, dirnames, filenames in os.walk(path): - for f in filenames: -- fp = os.path.join(dirpath, f) -+ fp = self.path_join(dirpath, f) - total_size += os.path.getsize(fp) - return total_size - -@@ -86,7 +86,7 @@ class Pcp(Plugin, RedHatPlugin, DebianPlugin): - # unconditionally. Obviously if someone messes up their /etc/pcp.conf - # in a ridiculous way (i.e. setting PCP_SYSCONF_DIR to '/') this will - # break badly. -- var_conf_dir = os.path.join(self.pcp_var_dir, 'config') -+ var_conf_dir = self.path_join(self.pcp_var_dir, 'config') - self.add_copy_spec([ - self.pcp_sysconf_dir, - self.pcp_conffile, -@@ -98,10 +98,10 @@ class Pcp(Plugin, RedHatPlugin, DebianPlugin): - # rpms. It does not make up for a lot of size but it contains many - # files - self.add_forbidden_path([ -- os.path.join(var_conf_dir, 'pmchart'), -- os.path.join(var_conf_dir, 'pmlogconf'), -- os.path.join(var_conf_dir, 'pmieconf'), -- os.path.join(var_conf_dir, 'pmlogrewrite') -+ self.path_join(var_conf_dir, 'pmchart'), -+ self.path_join(var_conf_dir, 'pmlogconf'), -+ self.path_join(var_conf_dir, 'pmieconf'), -+ self.path_join(var_conf_dir, 'pmlogrewrite') - ]) - - # Take PCP_LOG_DIR/pmlogger/`hostname` + PCP_LOG_DIR/pmmgr/`hostname` -@@ -121,13 +121,13 @@ class Pcp(Plugin, RedHatPlugin, DebianPlugin): - # we would collect everything - if self.pcp_hostname != '': - # collect pmmgr logs up to 'pmmgrlogs' size limit -- path = os.path.join(self.pcp_log_dir, 'pmmgr', -- self.pcp_hostname, '*') -+ path = self.path_join(self.pcp_log_dir, 'pmmgr', -+ self.pcp_hostname, '*') - self.add_copy_spec(path, sizelimit=self.sizelimit, tailit=False) - # collect newest pmlogger logs up to 'pmloggerfiles' count - files_collected = 0 -- path = os.path.join(self.pcp_log_dir, 'pmlogger', -- self.pcp_hostname, '*') -+ path = self.path_join(self.pcp_log_dir, 'pmlogger', -+ self.pcp_hostname, '*') - pmlogger_ls = self.exec_cmd("ls -t1 %s" % path) - if pmlogger_ls['status'] == 0: - for line in pmlogger_ls['output'].splitlines(): -@@ -138,15 +138,15 @@ class Pcp(Plugin, RedHatPlugin, DebianPlugin): - - self.add_copy_spec([ - # Collect PCP_LOG_DIR/pmcd and PCP_LOG_DIR/NOTICES -- os.path.join(self.pcp_log_dir, 'pmcd'), -- os.path.join(self.pcp_log_dir, 'NOTICES*'), -+ self.path_join(self.pcp_log_dir, 'pmcd'), -+ self.path_join(self.pcp_log_dir, 'NOTICES*'), - # Collect PCP_VAR_DIR/pmns -- os.path.join(self.pcp_var_dir, 'pmns'), -+ self.path_join(self.pcp_var_dir, 'pmns'), - # Also collect any other log and config files - # (as suggested by fche) -- os.path.join(self.pcp_log_dir, '*/*.log*'), -- os.path.join(self.pcp_log_dir, '*/*/*.log*'), -- os.path.join(self.pcp_log_dir, '*/*/config*') -+ self.path_join(self.pcp_log_dir, '*/*.log*'), -+ self.path_join(self.pcp_log_dir, '*/*/*.log*'), -+ self.path_join(self.pcp_log_dir, '*/*/config*') - ]) - - # Collect a summary for the current day -diff --git a/sos/report/plugins/postfix.py b/sos/report/plugins/postfix.py -index 8f584430..3ca0c4ad 100644 ---- a/sos/report/plugins/postfix.py -+++ b/sos/report/plugins/postfix.py -@@ -41,7 +41,7 @@ class Postfix(Plugin): - ] - fp = [] - try: -- with open('/etc/postfix/main.cf', 'r') as cffile: -+ with open(self.path_join('/etc/postfix/main.cf'), 'r') as cffile: - for line in cffile.readlines(): - # ignore comments and take the first word after '=' - if line.startswith('#'): -diff --git a/sos/report/plugins/postgresql.py b/sos/report/plugins/postgresql.py -index bec0b019..00824db7 100644 ---- a/sos/report/plugins/postgresql.py -+++ b/sos/report/plugins/postgresql.py -@@ -124,7 +124,7 @@ class RedHatPostgreSQL(PostgreSQL, SCLPlugin): - - # copy PG_VERSION and postmaster.opts - for f in ["PG_VERSION", "postmaster.opts"]: -- self.add_copy_spec(os.path.join(_dir, "data", f)) -+ self.add_copy_spec(self.path_join(_dir, "data", f)) - - - class DebianPostgreSQL(PostgreSQL, DebianPlugin, UbuntuPlugin): -diff --git a/sos/report/plugins/powerpc.py b/sos/report/plugins/powerpc.py -index 4fb4f87c..50f88650 100644 ---- a/sos/report/plugins/powerpc.py -+++ b/sos/report/plugins/powerpc.py -@@ -22,7 +22,7 @@ class PowerPC(Plugin, IndependentPlugin): - - def setup(self): - try: -- with open('/proc/cpuinfo', 'r') as fp: -+ with open(self.path_join('/proc/cpuinfo'), 'r') as fp: - contents = fp.read() - ispSeries = "pSeries" in contents - isPowerNV = "PowerNV" in contents -diff --git a/sos/report/plugins/processor.py b/sos/report/plugins/processor.py -index 2df2dc9a..c3d8930c 100644 ---- a/sos/report/plugins/processor.py -+++ b/sos/report/plugins/processor.py -@@ -7,7 +7,6 @@ - # See the LICENSE file in the source distribution for further information. - - from sos.report.plugins import Plugin, IndependentPlugin --import os - - - class Processor(Plugin, IndependentPlugin): -@@ -41,7 +40,7 @@ class Processor(Plugin, IndependentPlugin): - # cumulative directory size exceeds 25MB or even 100MB. - cdirs = self.listdir('/sys/devices/system/cpu') - self.add_copy_spec([ -- os.path.join('/sys/devices/system/cpu', cdir) for cdir in cdirs -+ self.path_join('/sys/devices/system/cpu', cdir) for cdir in cdirs - ]) - - self.add_cmd_output([ -diff --git a/sos/report/plugins/python.py b/sos/report/plugins/python.py -index e2ab39ab..a8ec0cd8 100644 ---- a/sos/report/plugins/python.py -+++ b/sos/report/plugins/python.py -@@ -68,9 +68,9 @@ class RedHatPython(Python, RedHatPlugin): - ] - - for py_path in py_paths: -- for root, _, files in os.walk(py_path): -+ for root, _, files in os.walk(self.path_join(py_path)): - for file_ in files: -- filepath = os.path.join(root, file_) -+ filepath = self.path_join(root, file_) - if filepath.endswith('.py'): - try: - with open(filepath, 'rb') as f: -diff --git a/sos/report/plugins/sar.py b/sos/report/plugins/sar.py -index 669f5d7b..b60005b1 100644 ---- a/sos/report/plugins/sar.py -+++ b/sos/report/plugins/sar.py -@@ -7,7 +7,6 @@ - # See the LICENSE file in the source distribution for further information. - - from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin --import os - import re - - -@@ -24,7 +23,7 @@ class Sar(Plugin,): - "", False)] - - def setup(self): -- self.add_copy_spec(os.path.join(self.sa_path, '*'), -+ self.add_copy_spec(self.path_join(self.sa_path, '*'), - sizelimit=0 if self.get_option("all_sar") else None, - tailit=False) - -@@ -44,7 +43,7 @@ class Sar(Plugin,): - # as option for sadc - for fname in dir_list: - if sa_regex.match(fname): -- sa_data_path = os.path.join(self.sa_path, fname) -+ sa_data_path = self.path_join(self.sa_path, fname) - sar_filename = 'sar' + fname[2:] - if sar_filename not in dir_list: - sar_cmd = 'sh -c "sar -A -f %s"' % sa_data_path -diff --git a/sos/report/plugins/sos_extras.py b/sos/report/plugins/sos_extras.py -index ffde4138..55bc4dc0 100644 ---- a/sos/report/plugins/sos_extras.py -+++ b/sos/report/plugins/sos_extras.py -@@ -58,7 +58,7 @@ class SosExtras(Plugin, IndependentPlugin): - - for path, dirlist, filelist in os.walk(self.extras_dir): - for f in filelist: -- _file = os.path.join(path, f) -+ _file = self.path_join(path, f) - self._log_warn("Collecting data from extras file %s" % _file) - try: - for line in open(_file).read().splitlines(): -diff --git a/sos/report/plugins/ssh.py b/sos/report/plugins/ssh.py -index 971cda8b..9ac9dec0 100644 ---- a/sos/report/plugins/ssh.py -+++ b/sos/report/plugins/ssh.py -@@ -42,7 +41,7 @@ class Ssh(Plugin, IndependentPlugin): - try: - for sshcfg in sshcfgs: - tag = sshcfg.split('/')[-1] -- with open(sshcfg, 'r') as cfgfile: -+ with open(self.path_join(sshcfg), 'r') as cfgfile: - for line in cfgfile: - # skip empty lines and comments - if len(line.split()) == 0 or line.startswith('#'): -diff --git a/sos/report/plugins/unpackaged.py b/sos/report/plugins/unpackaged.py -index 9205e53f..772b1d1f 100644 ---- a/sos/report/plugins/unpackaged.py -+++ b/sos/report/plugins/unpackaged.py -@@ -40,7 +40,7 @@ class Unpackaged(Plugin, RedHatPlugin): - for e in exclude: - dirs[:] = [d for d in dirs if d not in e] - for name in files: -- path = os.path.join(root, name) -+ path = self.path_join(root, name) - try: - if stat.S_ISLNK(os.lstat(path).st_mode): - path = Path(path).resolve() -@@ -49,7 +49,7 @@ class Unpackaged(Plugin, RedHatPlugin): - file_list.append(os.path.realpath(path)) - for name in dirs: - file_list.append(os.path.realpath( -- os.path.join(root, name))) -+ self.path_join(root, name))) - - return file_list - -diff --git a/sos/report/plugins/watchdog.py b/sos/report/plugins/watchdog.py -index 1bf3f4cb..bf2dc9cb 100644 ---- a/sos/report/plugins/watchdog.py -+++ b/sos/report/plugins/watchdog.py -@@ -11,7 +11,6 @@ - from sos.report.plugins import Plugin, RedHatPlugin - - from glob import glob --import os - - - class Watchdog(Plugin, RedHatPlugin): -@@ -56,8 +55,8 @@ class Watchdog(Plugin, RedHatPlugin): - Collect configuration files, custom executables for test-binary - and repair-binary, and stdout/stderr logs. - """ -- conf_file = self.get_option('conf_file') -- log_dir = '/var/log/watchdog' -+ conf_file = self.path_join(self.get_option('conf_file')) -+ log_dir = self.path_join('/var/log/watchdog') - - # Get service configuration and sysconfig files - self.add_copy_spec([ -@@ -80,15 +79,15 @@ class Watchdog(Plugin, RedHatPlugin): - self._log_warn("Could not read %s: %s" % (conf_file, ex)) - - if self.get_option('all_logs'): -- log_files = glob(os.path.join(log_dir, '*')) -+ log_files = glob(self.path_join(log_dir, '*')) - else: -- log_files = (glob(os.path.join(log_dir, '*.stdout')) + -- glob(os.path.join(log_dir, '*.stderr'))) -+ log_files = (glob(self.path_join(log_dir, '*.stdout')) + -+ glob(self.path_join(log_dir, '*.stderr'))) - - self.add_copy_spec(log_files) - - # Get output of "wdctl " for each /dev/watchdog* -- for dev in glob('/dev/watchdog*'): -+ for dev in glob(self.path_join('/dev/watchdog*')): - self.add_cmd_output("wdctl %s" % dev) - - # vim: set et ts=4 sw=4 : -diff --git a/sos/report/plugins/yum.py b/sos/report/plugins/yum.py -index 148464cb..e5256642 100644 ---- a/sos/report/plugins/yum.py -+++ b/sos/report/plugins/yum.py -@@ -61,7 +61,7 @@ class Yum(Plugin, RedHatPlugin): - if not p.endswith(".py"): - continue - plugins = plugins + " " if len(plugins) else "" -- plugins = plugins + os.path.join(YUM_PLUGIN_PATH, p) -+ plugins = plugins + self.path_join(YUM_PLUGIN_PATH, p) - if len(plugins): - self.add_cmd_output("rpm -qf %s" % plugins, - suggest_filename="plugin-packages") --- -2.31.1 - -From f4af5efdc79aefe1aa685c36d095925bae14dc4a Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 28 Sep 2021 13:00:17 -0400 -Subject: [PATCH 1/4] [collect] Add --transport option and allow clusters to - set transport type - -Adds a new `--transport` option for users to be able to specify the type -of transport to use when connecting to nodes. The default value of -`auto` will defer to the cluster profile to set the transport type, -which will continue to default to use OpenSSH's ControlPersist feature. - -Clusters may override the new `set_transport_type()` method to change -the default transport used. - -If `--transport` is anything besides `auto`, then the cluster profile -will not be deferred to when choosing a transport for each remote node. - -Signed-off-by: Jake Hunsaker ---- - man/en/sos-collect.1 | 15 +++++++++++++++ - sos/collector/__init__.py | 6 ++++++ - sos/collector/clusters/__init__.py | 10 ++++++++++ - sos/collector/exceptions.py | 13 ++++++++++++- - sos/collector/sosnode.py | 16 +++++++++++++++- - 5 files changed, 58 insertions(+), 2 deletions(-) - -diff --git a/man/en/sos-collect.1 b/man/en/sos-collect.1 -index e930023e..8ad4fe5e 100644 ---- a/man/en/sos-collect.1 -+++ b/man/en/sos-collect.1 -@@ -43,6 +43,7 @@ sos collect \- Collect sosreports from multiple (cluster) nodes - [\-\-sos-cmd SOS_CMD] - [\-t|\-\-threads THREADS] - [\-\-timeout TIMEOUT] -+ [\-\-transport TRANSPORT] - [\-\-tmp\-dir TMP_DIR] - [\-v|\-\-verbose] - [\-\-verify] -@@ -350,6 +351,20 @@ Note that sosreports are collected in parallel, so you can approximate the total - runtime of sos collect via timeout*(number of nodes/jobs). - - Default is 180 seconds. -+.TP -+\fB\-\-transport\fR TRANSPORT -+Specify the type of remote transport to use to manage connections to remote nodes. -+ -+\fBsos collect\fR uses locally installed binaries to connect to and interact with remote -+nodes, instead of directly establishing those connections. By default, OpenSSH's ControlPersist -+feature is preferred, however certain cluster types may have preferences of their own for how -+remote sessions should be established. -+ -+The types of transports supported are currently as follows: -+ -+ \fBauto\fR Allow the cluster type to determine the transport used -+ \fBcontrol_persist\fR Use OpenSSH's ControlPersist feature. This is the default behavior -+ - .TP - \fB\-\-tmp\-dir\fR TMP_DIR - Specify a temporary directory to save sos archives to. By default one will be created in -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index da912655..fecfe6aa 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -98,6 +98,7 @@ class SoSCollector(SoSComponent): - 'ssh_port': 22, - 'ssh_user': 'root', - 'timeout': 600, -+ 'transport': 'auto', - 'verify': False, - 'usernames': [], - 'upload': False, -@@ -378,6 +379,8 @@ class SoSCollector(SoSComponent): - help='Specify an SSH user. Default root') - collect_grp.add_argument('--timeout', type=int, required=False, - help='Timeout for sosreport on each node.') -+ collect_grp.add_argument('--transport', default='auto', type=str, -+ help='Remote connection transport to use') - collect_grp.add_argument("--upload", action="store_true", - default=False, - help="Upload archive to a policy-default " -@@ -813,6 +813,8 @@ class SoSCollector(SoSComponent): - self.collect_md.add_field('cluster_type', self.cluster_type) - if self.cluster: - self.master.cluster = self.cluster -+ if self.opts.transport == 'auto': -+ self.opts.transport = self.cluster.set_transport_type() - self.cluster.setup() - if self.cluster.cluster_ssh_key: - if not self.opts.ssh_key: -@@ -1041,6 +1046,7 @@ class SoSCollector(SoSComponent): - else: - client.disconnect() - except Exception: -+ # all exception logging is handled within SoSNode - pass - - def intro(self): -diff --git a/sos/collector/clusters/__init__.py b/sos/collector/clusters/__init__.py -index 64ac2a44..cf1e7a0b 100644 ---- a/sos/collector/clusters/__init__.py -+++ b/sos/collector/clusters/__init__.py -@@ -149,6 +149,16 @@ class Cluster(): - """ - pass - -+ def set_transport_type(self): -+ """The default connection type used by sos collect is to leverage the -+ local system's SSH installation using ControlPersist, however certain -+ cluster types may want to use something else. -+ -+ Override this in a specific cluster profile to set the ``transport`` -+ option according to what type of transport should be used. -+ """ -+ return 'control_persist' -+ - def set_master_options(self, node): - """If there is a need to set specific options in the sos command being - run on the cluster's master nodes, override this method in the cluster -diff --git a/sos/collector/exceptions.py b/sos/collector/exceptions.py -index 1e44768b..2bb07e7b 100644 ---- a/sos/collector/exceptions.py -+++ b/sos/collector/exceptions.py -@@ -94,6 +94,16 @@ class UnsupportedHostException(Exception): - super(UnsupportedHostException, self).__init__(message) - - -+class InvalidTransportException(Exception): -+ """Raised when a transport is requested but it does not exist or is -+ not supported locally""" -+ -+ def __init__(self, transport=None): -+ message = ("Connection failed: unknown or unsupported transport %s" -+ % transport if transport else '') -+ super(InvalidTransportException, self).__init__(message) -+ -+ - __all__ = [ - 'AuthPermissionDeniedException', - 'CommandTimeoutException', -@@ -104,5 +114,6 @@ __all__ = [ - 'InvalidPasswordException', - 'PasswordRequestException', - 'TimeoutPasswordAuthException', -- 'UnsupportedHostException' -+ 'UnsupportedHostException', -+ 'InvalidTransportException' - ] -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index f79bd5ff..5c5c7201 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -22,7 +22,13 @@ from sos.collector.transports.control_persist import SSHControlPersist - from sos.collector.transports.local import LocalTransport - from sos.collector.exceptions import (CommandTimeoutException, - ConnectionException, -- UnsupportedHostException) -+ UnsupportedHostException, -+ InvalidTransportException) -+ -+TRANSPORTS = { -+ 'local': LocalTransport, -+ 'control_persist': SSHControlPersist, -+} - - - class SosNode(): -@@ -107,6 +113,14 @@ class SosNode(): - if self.address in ['localhost', '127.0.0.1']: - self.local = True - return LocalTransport(self.address, commons) -+ elif self.opts.transport in TRANSPORTS.keys(): -+ return TRANSPORTS[self.opts.transport](self.address, commons) -+ elif self.opts.transport != 'auto': -+ self.log_error( -+ "Connection failed: unknown or unsupported transport %s" -+ % self.opts.transport -+ ) -+ raise InvalidTransportException(self.opts.transport) - return SSHControlPersist(self.address, commons) - - def _fmt_msg(self, msg): --- -2.31.1 - - -From dbc49345384404600f45d68b8d3c6541b2a26480 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 30 Sep 2021 10:38:18 -0400 -Subject: [PATCH 2/4] [transports] Add 'oc' as a transport option for remote - nodes - -This commit adds a new transport for `sos collect` by leveraging a -locally available `oc` binary that has been properly configured for -access to an OCP cluster. - -This transport will allow users to use `sos collect` to collect reports -from an OCP cluster without directly connecting to any of the nodes -involved. We do this by using the `oc` binary to first launch a pod on -target node(s) and then exec our discovery commands and eventual `sos -report` command to that pod. This in turn is dependent on a function API -for the `oc` binary to communicate with. In the event that `oc` is not -__locally__ available or is not properly configured, we will fallback to -the current default of using SSH ControlPersist to directly connect to -the nodes. Otherwise, the OCP cluster will attempt to automatically use -this new transport. ---- - man/en/sos-collect.1 | 1 + - sos/collector/clusters/ocp.py | 14 ++ - sos/collector/sosnode.py | 8 +- - sos/collector/transports/__init__.py | 20 ++- - sos/collector/transports/oc.py | 220 +++++++++++++++++++++++++++ - 5 files changed, 257 insertions(+), 6 deletions(-) - create mode 100644 sos/collector/transports/oc.py - -diff --git a/man/en/sos-collect.1 b/man/en/sos-collect.1 -index 8ad4fe5e..a1f6c10e 100644 ---- a/man/en/sos-collect.1 -+++ b/man/en/sos-collect.1 -@@ -364,6 +364,7 @@ The types of transports supported are currently as follows: - - \fBauto\fR Allow the cluster type to determine the transport used - \fBcontrol_persist\fR Use OpenSSH's ControlPersist feature. This is the default behavior -+ \fBoc\fR Use a \fBlocally\fR configured \fBoc\fR binary to deploy collection pods on OCP nodes - - .TP - \fB\-\-tmp\-dir\fR TMP_DIR -diff --git a/sos/collector/clusters/ocp.py b/sos/collector/clusters/ocp.py -index ad97587f..a9357dbf 100644 ---- a/sos/collector/clusters/ocp.py -+++ b/sos/collector/clusters/ocp.py -@@ -12,6 +12,7 @@ import os - - from pipes import quote - from sos.collector.clusters import Cluster -+from sos.utilities import is_executable - - - class ocp(Cluster): -@@ -83,6 +84,19 @@ class ocp(Cluster): - nodes[_node[0]][column] = _node[idx[column]] - return nodes - -+ def set_transport_type(self): -+ if is_executable('oc'): -+ return 'oc' -+ self.log_info("Local installation of 'oc' not found or is not " -+ "correctly configured. Will use ControlPersist") -+ self.ui_log.warn( -+ "Preferred transport 'oc' not available, will fallback to SSH." -+ ) -+ if not self.opts.batch: -+ input("Press ENTER to continue connecting with SSH, or Ctrl+C to" -+ "abort.") -+ return 'control_persist' -+ - def get_nodes(self): - nodes = [] - self.node_dict = {} -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 5c5c7201..8a9dbd7a 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -20,6 +20,7 @@ from sos.policies import load - from sos.policies.init_systems import InitSystem - from sos.collector.transports.control_persist import SSHControlPersist - from sos.collector.transports.local import LocalTransport -+from sos.collector.transports.oc import OCTransport - from sos.collector.exceptions import (CommandTimeoutException, - ConnectionException, - UnsupportedHostException, -@@ -28,6 +29,7 @@ from sos.collector.exceptions import (CommandTimeoutException, - TRANSPORTS = { - 'local': LocalTransport, - 'control_persist': SSHControlPersist, -+ 'oc': OCTransport - } - - -@@ -421,13 +423,11 @@ class SosNode(): - if 'atomic' in cmd: - get_pty = True - -- if get_pty: -- cmd = "/bin/bash -c %s" % quote(cmd) -- - if env: - _cmd_env = self.env_vars - env = _cmd_env.update(env) -- return self._transport.run_command(cmd, timeout, need_root, env) -+ return self._transport.run_command(cmd, timeout, need_root, env, -+ get_pty) - - def sosreport(self): - """Run an sos report on the node, then collect it""" -diff --git a/sos/collector/transports/__init__.py b/sos/collector/transports/__init__.py -index 5be7dc6d..7bffee62 100644 ---- a/sos/collector/transports/__init__.py -+++ b/sos/collector/transports/__init__.py -@@ -144,7 +144,8 @@ class RemoteTransport(): - raise NotImplementedError("Transport %s does not define disconnect" - % self.name) - -- def run_command(self, cmd, timeout=180, need_root=False, env=None): -+ def run_command(self, cmd, timeout=180, need_root=False, env=None, -+ get_pty=False): - """Run a command on the node, returning its output and exit code. - This should return the exit code of the command being executed, not the - exit code of whatever mechanism the transport uses to execute that -@@ -165,10 +166,15 @@ class RemoteTransport(): - :param env: Specify env vars to be passed to the ``cmd`` - :type env: ``dict`` - -+ :param get_pty: Does ``cmd`` require execution with a pty? -+ :type get_pty: ``bool`` -+ - :returns: Output of ``cmd`` and the exit code - :rtype: ``dict`` with keys ``output`` and ``status`` - """ - self.log_debug('Running command %s' % cmd) -+ if get_pty: -+ cmd = "/bin/bash -c %s" % quote(cmd) - # currently we only use/support the use of pexpect for handling the - # execution of these commands, as opposed to directly invoking - # subprocess.Popen() in conjunction with tools like sshpass. -@@ -212,6 +218,13 @@ class RemoteTransport(): - :type env: ``dict`` - """ - cmd = self._format_cmd_for_exec(cmd) -+ -+ # if for any reason env is empty, set it to None as otherwise -+ # pexpect interprets this to mean "run this command with no env vars of -+ # any kind" -+ if not env: -+ env = None -+ - result = pexpect.spawn(cmd, encoding='utf-8', env=env) - - _expects = [pexpect.EOF, pexpect.TIMEOUT] -@@ -268,6 +281,9 @@ class RemoteTransport(): - _out = self.run_command('hostname') - if _out['status'] == 0: - self._hostname = _out['output'].strip() -+ -+ if not self._hostname: -+ self._hostname = self.address - self.log_info("Hostname set to %s" % self._hostname) - return self._hostname - -@@ -302,7 +318,7 @@ class RemoteTransport(): - return self._read_file(fname) - - def _read_file(self, fname): -- res = self.run_command("cat %s" % fname, timeout=5) -+ res = self.run_command("cat %s" % fname, timeout=10) - if res['status'] == 0: - return res['output'] - else: -diff --git a/sos/collector/transports/oc.py b/sos/collector/transports/oc.py -new file mode 100644 -index 00000000..649037b9 ---- /dev/null -+++ b/sos/collector/transports/oc.py -@@ -0,0 +1,220 @@ -+# Copyright Red Hat 2021, Jake Hunsaker -+ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+import json -+import tempfile -+import os -+ -+from sos.collector.transports import RemoteTransport -+from sos.utilities import (is_executable, sos_get_command_output, -+ SoSTimeoutError) -+ -+ -+class OCTransport(RemoteTransport): -+ """This transport leverages the execution of commands via a locally -+ available and configured ``oc`` binary for OCPv4 environments. -+ -+ OCPv4 clusters generally discourage the use of SSH, so this transport may -+ be used to remove our use of SSH in favor of the environment provided -+ method of connecting to nodes and executing commands via debug pods. -+ -+ Note that this approach will generate multiple debug pods over the course -+ of our execution -+ """ -+ -+ name = 'oc' -+ project = 'sos-collect-tmp' -+ -+ def run_oc(self, cmd, **kwargs): -+ """Format and run a command with `oc` in the project defined for our -+ execution -+ """ -+ return sos_get_command_output( -+ "oc -n sos-collect-tmp %s" % cmd, -+ **kwargs -+ ) -+ -+ @property -+ def connected(self): -+ up = self.run_oc( -+ "wait --timeout=0s --for=condition=ready pod/%s" % self.pod_name -+ ) -+ return up['status'] == 0 -+ -+ def get_node_pod_config(self): -+ """Based on our template for the debug container, add the node-specific -+ items so that we can deploy one of these on each node we're collecting -+ from -+ """ -+ return { -+ "kind": "Pod", -+ "apiVersion": "v1", -+ "metadata": { -+ "name": "%s-sos-collector" % self.address.split('.')[0], -+ "namespace": "sos-collect-tmp" -+ }, -+ "priorityClassName": "system-cluster-critical", -+ "spec": { -+ "volumes": [ -+ { -+ "name": "host", -+ "hostPath": { -+ "path": "/", -+ "type": "Directory" -+ } -+ }, -+ { -+ "name": "run", -+ "hostPath": { -+ "path": "/run", -+ "type": "Directory" -+ } -+ }, -+ { -+ "name": "varlog", -+ "hostPath": { -+ "path": "/var/log", -+ "type": "Directory" -+ } -+ }, -+ { -+ "name": "machine-id", -+ "hostPath": { -+ "path": "/etc/machine-id", -+ "type": "File" -+ } -+ } -+ ], -+ "containers": [ -+ { -+ "name": "sos-collector-tmp", -+ "image": "registry.redhat.io/rhel8/support-tools", -+ "command": [ -+ "/bin/bash" -+ ], -+ "env": [ -+ { -+ "name": "HOST", -+ "value": "/host" -+ } -+ ], -+ "resources": {}, -+ "volumeMounts": [ -+ { -+ "name": "host", -+ "mountPath": "/host" -+ }, -+ { -+ "name": "run", -+ "mountPath": "/run" -+ }, -+ { -+ "name": "varlog", -+ "mountPath": "/var/log" -+ }, -+ { -+ "name": "machine-id", -+ "mountPath": "/etc/machine-id" -+ } -+ ], -+ "securityContext": { -+ "privileged": True, -+ "runAsUser": 0 -+ }, -+ "stdin": True, -+ "stdinOnce": True, -+ "tty": True -+ } -+ ], -+ "restartPolicy": "Never", -+ "nodeName": self.address, -+ "hostNetwork": True, -+ "hostPID": True, -+ "hostIPC": True -+ } -+ } -+ -+ def _connect(self, password): -+ # the oc binary must be _locally_ available for this to work -+ if not is_executable('oc'): -+ return False -+ -+ # deploy the debug container we'll exec into -+ podconf = self.get_node_pod_config() -+ self.pod_name = podconf['metadata']['name'] -+ fd, self.pod_tmp_conf = tempfile.mkstemp(dir=self.tmpdir) -+ with open(fd, 'w') as cfile: -+ json.dump(podconf, cfile) -+ self.log_debug("Starting sos collector container '%s'" % self.pod_name) -+ # this specifically does not need to run with a project definition -+ out = sos_get_command_output( -+ "oc create -f %s" % self.pod_tmp_conf -+ ) -+ if (out['status'] != 0 or "pod/%s created" % self.pod_name not in -+ out['output']): -+ self.log_error("Unable to deploy sos collect pod") -+ self.log_debug("Debug pod deployment failed: %s" % out['output']) -+ return False -+ self.log_debug("Pod '%s' successfully deployed, waiting for pod to " -+ "enter ready state" % self.pod_name) -+ -+ # wait for the pod to report as running -+ try: -+ up = self.run_oc("wait --for=condition=Ready pod/%s --timeout=30s" -+ % self.pod_name, -+ # timeout is for local safety, not oc -+ timeout=40) -+ if not up['status'] == 0: -+ self.log_error("Pod not available after 30 seconds") -+ return False -+ except SoSTimeoutError: -+ self.log_error("Timeout while polling for pod readiness") -+ return False -+ except Exception as err: -+ self.log_error("Error while waiting for pod to be ready: %s" -+ % err) -+ return False -+ -+ return True -+ -+ def _format_cmd_for_exec(self, cmd): -+ if cmd.startswith('oc'): -+ return ("oc -n %s exec --request-timeout=0 %s -- chroot /host %s" -+ % (self.project, self.pod_name, cmd)) -+ return super(OCTransport, self)._format_cmd_for_exec(cmd) -+ -+ def run_command(self, cmd, timeout=180, need_root=False, env=None, -+ get_pty=False): -+ # debug pod setup is slow, extend all timeouts to account for this -+ if timeout: -+ timeout += 10 -+ -+ # since we always execute within a bash shell, force disable get_pty -+ # to avoid double-quoting -+ return super(OCTransport, self).run_command(cmd, timeout, need_root, -+ env, False) -+ -+ def _disconnect(self): -+ os.unlink(self.pod_tmp_conf) -+ removed = self.run_oc("delete pod %s" % self.pod_name) -+ if "deleted" not in removed['output']: -+ self.log_debug("Calling delete on pod '%s' failed: %s" -+ % (self.pod_name, removed)) -+ return False -+ return True -+ -+ @property -+ def remote_exec(self): -+ return ("oc -n %s exec --request-timeout=0 %s -- /bin/bash -c" -+ % (self.project, self.pod_name)) -+ -+ def _retrieve_file(self, fname, dest): -+ cmd = self.run_oc("cp %s:%s %s" % (self.pod_name, fname, dest)) -+ return cmd['status'] == 0 --- -2.31.1 - - -From 460494c4296db1a7529b44fe8f6597544c917c02 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Mon, 11 Oct 2021 11:50:44 -0400 -Subject: [PATCH 3/4] [ocp] Create temporary project and restrict default node - list to masters - -Adds explicit setup of a new project to use in the `ocp` cluster and -adds better handling of cluster setup generally, which the `ocp` cluster -is the first to make use of. - -Included in this change is a correction to -`Cluster.exec_primary_cmd()`'s use of `get_pty` to now be determined on -if the primary node is the local node or not. - -Additionally, based on feedback from the OCP engineering team, by -default restrict node lists to masters. - -Signed-off-by: Jake Hunsaker ---- - sos/collector/__init__.py | 5 ++++ - sos/collector/clusters/__init__.py | 13 +++++++- - sos/collector/clusters/ocp.py | 48 ++++++++++++++++++++++++++++-- - sos/collector/transports/oc.py | 4 +-- - 4 files changed, 64 insertions(+), 6 deletions(-) - -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index fecfe6aa..a76f8a79 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -850,6 +850,7 @@ class SoSCollector(SoSComponent): - "CTRL-C to quit\n") - self.ui_log.info("") - except KeyboardInterrupt: -+ self.cluster.cleanup() - self.exit("Exiting on user cancel", 130) - - def configure_sos_cmd(self): -@@ -1098,6 +1099,7 @@ this utility or remote systems that it connects to. - self.archive.makedirs('sos_logs', 0o755) - - self.collect() -+ self.cluster.cleanup() - self.cleanup() - - def collect(self): -@@ -1156,9 +1158,11 @@ this utility or remote systems that it connects to. - pool.shutdown(wait=True) - except KeyboardInterrupt: - self.log_error('Exiting on user cancel\n') -+ self.cluster.cleanup() - os._exit(130) - except Exception as err: - self.log_error('Could not connect to nodes: %s' % err) -+ self.cluster.cleanup() - os._exit(1) - - if hasattr(self.cluster, 'run_extra_cmd'): -@@ -1199,6 +1199,7 @@ this utility or remote systems that it c - arc_name = self.create_cluster_archive() - else: - msg = 'No sosreports were collected, nothing to archive...' -+ self.cluster.cleanup() - self.exit(msg, 1) - - if self.opts.upload and self.policy.get_upload_url(): -diff --git a/sos/collector/clusters/__init__.py b/sos/collector/clusters/__init__.py -index cf1e7a0b..2a4665a1 100644 ---- a/sos/collector/clusters/__init__.py -+++ b/sos/collector/clusters/__init__.py -@@ -192,7 +192,8 @@ class Cluster(): - :returns: The output and status of `cmd` - :rtype: ``dict`` - """ -- res = self.master.run_command(cmd, get_pty=True, need_root=need_root) -+ pty = self.master.local is False -+ res = self.master.run_command(cmd, get_pty=pty, need_root=need_root) - if res['output']: - res['output'] = res['output'].replace('Password:', '') - return res -@@ -223,6 +224,16 @@ class Cluster(): - return True - return False - -+ def cleanup(self): -+ """ -+ This may be overridden by clusters -+ -+ Perform any necessary cleanup steps required by the cluster profile. -+ This helps ensure that sos does make lasting changes to the environment -+ in which we are running -+ """ -+ pass -+ - def get_nodes(self): - """ - This MUST be overridden by a cluster profile subclassing this class -diff --git a/sos/collector/clusters/ocp.py b/sos/collector/clusters/ocp.py -index a9357dbf..92da4e6e 100644 ---- a/sos/collector/clusters/ocp.py -+++ b/sos/collector/clusters/ocp.py -@@ -23,10 +23,12 @@ class ocp(Cluster): - - api_collect_enabled = False - token = None -+ project = 'sos-collect-tmp' -+ oc_cluster_admin = None - - option_list = [ - ('label', '', 'Colon delimited list of labels to select nodes with'), -- ('role', '', 'Colon delimited list of roles to select nodes with'), -+ ('role', 'master', 'Colon delimited list of roles to filter on'), - ('kubeconfig', '', 'Path to the kubeconfig file'), - ('token', '', 'Service account token to use for oc authorization') - ] -@@ -58,6 +58,42 @@ class ocp(Cluster): - _who = self.fmt_oc_cmd('whoami') - return self.exec_master_cmd(_who)['status'] == 0 - -+ def setup(self): -+ """Create the project that we will be executing in for any nodes' -+ collection via a container image -+ """ -+ if not self.set_transport_type() == 'oc': -+ return -+ -+ out = self.exec_master_cmd(self.fmt_oc_cmd("auth can-i '*' '*'")) -+ self.oc_cluster_admin = out['status'] == 0 -+ if not self.oc_cluster_admin: -+ self.log_debug("Check for cluster-admin privileges returned false," -+ " cannot create project in OCP cluster") -+ raise Exception("Insufficient permissions to create temporary " -+ "collection project.\nAborting...") -+ -+ self.log_info("Creating new temporary project '%s'" % self.project) -+ ret = self.exec_master_cmd("oc new-project %s" % self.project) -+ if ret['status'] == 0: -+ return True -+ -+ self.log_debug("Failed to create project: %s" % ret['output']) -+ raise Exception("Failed to create temporary project for collection. " -+ "\nAborting...") -+ -+ def cleanup(self): -+ """Remove the project we created to execute within -+ """ -+ if self.project: -+ ret = self.exec_master_cmd("oc delete project %s" % self.project) -+ if not ret['status'] == 0: -+ self.log_error("Error deleting temporary project: %s" -+ % ret['output']) -+ # don't leave the config on a non-existing project -+ self.exec_master_cmd("oc project default") -+ return True -+ - def _build_dict(self, nodelist): - """From the output of get_nodes(), construct an easier-to-reference - dict of nodes that will be used in determining labels, master status, -@@ -85,10 +123,10 @@ class ocp(Cluster): - return nodes - - def set_transport_type(self): -- if is_executable('oc'): -+ if is_executable('oc') or self.opts.transport == 'oc': - return 'oc' - self.log_info("Local installation of 'oc' not found or is not " -- "correctly configured. Will use ControlPersist") -+ "correctly configured. Will use ControlPersist.") - self.ui_log.warn( - "Preferred transport 'oc' not available, will fallback to SSH." - ) -@@ -106,6 +144,10 @@ class ocp(Cluster): - cmd += " -l %s" % quote(labels) - res = self.exec_master_cmd(self.fmt_oc_cmd(cmd)) - if res['status'] == 0: -+ if self.get_option('role') == 'master': -+ self.log_warn("NOTE: By default, only master nodes are listed." -+ "\nTo collect from all/more nodes, override the " -+ "role option with '-c ocp.role=role1:role2'") - roles = [r for r in self.get_option('role').split(':')] - self.node_dict = self._build_dict(res['output'].splitlines()) - for node in self.node_dict: -diff --git a/sos/collector/transports/oc.py b/sos/collector/transports/oc.py -index 649037b9..de044ccb 100644 ---- a/sos/collector/transports/oc.py -+++ b/sos/collector/transports/oc.py -@@ -37,7 +37,7 @@ class OCTransport(RemoteTransport): - execution - """ - return sos_get_command_output( -- "oc -n sos-collect-tmp %s" % cmd, -+ "oc -n %s %s" % (self.project, cmd), - **kwargs - ) - -@@ -58,7 +58,7 @@ class OCTransport(RemoteTransport): - "apiVersion": "v1", - "metadata": { - "name": "%s-sos-collector" % self.address.split('.')[0], -- "namespace": "sos-collect-tmp" -+ "namespace": self.project - }, - "priorityClassName": "system-cluster-critical", - "spec": { --- -2.31.1 - - -From 1bc0e9fe32491e764e622368bfe216f97bf32620 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Mon, 4 Oct 2021 15:12:04 -0400 -Subject: [PATCH 4/4] [sosnode] Fix typo and small logic break - -Fixes a typo in setting the non-primary node options from the ocp -profile against the sosnode object. Second, fixes a small break in -checksum handling for the manifest discovered during `oc` transport -testing for edge cases. - -Signed-off-by: Jake Hunsaker ---- - sos/collector/clusters/ocp.py | 4 ++-- - sos/collector/sosnode.py | 4 +++- - 2 files changed, 5 insertions(+), 3 deletions(-) - -diff --git a/sos/collector/clusters/ocp.py b/sos/collector/clusters/ocp.py -index 92da4e6e..22a7289a 100644 ---- a/sos/collector/clusters/ocp.py -+++ b/sos/collector/clusters/ocp.py -@@ -183,7 +183,7 @@ class ocp(Cluster): - if self.api_collect_enabled: - # a primary has already been enabled for API collection, disable - # it among others -- node.plugin_options.append('openshift.no-oc=on') -+ node.plugopts.append('openshift.no-oc=on') - else: - _oc_cmd = 'oc' - if node.host.containerized: -@@ -223,6 +223,6 @@ class ocp(Cluster): - - def set_node_options(self, node): - # don't attempt OC API collections on non-primary nodes -- node.plugin_options.append('openshift.no-oc=on') -+ node.plugopts.append('openshift.no-oc=on') - - # vim: set et ts=4 sw=4 : -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 8a9dbd7a..ab7f23cc 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -714,7 +714,7 @@ class SosNode(): - elif line.startswith("The checksum is: "): - checksum = line.split()[3] - -- if checksum is not None: -+ if checksum: - self.manifest.add_field('checksum', checksum) - if len(checksum) == 32: - self.manifest.add_field('checksum_type', 'md5') -@@ -722,6 +722,8 @@ class SosNode(): - self.manifest.add_field('checksum_type', 'sha256') - else: - self.manifest.add_field('checksum_type', 'unknown') -+ else: -+ self.manifest.add_field('checksum_type', 'unknown') - else: - err = self.determine_sos_error(res['status'], res['output']) - self.log_debug("Error running sos report. rc = %s msg = %s" --- -2.31.1 - -From 38a0533de3dd2613eefcc4865a2916e225e3ceed Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 9 Nov 2021 19:34:25 +0100 -Subject: [PATCH] [presets] Optimise OCP preset for hundreds of network - namespaces - -Sos report on OCP having hundreds of namespaces timeouts in networking -plugin, as it collects >10 commands for each namespace. - -Let use a balanced approach in: -- increasing network.timeout -- limiting namespaces to traverse -- disabling ethtool per namespace - -to ensure sos report successfully finish in a reasonable time, -collecting rasonable amount of data. - -Resolves: #2754 - -Signed-off-by: Pavel Moravec ---- - sos/presets/redhat/__init__.py | 10 +++++++--- - 1 file changed, 7 insertions(+), 3 deletions(-) - -diff --git a/sos/presets/redhat/__init__.py b/sos/presets/redhat/__init__.py -index e6d63611..865c9b6b 100644 ---- a/sos/presets/redhat/__init__.py -+++ b/sos/presets/redhat/__init__.py -@@ -29,11 +29,15 @@ RHEL_DESC = RHEL_RELEASE_STR - - RHOSP = "rhosp" - RHOSP_DESC = "Red Hat OpenStack Platform" -+RHOSP_OPTS = SoSOptions(plugopts=[ -+ 'process.lsof=off', -+ 'networking.ethtool_namespaces=False', -+ 'networking.namespaces=200']) - - RHOCP = "ocp" - RHOCP_DESC = "OpenShift Container Platform by Red Hat" --RHOSP_OPTS = SoSOptions(plugopts=[ -- 'process.lsof=off', -+RHOCP_OPTS = SoSOptions(all_logs=True, verify=True, plugopts=[ -+ 'networking.timeout=600', - 'networking.ethtool_namespaces=False', - 'networking.namespaces=200']) - -@@ -62,7 +66,7 @@ RHEL_PRESETS = { - RHEL: PresetDefaults(name=RHEL, desc=RHEL_DESC), - RHOSP: PresetDefaults(name=RHOSP, desc=RHOSP_DESC, opts=RHOSP_OPTS), - RHOCP: PresetDefaults(name=RHOCP, desc=RHOCP_DESC, note=NOTE_SIZE_TIME, -- opts=_opts_all_logs_verify), -+ opts=RHOCP_OPTS), - RH_CFME: PresetDefaults(name=RH_CFME, desc=RH_CFME_DESC, note=NOTE_TIME, - opts=_opts_verify), - RH_SATELLITE: PresetDefaults(name=RH_SATELLITE, desc=RH_SATELLITE_DESC, --- -2.31.1 - -From 97b93c7f8755d04bdeb4f93759c20dcb787f2046 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 2 Nov 2021 11:34:13 -0400 -Subject: [PATCH] [Plugin] Rework get_container_logs to be more useful - -`get_container_logs()` is now `add_container_logs()` to align it better -with our more common `add_*` methods for plugin collections. - -Additionally, it has been extended to accept either a single string or a -list of strings like the other methods, and plugin authors may now -specify either specific container names or regexes. - -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/__init__.py | 22 +++++++++++++++++----- - sos/report/plugins/rabbitmq.py | 2 +- - 2 files changed, 18 insertions(+), 6 deletions(-) - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index 08eee118..4b0e4fd5 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -2366,20 +2366,32 @@ class Plugin(): - return _runtime.volumes - return [] - -- def get_container_logs(self, container, **kwargs): -- """Helper to get the ``logs`` output for a given container -+ def add_container_logs(self, containers, get_all=False, **kwargs): -+ """Helper to get the ``logs`` output for a given container or list -+ of container names and/or regexes. - - Supports passthru of add_cmd_output() options - -- :param container: The name of the container to retrieve logs from -- :type container: ``str`` -+ :param containers: The name of the container to retrieve logs from, -+ may be a single name or a regex -+ :type containers: ``str`` or ``list` of strs -+ -+ :param get_all: Should non-running containers also be queried? -+ Default: False -+ :type get_all: ``bool`` - - :param kwargs: Any kwargs supported by ``add_cmd_output()`` are - supported here - """ - _runtime = self._get_container_runtime() - if _runtime is not None: -- self.add_cmd_output(_runtime.get_logs_command(container), **kwargs) -+ if isinstance(containers, str): -+ containers = [containers] -+ for container in containers: -+ _cons = self.get_all_containers_by_regex(container, get_all) -+ for _con in _cons: -+ cmd = _runtime.get_logs_command(_con[1]) -+ self.add_cmd_output(cmd, **kwargs) - - def fmt_container_cmd(self, container, cmd, quotecmd=False): - """Format a command to be executed by the loaded ``ContainerRuntime`` -diff --git a/sos/report/plugins/rabbitmq.py b/sos/report/plugins/rabbitmq.py -index e84b52da..1bfa741f 100644 ---- a/sos/report/plugins/rabbitmq.py -+++ b/sos/report/plugins/rabbitmq.py -@@ -32,7 +32,7 @@ class RabbitMQ(Plugin, IndependentPlugin): - - if in_container: - for container in container_names: -- self.get_container_logs(container) -+ self.add_container_logs(container) - self.add_cmd_output( - self.fmt_container_cmd(container, 'rabbitmqctl report'), - foreground=True --- -2.31.1 - -From 8bf602108f75db10e449eff5e2266c6466504086 Mon Sep 17 00:00:00 2001 -From: Nadia Pinaeva -Date: Thu, 2 Dec 2021 16:30:44 +0100 -Subject: [PATCH] [clusters:ocp] fix get_nodes function - -Signed-off-by: Nadia Pinaeva ---- - sos/collector/clusters/ocp.py | 8 ++++---- - 1 file changed, 4 insertions(+), 4 deletions(-) - -diff --git a/sos/collector/clusters/ocp.py b/sos/collector/clusters/ocp.py -index 22a7289a..2ce4e977 100644 ---- a/sos/collector/clusters/ocp.py -+++ b/sos/collector/clusters/ocp.py -@@ -150,13 +150,13 @@ class ocp(Cluster): - "role option with '-c ocp.role=role1:role2'") - roles = [r for r in self.get_option('role').split(':')] - self.node_dict = self._build_dict(res['output'].splitlines()) -- for node in self.node_dict: -+ for node_name, node in self.node_dict.items(): - if roles: - for role in roles: -- if role in node: -- nodes.append(node) -+ if role == node['roles']: -+ nodes.append(node_name) - else: -- nodes.append(node) -+ nodes.append(node_name) - else: - msg = "'oc' command failed" - if 'Missing or incomplete' in res['output']: --- -2.31.1 - -From 5d80ac6dc67e12ef00903436c088a1694f9a7dd7 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 1 Dec 2021 14:13:16 -0500 -Subject: [PATCH] [collect] Catch command not found exceptions from pexpect - -When running a command that does not exist on the system, catch the -resulting pexpect exception and return the proper status code rather -than allowing an untrapped exception. - -Closes: #2768 - -Signed-off-by: Jake Hunsaker ---- - sos/collector/transports/__init__.py | 6 +++++- - 1 file changed, 5 insertions(+), 1 deletion(-) - -diff --git a/sos/collector/transports/__init__.py b/sos/collector/transports/__init__.py -index 7bffee62..33f2f66d 100644 ---- a/sos/collector/transports/__init__.py -+++ b/sos/collector/transports/__init__.py -@@ -225,7 +225,11 @@ class RemoteTransport(): - if not env: - env = None - -- result = pexpect.spawn(cmd, encoding='utf-8', env=env) -+ try: -+ result = pexpect.spawn(cmd, encoding='utf-8', env=env) -+ except pexpect.exceptions.ExceptionPexpect as err: -+ self.log_debug(err.value) -+ return {'status': 127, 'output': ''} - - _expects = [pexpect.EOF, pexpect.TIMEOUT] - if need_root and self.opts.ssh_user != 'root': --- -2.31.1 - -From decb5d26c165e664fa879a669f2d80165181f0e1 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 2 Dec 2021 14:02:17 -0500 -Subject: [PATCH] [report,collect] Add option to control default container - runtime - -Adds a new `--container-runtime` option that allows users to control -what default container runtime is used by plugins for container based -collections, effectively overriding policy defaults. - -If no runtimes are active, this option is effectively ignored. If -however runtimes are active, but the requested one is not, raise an -exception to abort collection with an appropriate message to the user. - -Signed-off-by: Jake Hunsaker ---- - man/en/sos-collect.1 | 6 ++++++ - man/en/sos-report.1 | 19 +++++++++++++++++++ - sos/collector/__init__.py | 4 ++++ - sos/collector/sosnode.py | 6 ++++++ - sos/report/__init__.py | 36 ++++++++++++++++++++++++++++++++++++ - 5 files changed, 71 insertions(+) - -diff --git a/man/en/sos-collect.1 b/man/en/sos-collect.1 -index a1f6c10e..9b0a5d7b 100644 ---- a/man/en/sos-collect.1 -+++ b/man/en/sos-collect.1 -@@ -11,6 +11,7 @@ sos collect \- Collect sosreports from multiple (cluster) nodes - [\-\-chroot CHROOT] - [\-\-case\-id CASE_ID] - [\-\-cluster\-type CLUSTER_TYPE] -+ [\-\-container\-runtime RUNTIME] - [\-e ENABLE_PLUGINS] - [--encrypt-key KEY]\fR - [--encrypt-pass PASS]\fR -@@ -113,6 +114,11 @@ Example: \fBsos collect --cluster-type=kubernetes\fR will force the kubernetes p - to be run, and thus set sosreport options and attempt to determine a list of nodes using - that profile. - .TP -+\fB\-\-container\-runtime\fR RUNTIME -+\fB sos report\fR option. Using this with \fBcollect\fR will pass this option thru -+to nodes with sos version 4.3 or later. This option controls the default container -+runtime plugins will use for collections. See \fBman sos-report\fR. -+.TP - \fB\-e\fR ENABLE_PLUGINS, \fB\-\-enable\-plugins\fR ENABLE_PLUGINS - Sosreport option. Use this to enable a plugin that would otherwise not be run. - -diff --git a/man/en/sos-report.1 b/man/en/sos-report.1 -index e8efc8f8..464a77e5 100644 ---- a/man/en/sos-report.1 -+++ b/man/en/sos-report.1 -@@ -19,6 +19,7 @@ sos report \- Collect and package diagnostic and support data - [--plugin-timeout TIMEOUT]\fR - [--cmd-timeout TIMEOUT]\fR - [--namespaces NAMESPACES]\fR -+ [--container-runtime RUNTIME]\fR - [-s|--sysroot SYSROOT]\fR - [-c|--chroot {auto|always|never}\fR - [--tmp-dir directory]\fR -@@ -299,6 +300,24 @@ Use '0' (default) for no limit - all namespaces will be used for collections. - - Note that specific plugins may provide a similar `namespaces` plugin option. If - the plugin option is used, it will override this option. -+.TP -+.B \--container-runtime RUNTIME -+Force the use of the specified RUNTIME as the default runtime that plugins will -+use to collect data from and about containers and container images. By default, -+the setting of \fBauto\fR results in the local policy determining what runtime -+will be the default runtime (in configurations where multiple runtimes are installed -+and active). -+ -+If no container runtimes are active, this option is ignored. If there are runtimes -+active, but not one with a name matching RUNTIME, sos will abort. -+ -+Setting this to \fBnone\fR, \fBoff\fR, or \fBdisabled\fR will cause plugins to -+\fBNOT\fR leverage any active runtimes for collections. Note that if disabled, plugins -+specifically for runtimes (e.g. the podman or docker plugins) will still collect -+general data about the runtime, but will not inspect existing containers or images. -+ -+Default: 'auto' (policy determined) -+.TP - .B \--case-id NUMBER - Specify a case identifier to associate with the archive. - Identifiers may include alphanumeric characters, commas and periods ('.'). -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index 42a7731d..3ad703d3 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -55,6 +55,7 @@ class SoSCollector(SoSComponent): - 'clean': False, - 'cluster_options': [], - 'cluster_type': None, -+ 'container_runtime': 'auto', - 'domains': [], - 'enable_plugins': [], - 'encrypt_key': '', -@@ -268,6 +269,9 @@ class SoSCollector(SoSComponent): - sos_grp.add_argument('--chroot', default='', - choices=['auto', 'always', 'never'], - help="chroot executed commands to SYSROOT") -+ sos_grp.add_argument("--container-runtime", default="auto", -+ help="Default container runtime to use for " -+ "collections. 'auto' for policy control.") - sos_grp.add_argument('-e', '--enable-plugins', action="extend", - help='Enable specific plugins for sosreport') - sos_grp.add_argument('-k', '--plugin-option', '--plugopts', -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index ab7f23cc..f5957e17 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -586,6 +586,12 @@ class SosNode(): - sos_opts.append('--cmd-timeout=%s' - % quote(str(self.opts.cmd_timeout))) - -+ if self.check_sos_version('4.3'): -+ if self.opts.container_runtime != 'auto': -+ sos_opts.append( -+ "--container-runtime=%s" % self.opts.container_runtime -+ ) -+ - self.update_cmd_from_cluster() - - sos_cmd = sos_cmd.replace( -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index a6c72778..0daad82f 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -82,6 +82,7 @@ class SoSReport(SoSComponent): - 'case_id': '', - 'chroot': 'auto', - 'clean': False, -+ 'container_runtime': 'auto', - 'keep_binary_files': False, - 'desc': '', - 'domains': [], -@@ -187,6 +188,7 @@ class SoSReport(SoSComponent): - self.tempfile_util.clean() - self._exit(1) - -+ self._check_container_runtime() - self._get_hardware_devices() - self._get_namespaces() - -@@ -218,6 +220,9 @@ class SoSReport(SoSComponent): - dest="chroot", default='auto', - help="chroot executed commands to SYSROOT " - "[auto, always, never] (default=auto)") -+ report_grp.add_argument("--container-runtime", default="auto", -+ help="Default container runtime to use for " -+ "collections. 'auto' for policy control.") - report_grp.add_argument("--desc", "--description", type=str, - action="store", default="", - help="Description for a new preset",) -@@ -373,6 +378,37 @@ class SoSReport(SoSComponent): - } - # TODO: enumerate network devices, preferably with devtype info - -+ def _check_container_runtime(self): -+ """Check the loaded container runtimes, and the policy default runtime -+ (if set), against any requested --container-runtime value. This can be -+ useful for systems that have multiple runtimes, such as RHCOS, but do -+ not have a clearly defined 'default' (or one that is determined based -+ entirely on configuration). -+ """ -+ if self.opts.container_runtime != 'auto': -+ crun = self.opts.container_runtime.lower() -+ if crun in ['none', 'off', 'diabled']: -+ self.policy.runtimes = {} -+ self.soslog.info( -+ "Disabled all container runtimes per user option." -+ ) -+ elif not self.policy.runtimes: -+ msg = ("WARNING: No container runtimes are active, ignoring " -+ "option to set default runtime to '%s'\n" % crun) -+ self.soslog.warn(msg) -+ elif crun not in self.policy.runtimes.keys(): -+ valid = ', '.join(p for p in self.policy.runtimes.keys() -+ if p != 'default') -+ raise Exception("Cannot use container runtime '%s': no such " -+ "runtime detected. Available runtimes: %s" -+ % (crun, valid)) -+ else: -+ self.policy.runtimes['default'] = self.policy.runtimes[crun] -+ self.soslog.info( -+ "Set default container runtime to '%s'" -+ % self.policy.runtimes['default'].name -+ ) -+ - def get_fibre_devs(self): - """Enumerate a list of fibrechannel devices on this system so that - plugins can iterate over them --- -2.31.1 - -From 9d4b5af39d76ac99afa40d004fe9888633218356 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Fri, 3 Dec 2021 13:37:09 -0500 -Subject: [PATCH 1/2] [Plugin] Add container parameter for add_cmd_output() - -Adds a new `container` parameter for `Plugin.add_cmd_output()`, which if -set will format all commands passed to that call for execution in the -specified container. - -`Plugin.fmt_container_cmd()` is called for this purpose, and has been -modified so that if the given container does not exist, an empty string -is returned instead, thus preventing execution on the host. - -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/__init__.py | 16 ++++++++++++++-- - 1 file changed, 14 insertions(+), 2 deletions(-) - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index e180ae17..3ff7c191 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -1707,7 +1707,7 @@ class Plugin(): - chroot=True, runat=None, env=None, binary=False, - sizelimit=None, pred=None, subdir=None, - changes=False, foreground=False, tags=[], -- priority=10, cmd_as_tag=False): -+ priority=10, cmd_as_tag=False, container=None): - """Run a program or a list of programs and collect the output - - Output will be limited to `sizelimit`, collecting the last X amount -@@ -1772,6 +1772,10 @@ class Plugin(): - :param cmd_as_tag: Should the command string be automatically formatted - to a tag? - :type cmd_as_tag: ``bool`` -+ -+ :param container: Run the specified `cmds` inside a container with this -+ ID or name -+ :type container: ``str`` - """ - if isinstance(cmds, str): - cmds = [cmds] -@@ -1782,6 +1786,14 @@ class Plugin(): - if pred is None: - pred = self.get_predicate(cmd=True) - for cmd in cmds: -+ if container: -+ ocmd = cmd -+ cmd = self.fmt_container_cmd(container, cmd) -+ if not cmd: -+ self._log_debug("Skipping command '%s' as the requested " -+ "container '%s' does not exist." -+ % (ocmd, container)) -+ continue - self._add_cmd_output(cmd=cmd, suggest_filename=suggest_filename, - root_symlink=root_symlink, timeout=timeout, - stderr=stderr, chroot=chroot, runat=runat, -@@ -2420,7 +2432,7 @@ class Plugin(): - if self.container_exists(container): - _runtime = self._get_container_runtime() - return _runtime.fmt_container_cmd(container, cmd, quotecmd) -- return cmd -+ return '' - - def is_module_loaded(self, module_name): - """Determine whether specified module is loaded or not --- -2.31.1 - - -From 874d2adfbff9e51dc902669af3c4a5083dbc19b1 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Fri, 3 Dec 2021 14:49:43 -0500 -Subject: [PATCH 2/2] [plugins] Update existing plugins to use a_c_o container - parameter - -Updates plugins currently calling `fmt_container_cmd()` in their -`add_cmd_output()` calls to instead use the new `container` parameter -and rely on the automatic formatting. - -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/opencontrail.py | 3 +-- - sos/report/plugins/openstack_database.py | 20 ++++++-------------- - sos/report/plugins/openstack_designate.py | 6 ++---- - sos/report/plugins/openstack_ironic.py | 3 +-- - sos/report/plugins/ovn_central.py | 7 +++---- - sos/report/plugins/rabbitmq.py | 11 ++++++----- - 9 files changed, 47 insertions(+), 69 deletions(-) - -diff --git a/sos/report/plugins/opencontrail.py b/sos/report/plugins/opencontrail.py -index b368bffe..76c03e21 100644 ---- a/sos/report/plugins/opencontrail.py -+++ b/sos/report/plugins/opencontrail.py -@@ -25,8 +25,7 @@ class OpenContrail(Plugin, IndependentPlugin): - cnames = self.get_containers(get_all=True) - cnames = [c[1] for c in cnames if 'opencontrail' in c[1]] - for cntr in cnames: -- _cmd = self.fmt_container_cmd(cntr, 'contrail-status') -- self.add_cmd_output(_cmd) -+ self.add_cmd_output('contrail-status', container=cntr) - else: - self.add_cmd_output("contrail-status") - -diff --git a/sos/report/plugins/openstack_database.py b/sos/report/plugins/openstack_database.py -index 1e98fabf..e9f84cf8 100644 ---- a/sos/report/plugins/openstack_database.py -+++ b/sos/report/plugins/openstack_database.py -@@ -37,36 +37,28 @@ class OpenStackDatabase(Plugin): - ] - - def setup(self): -- -- in_container = False - # determine if we're running databases on the host or in a container - _db_containers = [ - 'galera-bundle-.*', # overcloud - 'mysql' # undercloud - ] - -+ cname = None - for container in _db_containers: - cname = self.get_container_by_name(container) -- if cname is not None: -- in_container = True -+ if cname: - break - -- if in_container: -- fname = "clustercheck_%s" % cname -- cmd = self.fmt_container_cmd(cname, 'clustercheck') -- self.add_cmd_output(cmd, timeout=15, suggest_filename=fname) -- else: -- self.add_cmd_output('clustercheck', timeout=15) -+ fname = "clustercheck_%s" % cname if cname else None -+ self.add_cmd_output('clustercheck', container=cname, timeout=15, -+ suggest_filename=fname) - - if self.get_option('dump') or self.get_option('dumpall'): - db_dump = self.get_mysql_db_string(container=cname) - db_cmd = "mysqldump --opt %s" % db_dump - -- if in_container: -- db_cmd = self.fmt_container_cmd(cname, db_cmd) -- - self.add_cmd_output(db_cmd, suggest_filename='mysql_dump.sql', -- sizelimit=0) -+ sizelimit=0, container=cname) - - def get_mysql_db_string(self, container=None): - -diff --git a/sos/report/plugins/openstack_designate.py b/sos/report/plugins/openstack_designate.py -index 0ae991b0..a2ea37ab 100644 ---- a/sos/report/plugins/openstack_designate.py -+++ b/sos/report/plugins/openstack_designate.py -@@ -20,12 +20,10 @@ class OpenStackDesignate(Plugin): - - def setup(self): - # collect current pool config -- pools_cmd = self.fmt_container_cmd( -- self.get_container_by_name(".*designate_central"), -- "designate-manage pool generate_file --file /dev/stdout") - - self.add_cmd_output( -- pools_cmd, -+ "designate-manage pool generate_file --file /dev/stdout", -+ container=self.get_container_by_name(".*designate_central"), - suggest_filename="openstack_designate_current_pools.yaml" - ) - -diff --git a/sos/report/plugins/openstack_ironic.py b/sos/report/plugins/openstack_ironic.py -index c36fb6b6..49beb2d9 100644 ---- a/sos/report/plugins/openstack_ironic.py -+++ b/sos/report/plugins/openstack_ironic.py -@@ -80,8 +80,7 @@ class OpenStackIronic(Plugin): - 'ironic_pxe_tftp', 'ironic_neutron_agent', - 'ironic_conductor', 'ironic_api']: - if self.container_exists('.*' + container_name): -- self.add_cmd_output(self.fmt_container_cmd(container_name, -- 'rpm -qa')) -+ self.add_cmd_output('rpm -qa', container=container_name) - - else: - self.conf_list = [ -diff --git a/sos/report/plugins/ovn_central.py b/sos/report/plugins/ovn_central.py -index 914eda60..ddbf288d 100644 ---- a/sos/report/plugins/ovn_central.py -+++ b/sos/report/plugins/ovn_central.py -@@ -123,11 +123,10 @@ class OVNCentral(Plugin): - - # If OVN is containerized, we need to run the above commands inside - # the container. -- cmds = [ -- self.fmt_container_cmd(self._container_name, cmd) for cmd in cmds -- ] - -- self.add_cmd_output(cmds, foreground=True) -+ self.add_cmd_output( -+ cmds, foreground=True, container=self._container_name -+ ) - - self.add_copy_spec("/etc/sysconfig/ovn-northd") - -diff --git a/sos/report/plugins/rabbitmq.py b/sos/report/plugins/rabbitmq.py -index 1bfa741f..607802e4 100644 ---- a/sos/report/plugins/rabbitmq.py -+++ b/sos/report/plugins/rabbitmq.py -@@ -34,14 +34,15 @@ class RabbitMQ(Plugin, IndependentPlugin): - for container in container_names: - self.add_container_logs(container) - self.add_cmd_output( -- self.fmt_container_cmd(container, 'rabbitmqctl report'), -+ 'rabbitmqctl report', -+ container=container, - foreground=True - ) - self.add_cmd_output( -- self.fmt_container_cmd( -- container, "rabbitmqctl eval " -- "'rabbit_diagnostics:maybe_stuck().'"), -- foreground=True, timeout=10 -+ "rabbitmqctl eval 'rabbit_diagnostics:maybe_stuck().'", -+ container=container, -+ foreground=True, -+ timeout=10 - ) - else: - self.add_cmd_output("rabbitmqctl report") --- -2.31.1 - -From faa15754f82e9841cd624afe18dc2198644decdf Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 8 Dec 2021 13:51:20 -0500 -Subject: [PATCH] [Policy,collect] Prevent remote node policies from setting - local PATH - -This commit fixes an issue where policies loaded for remote nodes when -using `sos collect` would override the PATH setting for the local -policy, which in turn could prevent successful execution of cluster -profile operations. - -Related: #2777 - -Signed-off-by: Jake Hunsaker ---- - sos/policies/__init__.py | 8 +++++--- - sos/policies/distros/__init__.py | 6 ++++-- - sos/policies/distros/debian.py | 3 ++- - sos/policies/distros/redhat.py | 6 ++++-- - sos/policies/distros/suse.py | 3 ++- - 5 files changed, 17 insertions(+), 9 deletions(-) - -diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py -index ef9188de..826d03a1 100644 ---- a/sos/policies/__init__.py -+++ b/sos/policies/__init__.py -@@ -45,7 +45,7 @@ def load(cache={}, sysroot=None, init=None, probe_runtime=True, - return cache['policy'] - - --class Policy(object): -+class Policy(): - """Policies represent distributions that sos supports, and define the way - in which sos behaves on those distributions. A policy should define at - minimum a way to identify the distribution, and a package manager to allow -@@ -111,7 +111,7 @@ any third party. - presets_path = PRESETS_PATH - _in_container = False - -- def __init__(self, sysroot=None, probe_runtime=True): -+ def __init__(self, sysroot=None, probe_runtime=True, remote_exec=None): - """Subclasses that choose to override this initializer should call - super() to ensure that they get the required platform bits attached. - super(SubClass, self).__init__(). Policies that require runtime -@@ -122,7 +122,9 @@ any third party. - self.probe_runtime = probe_runtime - self.package_manager = PackageManager() - self.valid_subclasses = [IndependentPlugin] -- self.set_exec_path() -+ self.remote_exec = remote_exec -+ if not self.remote_exec: -+ self.set_exec_path() - self.sysroot = sysroot - self.register_presets(GENERIC_PRESETS) - -diff --git a/sos/policies/distros/__init__.py b/sos/policies/distros/__init__.py -index c69fc1e7..9c91a918 100644 ---- a/sos/policies/distros/__init__.py -+++ b/sos/policies/distros/__init__.py -@@ -68,9 +68,11 @@ class LinuxPolicy(Policy): - container_version_command = None - container_authfile = None - -- def __init__(self, sysroot=None, init=None, probe_runtime=True): -+ def __init__(self, sysroot=None, init=None, probe_runtime=True, -+ remote_exec=None): - super(LinuxPolicy, self).__init__(sysroot=sysroot, -- probe_runtime=probe_runtime) -+ probe_runtime=probe_runtime, -+ remote_exec=remote_exec) - - if sysroot: - self.sysroot = sysroot -diff --git a/sos/policies/distros/debian.py b/sos/policies/distros/debian.py -index 639fd5eb..41f09428 100644 ---- a/sos/policies/distros/debian.py -+++ b/sos/policies/distros/debian.py -@@ -26,7 +26,8 @@ class DebianPolicy(LinuxPolicy): - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(DebianPolicy, self).__init__(sysroot=sysroot, init=init, -- probe_runtime=probe_runtime) -+ probe_runtime=probe_runtime, -+ remote_exec=remote_exec) - self.package_manager = DpkgPackageManager(chroot=self.sysroot, - remote_exec=remote_exec) - self.valid_subclasses += [DebianPlugin] -diff --git a/sos/policies/distros/redhat.py b/sos/policies/distros/redhat.py -index 4b14abaf..eb75e15b 100644 ---- a/sos/policies/distros/redhat.py -+++ b/sos/policies/distros/redhat.py -@@ -53,7 +53,8 @@ class RedHatPolicy(LinuxPolicy): - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(RedHatPolicy, self).__init__(sysroot=sysroot, init=init, -- probe_runtime=probe_runtime) -+ probe_runtime=probe_runtime, -+ remote_exec=remote_exec) - self.usrmove = False - - self.package_manager = RpmPackageManager(chroot=self.sysroot, -@@ -76,7 +77,8 @@ class RedHatPolicy(LinuxPolicy): - self.PATH = "/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" - self.PATH += os.pathsep + "/usr/local/bin" - self.PATH += os.pathsep + "/usr/local/sbin" -- self.set_exec_path() -+ if not self.remote_exec: -+ self.set_exec_path() - self.load_presets() - - @classmethod -diff --git a/sos/policies/distros/suse.py b/sos/policies/distros/suse.py -index 1c1feff5..b9d4a3b1 100644 ---- a/sos/policies/distros/suse.py -+++ b/sos/policies/distros/suse.py -@@ -25,7 +25,8 @@ class SuSEPolicy(LinuxPolicy): - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(SuSEPolicy, self).__init__(sysroot=sysroot, init=init, -- probe_runtime=probe_runtime) -+ probe_runtime=probe_runtime, -+ remote_exec=remote_exec) - self.valid_subclasses += [SuSEPlugin, RedHatPlugin] - - self.usrmove = False --- -2.31.1 - -From d4383fec5f8a80121aa4f5a37575b37988c51663 Mon Sep 17 00:00:00 2001 -From: Nadia Pinaeva -Date: Wed, 1 Dec 2021 12:23:34 +0100 -Subject: [PATCH] Add crio runtime and openshift_ovn plugin openshift_ovn - plugin collects logs from crio containers Fix get_container_by_name function - returning container_id and not name - -Signed-off-by: Nadia Pinaeva ---- - sos/policies/distros/__init__.py | 4 +- - sos/policies/runtimes/__init__.py | 2 +- - sos/policies/runtimes/crio.py | 79 +++++++++++++++++++++++++++++ - sos/report/plugins/openshift_ovn.py | 41 +++++++++++++++ - 4 files changed, 124 insertions(+), 2 deletions(-) - create mode 100644 sos/policies/runtimes/crio.py - create mode 100644 sos/report/plugins/openshift_ovn.py - -diff --git a/sos/policies/distros/__init__.py b/sos/policies/distros/__init__.py -index 9c91a918..7acc7e49 100644 ---- a/sos/policies/distros/__init__.py -+++ b/sos/policies/distros/__init__.py -@@ -17,6 +17,7 @@ from sos import _sos as _ - from sos.policies import Policy - from sos.policies.init_systems import InitSystem - from sos.policies.init_systems.systemd import SystemdInit -+from sos.policies.runtimes.crio import CrioContainerRuntime - from sos.policies.runtimes.podman import PodmanContainerRuntime - from sos.policies.runtimes.docker import DockerContainerRuntime - -@@ -92,7 +93,8 @@ class LinuxPolicy(Policy): - if self.probe_runtime: - _crun = [ - PodmanContainerRuntime(policy=self), -- DockerContainerRuntime(policy=self) -+ DockerContainerRuntime(policy=self), -+ CrioContainerRuntime(policy=self) - ] - for runtime in _crun: - if runtime.check_is_active(): -diff --git a/sos/policies/runtimes/__init__.py b/sos/policies/runtimes/__init__.py -index 2e60ad23..4e9a45c1 100644 ---- a/sos/policies/runtimes/__init__.py -+++ b/sos/policies/runtimes/__init__.py -@@ -100,7 +100,7 @@ class ContainerRuntime(): - return None - for c in self.containers: - if re.match(name, c[1]): -- return c[1] -+ return c[0] - return None - - def get_images(self): -diff --git a/sos/policies/runtimes/crio.py b/sos/policies/runtimes/crio.py -new file mode 100644 -index 00000000..980c3ea1 ---- /dev/null -+++ b/sos/policies/runtimes/crio.py -@@ -0,0 +1,79 @@ -+# Copyright (C) 2021 Red Hat, Inc., Nadia Pinaeva -+ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from sos.policies.runtimes import ContainerRuntime -+from sos.utilities import sos_get_command_output -+from pipes import quote -+ -+ -+class CrioContainerRuntime(ContainerRuntime): -+ """Runtime class to use for systems running crio""" -+ -+ name = 'crio' -+ binary = 'crictl' -+ -+ def get_containers(self, get_all=False): -+ """Get a list of containers present on the system. -+ -+ :param get_all: If set, include stopped containers as well -+ :type get_all: ``bool`` -+ """ -+ containers = [] -+ _cmd = "%s ps %s" % (self.binary, '-a' if get_all else '') -+ if self.active: -+ out = sos_get_command_output(_cmd, chroot=self.policy.sysroot) -+ if out['status'] == 0: -+ for ent in out['output'].splitlines()[1:]: -+ ent = ent.split() -+ # takes the form (container_id, container_name) -+ containers.append((ent[0], ent[-3])) -+ return containers -+ -+ def get_images(self): -+ """Get a list of images present on the system -+ -+ :returns: A list of 2-tuples containing (image_name, image_id) -+ :rtype: ``list`` -+ """ -+ images = [] -+ if self.active: -+ out = sos_get_command_output("%s images" % self.binary, -+ chroot=self.policy.sysroot) -+ if out['status'] == 0: -+ for ent in out['output'].splitlines(): -+ ent = ent.split() -+ # takes the form (image_name, image_id) -+ images.append((ent[0] + ':' + ent[1], ent[2])) -+ return images -+ -+ def fmt_container_cmd(self, container, cmd, quotecmd): -+ """Format a command to run inside a container using the runtime -+ -+ :param container: The name or ID of the container in which to run -+ :type container: ``str`` -+ -+ :param cmd: The command to run inside `container` -+ :type cmd: ``str`` -+ -+ :param quotecmd: Whether the cmd should be quoted. -+ :type quotecmd: ``bool`` -+ -+ :returns: Formatted string to run `cmd` inside `container` -+ :rtype: ``str`` -+ """ -+ if quotecmd: -+ quoted_cmd = quote(cmd) -+ else: -+ quoted_cmd = cmd -+ container_id = self.get_container_by_name(container) -+ return "%s %s %s" % (self.run_cmd, container_id, -+ quoted_cmd) if container_id is not None else '' -+ -+# vim: set et ts=4 sw=4 : -diff --git a/sos/report/plugins/openshift_ovn.py b/sos/report/plugins/openshift_ovn.py -new file mode 100644 -index 00000000..168f1dd3 ---- /dev/null -+++ b/sos/report/plugins/openshift_ovn.py -@@ -0,0 +1,41 @@ -+# Copyright (C) 2021 Nadia Pinaeva -+ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from sos.report.plugins import Plugin, RedHatPlugin -+ -+ -+class OpenshiftOVN(Plugin, RedHatPlugin): -+ """This plugin is used to collect OCP 4.x OVN logs. -+ """ -+ short_desc = 'Openshift OVN' -+ plugin_name = "openshift_ovn" -+ containers = ('ovnkube-master', 'ovn-ipsec') -+ profiles = ('openshift',) -+ -+ def setup(self): -+ self.add_copy_spec([ -+ "/var/lib/ovn/etc/ovnnb_db.db", -+ "/var/lib/ovn/etc/ovnsb_db.db", -+ "/var/lib/openvswitch/etc/keys", -+ "/var/log/openvswitch/libreswan.log", -+ "/var/log/openvswitch/ovs-monitor-ipsec.log" -+ ]) -+ -+ self.add_cmd_output([ -+ 'ovn-appctl -t /var/run/ovn/ovnnb_db.ctl ' + -+ 'cluster/status OVN_Northbound', -+ 'ovn-appctl -t /var/run/ovn/ovnsb_db.ctl ' + -+ 'cluster/status OVN_Southbound'], -+ container='ovnkube-master') -+ self.add_cmd_output([ -+ 'ovs-appctl -t ovs-monitor-ipsec tunnels/show', -+ 'ipsec status', -+ 'certutil -L -d sql:/etc/ipsec.d'], -+ container='ovn-ipsec') --- -2.31.1 - -From 17218ca17e49cb8491c688095b56503d041c1ae9 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 9 Dec 2021 15:07:23 -0500 -Subject: [PATCH 1/3] [ocp] Skip project setup whenever oc transport is not - used - -Fixes a corner case where we would still attempt to create a new project -within the OCP cluster even if we weren't using the `oc` transport. - -Signed-off-by: Jake Hunsaker ---- - sos/collector/clusters/ocp.py | 4 +++- - 1 file changed, 3 insertions(+), 1 deletion(-) - -diff --git a/sos/collector/clusters/ocp.py b/sos/collector/clusters/ocp.py -index 2ce4e977..56f8cc47 100644 ---- a/sos/collector/clusters/ocp.py -+++ b/sos/collector/clusters/ocp.py -@@ -123,7 +123,9 @@ class ocp(Cluster): - return nodes - - def set_transport_type(self): -- if is_executable('oc') or self.opts.transport == 'oc': -+ if self.opts.transport != 'auto': -+ return self.opts.transport -+ if is_executable('oc'): - return 'oc' - self.log_info("Local installation of 'oc' not found or is not " - "correctly configured. Will use ControlPersist.") --- -2.31.1 - - -From 9faabdc3df08516a91c1adb3326bbf43db155f71 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 9 Dec 2021 16:04:39 -0500 -Subject: [PATCH 2/3] [crio] Put inspect output in the containers subdir - -Given the environments where crio is run, having `crictl inspect` output -in the main plugin directory can be a bit overwhelming. As such, put -this output into a `containers` subdir, and nest container log output in -a `containers/logs/` subdir. - -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/crio.py | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) - -diff --git a/sos/report/plugins/crio.py b/sos/report/plugins/crio.py -index cb2c9796..56cf64a7 100644 ---- a/sos/report/plugins/crio.py -+++ b/sos/report/plugins/crio.py -@@ -79,10 +79,11 @@ class CRIO(Plugin, RedHatPlugin, UbuntuPlugin): - pods = self._get_crio_list(pod_cmd) - - for container in containers: -- self.add_cmd_output("crictl inspect %s" % container) -+ self.add_cmd_output("crictl inspect %s" % container, -+ subdir="containers") - if self.get_option('logs'): - self.add_cmd_output("crictl logs -t %s" % container, -- subdir="containers", priority=100) -+ subdir="containers/logs", priority=100) - - for image in images: - self.add_cmd_output("crictl inspecti %s" % image, subdir="images") --- -2.31.1 - - -From 9118562c47fb521da3eeeed1a8746d45aaa769e7 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 9 Dec 2021 16:06:06 -0500 -Subject: [PATCH 3/3] [networking] Put namespaced commands into subdirs - -Where networking namespaces are used, there tend to be large numbers of -namespaces used. This in turn results in sos running and collecting very -large numbers of namespaced commands. - -To aid in consumability, place these collections under a subdir for the -namespace under another "namespaces" subdir within the plugin directory. - -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/networking.py | 27 ++++++++++++--------------- - 1 file changed, 12 insertions(+), 15 deletions(-) - -diff --git a/sos/report/plugins/networking.py b/sos/report/plugins/networking.py -index 80e24abb..bcb5e6ae 100644 ---- a/sos/report/plugins/networking.py -+++ b/sos/report/plugins/networking.py -@@ -198,6 +198,7 @@ class Networking(Plugin): - pred=SoSPredicate(self, cmd_outputs=co6)) - else None) - for namespace in namespaces: -+ _subdir = "namespaces/%s" % namespace - ns_cmd_prefix = cmd_prefix + namespace + " " - self.add_cmd_output([ - ns_cmd_prefix + "ip address show", -@@ -213,29 +214,27 @@ class Networking(Plugin): - ns_cmd_prefix + "netstat -s", - ns_cmd_prefix + "netstat %s -agn" % self.ns_wide, - ns_cmd_prefix + "nstat -zas", -- ], priority=50) -+ ], priority=50, subdir=_subdir) - self.add_cmd_output([ns_cmd_prefix + "iptables-save"], - pred=iptables_with_nft, -+ subdir=_subdir, - priority=50) - self.add_cmd_output([ns_cmd_prefix + "ip6tables-save"], - pred=ip6tables_with_nft, -+ subdir=_subdir, - priority=50) - - ss_cmd = ns_cmd_prefix + "ss -peaonmi" - # --allow-system-changes is handled directly in predicate - # evaluation, so plugin code does not need to separately - # check for it -- self.add_cmd_output(ss_cmd, pred=ss_pred) -- -- # Collect ethtool commands only when ethtool_namespaces -- # is set to true. -- if self.get_option("ethtool_namespaces"): -- # Devices that exist in a namespace use less ethtool -- # parameters. Run this per namespace. -- for namespace in self.get_network_namespaces( -- self.get_option("namespace_pattern"), -- self.get_option("namespaces")): -- ns_cmd_prefix = cmd_prefix + namespace + " " -+ self.add_cmd_output(ss_cmd, pred=ss_pred, subdir=_subdir) -+ -+ # Collect ethtool commands only when ethtool_namespaces -+ # is set to true. -+ if self.get_option("ethtool_namespaces"): -+ # Devices that exist in a namespace use less ethtool -+ # parameters. Run this per namespace. - netns_netdev_list = self.exec_cmd( - ns_cmd_prefix + "ls -1 /sys/class/net/" - ) -@@ -250,9 +249,7 @@ class Networking(Plugin): - ns_cmd_prefix + "ethtool -i " + eth, - ns_cmd_prefix + "ethtool -k " + eth, - ns_cmd_prefix + "ethtool -S " + eth -- ], priority=50) -- -- return -+ ], priority=50, subdir=_subdir) - - - class RedHatNetworking(Networking, RedHatPlugin): --- -2.31.1 - -From 4bf5f9143c962c839c1d27217ba74127551a5c00 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Fri, 17 Dec 2021 11:10:15 -0500 -Subject: [PATCH] [transport] Detect retrieval failures and automatically retry - -If a paritcular attempt to retrieve a remote file fails, we should -automatically retry that collection up to a certain point. This provides -`sos collect` more resiliency for the collection of sos report archives. - -This change necessitates a change in how we handle the SoSNode flow for -failed sos report retrievals, and as such contains minor fixes to -transports to ensure that we do not incorrectly hit exceptions in error -handling that were not previously possible with how we exited the -SoSNode retrieval flow. - -Closes: #2777 - -Signed-off-by: Jake Hunsaker ---- - sos/collector/__init__.py | 5 +++-- - sos/collector/clusters/ocp.py | 1 + - sos/collector/sosnode.py | 17 ++++++++++------- - sos/collector/transports/__init__.py | 15 ++++++++++++++- - sos/collector/transports/local.py | 1 + - sos/collector/transports/oc.py | 3 ++- - 6 files changed, 31 insertions(+), 11 deletions(-) - -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index b825d8fc..a25e794e 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -1221,8 +1221,9 @@ this utility or remote systems that it connects to. - def close_all_connections(self): - """Close all sessions for nodes""" - for client in self.client_list: -- self.log_debug('Closing connection to %s' % client.address) -- client.disconnect() -+ if client.connected: -+ self.log_debug('Closing connection to %s' % client.address) -+ client.disconnect() - - def create_cluster_archive(self): - """Calls for creation of tar archive then cleans up the temporary -diff --git a/sos/collector/clusters/ocp.py b/sos/collector/clusters/ocp.py -index 56f8cc47..ae93ad58 100644 ---- a/sos/collector/clusters/ocp.py -+++ b/sos/collector/clusters/ocp.py -@@ -92,6 +92,7 @@ class ocp(Cluster): - % ret['output']) - # don't leave the config on a non-existing project - self.exec_master_cmd("oc project default") -+ self.project = None - return True - - def _build_dict(self, nodelist): -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 1341e39f..925f2790 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -751,12 +751,11 @@ class SosNode(): - if self.file_exists(path): - self.log_info("Copying remote %s to local %s" % - (path, destdir)) -- self._transport.retrieve_file(path, dest) -+ return self._transport.retrieve_file(path, dest) - else: - self.log_debug("Attempting to copy remote file %s, but it " - "does not exist on filesystem" % path) - return False -- return True - except Exception as err: - self.log_debug("Failed to retrieve %s: %s" % (path, err)) - return False -@@ -793,16 +792,20 @@ class SosNode(): - except Exception: - self.log_error('Failed to make archive readable') - return False -- self.soslog.info('Retrieving sos report from %s' % self.address) -+ self.log_info('Retrieving sos report from %s' % self.address) - self.ui_msg('Retrieving sos report...') -- ret = self.retrieve_file(self.sos_path) -+ try: -+ ret = self.retrieve_file(self.sos_path) -+ except Exception as err: -+ self.log_error(err) -+ return False - if ret: - self.ui_msg('Successfully collected sos report') - self.file_list.append(self.sos_path.split('/')[-1]) -+ return True - else: -- self.log_error('Failed to retrieve sos report') -- raise SystemExit -- return True -+ self.ui_msg('Failed to retrieve sos report') -+ return False - else: - # sos sometimes fails but still returns a 0 exit code - if self.stderr.read(): -diff --git a/sos/collector/transports/__init__.py b/sos/collector/transports/__init__.py -index 33f2f66d..dcdebdde 100644 ---- a/sos/collector/transports/__init__.py -+++ b/sos/collector/transports/__init__.py -@@ -303,7 +303,20 @@ class RemoteTransport(): - :returns: True if file was successfully copied from remote, or False - :rtype: ``bool`` - """ -- return self._retrieve_file(fname, dest) -+ attempts = 0 -+ try: -+ while attempts < 5: -+ attempts += 1 -+ ret = self._retrieve_file(fname, dest) -+ if ret: -+ return True -+ self.log_info("File retrieval attempt %s failed" % attempts) -+ self.log_info("File retrieval failed after 5 attempts") -+ return False -+ except Exception as err: -+ self.log_error("Exception encountered during retrieval attempt %s " -+ "for %s: %s" % (attempts, fname, err)) -+ raise err - - def _retrieve_file(self, fname, dest): - raise NotImplementedError("Transport %s does not support file copying" -diff --git a/sos/collector/transports/local.py b/sos/collector/transports/local.py -index a4897f19..2996d524 100644 ---- a/sos/collector/transports/local.py -+++ b/sos/collector/transports/local.py -@@ -35,6 +35,7 @@ class LocalTransport(RemoteTransport): - def _retrieve_file(self, fname, dest): - self.log_debug("Moving %s to %s" % (fname, dest)) - shutil.copy(fname, dest) -+ return True - - def _format_cmd_for_exec(self, cmd): - return cmd -diff --git a/sos/collector/transports/oc.py b/sos/collector/transports/oc.py -index de044ccb..720dd61d 100644 ---- a/sos/collector/transports/oc.py -+++ b/sos/collector/transports/oc.py -@@ -202,7 +202,8 @@ class OCTransport(RemoteTransport): - env, False) - - def _disconnect(self): -- os.unlink(self.pod_tmp_conf) -+ if os.path.exists(self.pod_tmp_conf): -+ os.unlink(self.pod_tmp_conf) - removed = self.run_oc("delete pod %s" % self.pod_name) - if "deleted" not in removed['output']: - self.log_debug("Calling delete on pod '%s' failed: %s" --- -2.31.1 - -From 304c9ef6c1015f1ebe1a8d569c3e16bada4d23f1 Mon Sep 17 00:00:00 2001 -From: Nadia Pinaeva -Date: Tue, 4 Jan 2022 16:37:09 +0100 -Subject: [PATCH] Add cluster cleanup for all exit() calls - -Signed-off-by: Nadia Pinaeva ---- - sos/collector/__init__.py | 3 +-- - 1 file changed, 1 insertion(+), 2 deletions(-) - -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index a25e794e1..ffd63bc63 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -443,6 +443,7 @@ def add_parser_options(cls, parser): - - def exit(self, msg, error=1): - """Used to safely terminate if sos-collector encounters an error""" -+ self.cluster.cleanup() - self.log_error(msg) - try: - self.close_all_connections() -@@ -858,8 +858,9 @@ class SoSCollector(SoSComponent): - "CTRL-C to quit\n") - self.ui_log.info("") - except KeyboardInterrupt: -- self.cluster.cleanup() - self.exit("Exiting on user cancel", 130) -+ except Exception as e: -+ self.exit(repr(e), 1) - - def configure_sos_cmd(self): - """Configures the sosreport command that is run on the nodes""" -@@ -1185,7 +1185,6 @@ def collect(self): - arc_name = self.create_cluster_archive() - else: - msg = 'No sosreports were collected, nothing to archive...' -- self.cluster.cleanup() - self.exit(msg, 1) - - if self.opts.upload and self.policy.get_upload_url(): -From 2c3a647817dfbac36be3768acf6026e91d1a6e8f Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 21 Dec 2021 14:20:19 -0500 -Subject: [PATCH] [options] Allow spaces in --keywords values in sos.conf - -The `--keywords` option supports spaces to allow for obfuscated phrases, -not just words. This however breaks if a phrase is added to the config -file *before* a run with the phrase in the cmdline option, due to the -safeguards we have for all other values that do not support spaces. - -Add a check in our flow for updating options from the config file to not -replace illegal spaces if we're checking the `keywords` option, for -which spaces are legal. - -Signed-off-by: Jake Hunsaker ---- - sos/options.py | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/sos/options.py b/sos/options.py -index 7bea3ffc1..4846a5096 100644 ---- a/sos/options.py -+++ b/sos/options.py -@@ -200,7 +200,10 @@ def _update_from_section(section, config): - odict[rename_opts[key]] = odict.pop(key) - # set the values according to the config file - for key, val in odict.items(): -- if isinstance(val, str): -+ # most option values do not tolerate spaces, special -+ # exception however for --keywords which we do want to -+ # support phrases, and thus spaces, for -+ if isinstance(val, str) and key != 'keywords': - val = val.replace(' ', '') - if key not in self.arg_defaults: - # read an option that is not loaded by the current -From f912fc9e31b406a24b7a9c012e12cda920632051 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 10 Jan 2022 14:13:42 +0100 -Subject: [PATCH] [collect] Deal None sos_version properly - -In case collector cluster hits an error during init, sos_version -is None what LooseVersion can't compare properly and raises exception - -'LooseVersion' object has no attribute 'version' - -Related: #2822 - -Signed-off-by: Pavel Moravec ---- - sos/collector/sosnode.py | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 925f27909..7bbe0cd1f 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -382,7 +382,8 @@ def check_sos_version(self, ver): - given ver. This means that if the installed version is greater than - ver, this will still return True - """ -- return LooseVersion(self.sos_info['version']) >= ver -+ return self.sos_info['version'] is not None and \ -+ LooseVersion(self.sos_info['version']) >= ver - - def is_installed(self, pkg): - """Checks if a given package is installed on the node""" -From 0c67e8ebaeef17dac3b5b9e42a59b4e673e4403b Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 10 Jan 2022 14:17:13 +0100 -Subject: [PATCH] [collector] Cleanup cluster only if defined - -In case cluster init fails, self.cluster = None and its cleanup -must be skipped. - -Resolves: #2822 - -Signed-off-by: Pavel Moravec ---- - sos/collector/__init__.py | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index ffd63bc63..3e22bca3e 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -443,7 +443,8 @@ def add_parser_options(cls, parser): - - def exit(self, msg, error=1): - """Used to safely terminate if sos-collector encounters an error""" -- self.cluster.cleanup() -+ if self.cluster: -+ self.cluster.cleanup() - self.log_error(msg) - try: - self.close_all_connections() -From ef27a6ee6737c23b3beda1437768a91679024697 Mon Sep 17 00:00:00 2001 -From: Nadia Pinaeva -Date: Fri, 3 Dec 2021 15:41:35 +0100 -Subject: [PATCH] Add journal logs for NetworkManager plugin - -Signed-off-by: Nadia Pinaeva ---- - sos/report/plugins/networkmanager.py | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/sos/report/plugins/networkmanager.py b/sos/report/plugins/networkmanager.py -index 30f99a1140..3aca0c7460 100644 ---- a/sos/report/plugins/networkmanager.py -+++ b/sos/report/plugins/networkmanager.py -@@ -25,6 +25,8 @@ def setup(self): - "/etc/NetworkManager/dispatcher.d" - ]) - -+ self.add_journal(units="NetworkManager") -+ - # There are some incompatible changes in nmcli since - # the release of NetworkManager >= 0.9.9. In addition, - # NetworkManager >= 0.9.9 will use the long names of -From 9eb60f0bb6ea36f9c1cf099c1fd20cf3938b4b26 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Mon, 17 Jan 2022 11:11:24 -0500 -Subject: [PATCH] [clean] Ignore empty items for obfuscation better - -This commit fixes a couple edge cases where an item empty (e.g. and -empty string '') was not being properly ignored, which in turned caused -failures in writing both obfuscations and replacement files. - -This should no longer be possible. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/mappings/__init__.py | 5 ++++- - sos/cleaner/mappings/username_map.py | 2 +- - sos/cleaner/parsers/username_parser.py | 2 +- - 3 files changed, 6 insertions(+), 3 deletions(-) - -diff --git a/sos/cleaner/mappings/__init__.py b/sos/cleaner/mappings/__init__.py -index 5cf5c8b2d..48171a052 100644 ---- a/sos/cleaner/mappings/__init__.py -+++ b/sos/cleaner/mappings/__init__.py -@@ -49,6 +49,8 @@ def add(self, item): - :param item: The plaintext object to obfuscate - """ - with self.lock: -+ if not item: -+ return item - self.dataset[item] = self.sanitize_item(item) - return self.dataset[item] - -@@ -67,7 +69,8 @@ def get(self, item): - """Retrieve an item's obfuscated counterpart from the map. If the item - does not yet exist in the map, add it by generating one on the fly - """ -- if self.ignore_item(item) or self.item_in_dataset_values(item): -+ if (not item or self.ignore_item(item) or -+ self.item_in_dataset_values(item)): - return item - if item not in self.dataset: - return self.add(item) -diff --git a/sos/cleaner/mappings/username_map.py b/sos/cleaner/mappings/username_map.py -index 7ecccd7bc..ed6dc0912 100644 ---- a/sos/cleaner/mappings/username_map.py -+++ b/sos/cleaner/mappings/username_map.py -@@ -24,7 +24,7 @@ class SoSUsernameMap(SoSMap): - - def load_names_from_options(self, opt_names): - for name in opt_names: -- if name not in self.dataset.keys(): -+ if name and name not in self.dataset.keys(): - self.add(name) - - def sanitize_item(self, username): -diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py -index 49640f7fd..2853c860f 100644 ---- a/sos/cleaner/parsers/username_parser.py -+++ b/sos/cleaner/parsers/username_parser.py -@@ -55,7 +55,7 @@ def load_usernames_into_map(self, content): - user = line.split()[0] - except Exception: - continue -- if user.lower() in self.skip_list: -+ if not user or user.lower() in self.skip_list: - continue - users.add(user) - for each in users: -From ed618678fd3d07e68e1a430eb7d225a9701332e0 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 13 Jan 2022 13:52:34 -0500 -Subject: [PATCH] [clean,parsers] Build regex lists for static items only once - -For parsers such as the username and keyword parsers, we don't discover -new items through parsing archives - these parsers use static lists -determined before we begin the actual obfuscation process. - -As such, we can build a list of regexes for these static items once, and -then reference those regexes during execution, rather than rebuilding -the regex for each of these items for every obfuscation. - -For use cases where hundreds of items, e.g. hundreds of usernames, are -being obfuscated this results in a significant performance increase. -Individual per-file gains are minor - fractions of a second - however -these gains build up over the course of the hundreds to thousands of -files a typical archive can be expected to contain. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/__init__.py | 9 +++++++++ - sos/cleaner/parsers/__init__.py | 10 ++++++++++ - sos/cleaner/parsers/keyword_parser.py | 15 ++++++++++----- - sos/cleaner/parsers/username_parser.py | 14 ++++++++------ - tests/unittests/cleaner_tests.py | 1 + - 5 files changed, 38 insertions(+), 11 deletions(-) - -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index 5686e2131..b76bef644 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -294,6 +294,7 @@ def execute(self): - # we have at least one valid target to obfuscate - self.completed_reports = [] - self.preload_all_archives_into_maps() -+ self.generate_parser_item_regexes() - self.obfuscate_report_paths() - - if not self.completed_reports: -@@ -498,6 +499,14 @@ def _replace_obfuscated_archives(self): - shutil.move(archive.final_archive_path, dest) - archive.final_archive_path = dest_name - -+ def generate_parser_item_regexes(self): -+ """For the parsers that use prebuilt lists of items, generate those -+ regexes now since all the parsers should be preloaded by the archive(s) -+ as well as being handed cmdline options and mapping file configuration. -+ """ -+ for parser in self.parsers: -+ parser.generate_item_regexes() -+ - def preload_all_archives_into_maps(self): - """Before doing the actual obfuscation, if we have multiple archives - to obfuscate then we need to preload each of them into the mappings -diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py -index e62fd9384..6def863a6 100644 ---- a/sos/cleaner/parsers/__init__.py -+++ b/sos/cleaner/parsers/__init__.py -@@ -46,9 +46,19 @@ class SoSCleanerParser(): - map_file_key = 'unset' - - def __init__(self, config={}): -+ self.regexes = {} - if self.map_file_key in config: - self.mapping.conf_update(config[self.map_file_key]) - -+ def generate_item_regexes(self): -+ """Generate regexes for items the parser will be searching for -+ repeatedly without needing to generate them for every file and/or line -+ we process -+ -+ Not used by all parsers. -+ """ -+ pass -+ - def parse_line(self, line): - """This will be called for every line in every file we process, so that - every parser has a chance to scrub everything. -diff --git a/sos/cleaner/parsers/keyword_parser.py b/sos/cleaner/parsers/keyword_parser.py -index 694c6073a..362a1929e 100644 ---- a/sos/cleaner/parsers/keyword_parser.py -+++ b/sos/cleaner/parsers/keyword_parser.py -@@ -9,6 +9,7 @@ - # See the LICENSE file in the source distribution for further information. - - import os -+import re - - from sos.cleaner.parsers import SoSCleanerParser - from sos.cleaner.mappings.keyword_map import SoSKeywordMap -@@ -33,16 +34,20 @@ def __init__(self, config, keywords=None, keyword_file=None): - # pre-generate an obfuscation mapping for each keyword - # this is necessary for cases where filenames are being - # obfuscated before or instead of file content -- self.mapping.get(keyword) -+ self.mapping.get(keyword.lower()) - self.user_keywords.append(keyword) - if keyword_file and os.path.exists(keyword_file): - with open(keyword_file, 'r') as kwf: - self.user_keywords.extend(kwf.read().splitlines()) - -+ def generate_item_regexes(self): -+ for kw in self.user_keywords: -+ self.regexes[kw] = re.compile(kw, re.I) -+ - def parse_line(self, line): - count = 0 -- for keyword in sorted(self.user_keywords, reverse=True): -- if keyword in line: -- line = line.replace(keyword, self.mapping.get(keyword)) -- count += 1 -+ for kwrd, reg in sorted(self.regexes.items(), key=len, reverse=True): -+ if reg.search(line): -+ line, _count = reg.subn(self.mapping.get(kwrd.lower()), line) -+ count += _count - return line, count -diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py -index 3208a6557..49640f7fd 100644 ---- a/sos/cleaner/parsers/username_parser.py -+++ b/sos/cleaner/parsers/username_parser.py -@@ -61,12 +61,14 @@ def load_usernames_into_map(self, content): - for each in users: - self.mapping.get(each) - -+ def generate_item_regexes(self): -+ for user in self.mapping.dataset: -+ self.regexes[user] = re.compile(user, re.I) -+ - def parse_line(self, line): - count = 0 -- for username in sorted(self.mapping.dataset.keys(), reverse=True): -- _reg = re.compile(username, re.I) -- if _reg.search(line): -- line, count = _reg.subn( -- self.mapping.get(username.lower()), line -- ) -+ for user, reg in sorted(self.regexes.items(), key=len, reverse=True): -+ if reg.search(line): -+ line, _count = reg.subn(self.mapping.get(user.lower()), line) -+ count += _count - return line, count -diff --git a/tests/unittests/cleaner_tests.py b/tests/unittests/cleaner_tests.py -index cb20772fd..b59eade9a 100644 ---- a/tests/unittests/cleaner_tests.py -+++ b/tests/unittests/cleaner_tests.py -@@ -105,6 +105,7 @@ def setUp(self): - self.host_parser = SoSHostnameParser(config={}, opt_domains='foobar.com') - self.kw_parser = SoSKeywordParser(config={}, keywords=['foobar']) - self.kw_parser_none = SoSKeywordParser(config={}) -+ self.kw_parser.generate_item_regexes() - - def test_ip_parser_valid_ipv4_line(self): - line = 'foobar foo 10.0.0.1/24 barfoo bar' -From 2ae16e0245e1b01b8547e507abb69c11871a8467 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Mon, 21 Feb 2022 14:37:09 -0500 -Subject: [PATCH] [sosnode] Handle downstream versioning for runtime option - check - -First, adds parsing and formatting for an sos installation's release -version according to the loaded package manager for that node. - -Adds a fallback version check for 4.2-13 for RHEL downstreams that -backport the `container-runtime` option into sos-4.2. - -Carry this in upstream to account for use cases where a workstation used -to run `collect` from may be from a different stream than those used by -cluster nodes. - -Signed-off-by: Jake Hunsaker ---- - sos/collector/sosnode.py | 60 ++++++++++++++++++++++++++++++++++------ - 1 file changed, 51 insertions(+), 9 deletions(-) - -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 7bbe0cd1..d9b998b0 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -275,21 +275,34 @@ class SosNode(): - def _load_sos_info(self): - """Queries the node for information about the installed version of sos - """ -+ ver = None -+ rel = None - if self.host.container_version_command is None: - pkg = self.host.package_manager.pkg_version(self.host.sos_pkg_name) - if pkg is not None: - ver = '.'.join(pkg['version']) -- self.sos_info['version'] = ver -+ if pkg['release']: -+ rel = pkg['release'] -+ - else: - # use the containerized policy's command - pkgs = self.run_command(self.host.container_version_command, - use_container=True, need_root=True) - if pkgs['status'] == 0: -- ver = pkgs['output'].strip().split('-')[1] -- if ver: -- self.sos_info['version'] = ver -- else: -- self.sos_info['version'] = None -+ _, ver, rel = pkgs['output'].strip().split('-') -+ -+ if ver: -+ if len(ver.split('.')) == 2: -+ # safeguard against maintenance releases throwing off the -+ # comparison by LooseVersion -+ ver += '.0' -+ try: -+ ver += '-%s' % rel.split('.')[0] -+ except Exception as err: -+ self.log_debug("Unable to fully parse sos release: %s" % err) -+ -+ self.sos_info['version'] = ver -+ - if self.sos_info['version']: - self.log_info('sos version is %s' % self.sos_info['version']) - else: -@@ -381,9 +394,37 @@ class SosNode(): - """Checks to see if the sos installation on the node is AT LEAST the - given ver. This means that if the installed version is greater than - ver, this will still return True -+ -+ :param ver: Version number we are trying to verify is installed -+ :type ver: ``str`` -+ -+ :returns: True if installed version is at least ``ver``, else False -+ :rtype: ``bool`` - """ -- return self.sos_info['version'] is not None and \ -- LooseVersion(self.sos_info['version']) >= ver -+ def _format_version(ver): -+ # format the version we're checking to a standard form of X.Y.Z-R -+ try: -+ _fver = ver.split('-')[0] -+ _rel = '' -+ if '-' in ver: -+ _rel = '-' + ver.split('-')[-1].split('.')[0] -+ if len(_fver.split('.')) == 2: -+ _fver += '.0' -+ -+ return _fver + _rel -+ except Exception as err: -+ self.log_debug("Unable to format '%s': %s" % (ver, err)) -+ return ver -+ -+ _ver = _format_version(ver) -+ -+ try: -+ _node_ver = LooseVersion(self.sos_info['version']) -+ _test_ver = LooseVersion(_ver) -+ return _node_ver >= _test_ver -+ except Exception as err: -+ self.log_error("Error checking sos version: %s" % err) -+ return False - - def is_installed(self, pkg): - """Checks if a given package is installed on the node""" -@@ -587,7 +628,8 @@ class SosNode(): - sos_opts.append('--cmd-timeout=%s' - % quote(str(self.opts.cmd_timeout))) - -- if self.check_sos_version('4.3'): -+ # handle downstream versions that backported this option -+ if self.check_sos_version('4.3') or self.check_sos_version('4.2-13'): - if self.opts.container_runtime != 'auto': - sos_opts.append( - "--container-runtime=%s" % self.opts.container_runtime --- -2.34.1 - diff --git a/SOURCES/sos-bz2041855-virsh-in-foreground.patch b/SOURCES/sos-bz2041855-virsh-in-foreground.patch deleted file mode 100644 index 66bca13..0000000 --- a/SOURCES/sos-bz2041855-virsh-in-foreground.patch +++ /dev/null @@ -1,146 +0,0 @@ -From 137abd394f64a63b6633949b5c81159af12038b7 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Fri, 14 Jan 2022 20:07:17 +0100 -Subject: [PATCH] [report] pass foreground argument to collect_cmd_output - -Related to: #2825 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/__init__.py | 12 +++++++++--- - 1 file changed, 9 insertions(+), 3 deletions(-) - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index 98f163ab9..1bbdf28a4 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -1920,6 +1920,8 @@ class Plugin(object): - :param subdir: Subdir in plugin directory to save to - :param changes: Does this cmd potentially make a change - on the system? -+ :param foreground: Run the `cmd` in the foreground with a -+ TTY - :param tags: Add tags in the archive manifest - :param cmd_as_tag: Format command string to tag - -@@ -2145,7 +2147,8 @@ def collect_cmd_output(self, cmd, suggest_filename=None, - root_symlink=False, timeout=None, - stderr=True, chroot=True, runat=None, env=None, - binary=False, sizelimit=None, pred=None, -- changes=False, subdir=None, tags=[]): -+ changes=False, foreground=False, subdir=None, -+ tags=[]): - """Execute a command and save the output to a file for inclusion in the - report, then return the results for further use by the plugin - -@@ -2188,6 +2191,9 @@ def collect_cmd_output(self, cmd, suggest_filename=None, - on the system? - :type changes: ``bool`` - -+ :param foreground: Run the `cmd` in the foreground with a TTY -+ :type foreground: ``bool`` -+ - :param tags: Add tags in the archive manifest - :type tags: ``str`` or a ``list`` of strings - -@@ -2206,8 +2212,8 @@ def collect_cmd_output(self, cmd, suggest_filename=None, - return self._collect_cmd_output( - cmd, suggest_filename=suggest_filename, root_symlink=root_symlink, - timeout=timeout, stderr=stderr, chroot=chroot, runat=runat, -- env=env, binary=binary, sizelimit=sizelimit, subdir=subdir, -- tags=tags -+ env=env, binary=binary, sizelimit=sizelimit, foreground=foreground, -+ subdir=subdir, tags=tags - ) - - def exec_cmd(self, cmd, timeout=None, stderr=True, chroot=True, -From 747fef695e4ff08f320c5f03090bdefa7154c761 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Fri, 14 Jan 2022 20:10:22 +0100 -Subject: [PATCH] [virsh] Call virsh commands in the foreground / with a TTY - -In some virsh errors (like unable to connect to a hypervisor), -the tool requires to communicate to TTY otherwise it can get stuck -(when called via Popen with a timeout). - -Calling it on foreground prevents the stuck / waiting on cmd timeout. - -Resolves: #2825 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/virsh.py | 14 +++++++++----- - 1 file changed, 9 insertions(+), 5 deletions(-) - -diff --git a/sos/report/plugins/virsh.py b/sos/report/plugins/virsh.py -index d6b7c16761..08f9a8488c 100644 ---- a/sos/report/plugins/virsh.py -+++ b/sos/report/plugins/virsh.py -@@ -39,26 +39,30 @@ def setup(self): - ] - - for subcmd in subcmds: -- self.add_cmd_output('%s %s' % (cmd, subcmd)) -+ self.add_cmd_output('%s %s' % (cmd, subcmd), foreground=True) - - # get network, pool and nwfilter elements - for k in ['net', 'nwfilter', 'pool']: -- k_list = self.collect_cmd_output('%s %s-list' % (cmd, k)) -+ k_list = self.collect_cmd_output('%s %s-list' % (cmd, k), -+ foreground=True) - if k_list['status'] == 0: - k_lines = k_list['output'].splitlines() - # the 'Name' column position changes between virsh cmds - pos = k_lines[0].split().index('Name') - for j in filter(lambda x: x, k_lines[2:]): - n = j.split()[pos] -- self.add_cmd_output('%s %s-dumpxml %s' % (cmd, k, n)) -+ self.add_cmd_output('%s %s-dumpxml %s' % (cmd, k, n), -+ foreground=True) - - # cycle through the VMs/domains list, ignore 2 header lines and latest - # empty line, and dumpxml domain name in 2nd column -- domains_output = self.exec_cmd('%s list --all' % cmd) -+ domains_output = self.exec_cmd('%s list --all' % cmd, foreground=True) - if domains_output['status'] == 0: - domains_lines = domains_output['output'].splitlines()[2:] - for domain in filter(lambda x: x, domains_lines): - d = domain.split()[1] - for x in ['dumpxml', 'dominfo', 'domblklist']: -- self.add_cmd_output('%s %s %s' % (cmd, x, d)) -+ self.add_cmd_output('%s %s %s' % (cmd, x, d), -+ foreground=True) -+ - # vim: et ts=4 sw=4 -From 9bc032129ec66766f07349dd115335f104888efa Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Wed, 26 Jan 2022 09:44:01 +0100 -Subject: [PATCH] [virsh] Catch parsing exception - -In case virsh output is malformed or missing 'Name' otherwise, -catch parsing exception and continue in next for loop iteration. - -Resolves: #2836 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/virsh.py | 6 +++++- - 1 file changed, 5 insertions(+), 1 deletion(-) - -diff --git a/sos/report/plugins/virsh.py b/sos/report/plugins/virsh.py -index 08f9a8488..2ce1df15c 100644 ---- a/sos/report/plugins/virsh.py -+++ b/sos/report/plugins/virsh.py -@@ -48,7 +48,11 @@ def setup(self): - if k_list['status'] == 0: - k_lines = k_list['output'].splitlines() - # the 'Name' column position changes between virsh cmds -- pos = k_lines[0].split().index('Name') -+ # catch the rare exceptions when 'Name' is not found -+ try: -+ pos = k_lines[0].split().index('Name') -+ except Exception: -+ continue - for j in filter(lambda x: x, k_lines[2:]): - n = j.split()[pos] - self.add_cmd_output('%s %s-dumpxml %s' % (cmd, k, n), diff --git a/SOURCES/sos-bz2043104-foreman-tasks-msgpack.patch b/SOURCES/sos-bz2043104-foreman-tasks-msgpack.patch deleted file mode 100644 index 900389c..0000000 --- a/SOURCES/sos-bz2043104-foreman-tasks-msgpack.patch +++ /dev/null @@ -1,59 +0,0 @@ -From 5634f7dd77eff821f37daa953fa86cc783d3b937 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Fri, 21 Jan 2022 16:27:33 +0100 -Subject: [PATCH] [foreman] Use psql-msgpack-decode wrapper for dynflow >= 1.6 - -In dynflow >=1.6.3, dynflow* tables in postgres are encoded by -msgpack which makes plain CSV dumps unreadable. In such a case, -psql-msgpack-decode wrapper tool from dynflow-utils (of any -version) must be used instead of the plain psql command. - -Resolves: #2830 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/foreman.py | 16 ++++++++++++---- - 1 file changed, 12 insertions(+), 4 deletions(-) - -diff --git a/sos/report/plugins/foreman.py b/sos/report/plugins/foreman.py -index 314a651d1..3fd80e6a8 100644 ---- a/sos/report/plugins/foreman.py -+++ b/sos/report/plugins/foreman.py -@@ -244,8 +244,16 @@ def setup(self): - self.add_cmd_output(_cmd, suggest_filename=table, timeout=600, - sizelimit=100, env=self.env) - -+ # dynflow* tables on dynflow >=1.6.3 are encoded and hence in that -+ # case, psql-msgpack-decode wrapper tool from dynflow-utils (any -+ # version) must be used instead of plain psql command -+ dynutils = self.is_installed('dynflow-utils') - for dyn in foremancsv: -- _cmd = self.build_query_cmd(foremancsv[dyn], csv=True) -+ binary = "psql" -+ if dyn != 'foreman_tasks_tasks' and dynutils: -+ binary = "/usr/libexec/psql-msgpack-decode" -+ _cmd = self.build_query_cmd(foremancsv[dyn], csv=True, -+ binary=binary) - self.add_cmd_output(_cmd, suggest_filename=dyn, timeout=600, - sizelimit=100, env=self.env) - -@@ -270,7 +278,7 @@ def setup(self): - # collect http[|s]_proxy env.variables - self.add_env_var(["http_proxy", "https_proxy"]) - -- def build_query_cmd(self, query, csv=False): -+ def build_query_cmd(self, query, csv=False, binary="psql"): - """ - Builds the command needed to invoke the pgsql query as the postgres - user. -@@ -281,8 +289,8 @@ def build_query_cmd(self, query, csv=False): - if csv: - query = "COPY (%s) TO STDOUT " \ - "WITH (FORMAT 'csv', DELIMITER ',', HEADER)" % query -- _dbcmd = "psql --no-password -h %s -p 5432 -U foreman -d foreman -c %s" -- return _dbcmd % (self.dbhost, quote(query)) -+ _dbcmd = "%s --no-password -h %s -p 5432 -U foreman -d foreman -c %s" -+ return _dbcmd % (binary, self.dbhost, quote(query)) - - def postproc(self): - self.do_path_regex_sub( diff --git a/SOURCES/sos-bz2043488-ovn-proper-package-enablement.patch b/SOURCES/sos-bz2043488-ovn-proper-package-enablement.patch deleted file mode 100644 index 16c48c4..0000000 --- a/SOURCES/sos-bz2043488-ovn-proper-package-enablement.patch +++ /dev/null @@ -1,252 +0,0 @@ -From 210b83e1d1164d29b1f6198675b8b596c4af8336 Mon Sep 17 00:00:00 2001 -From: Daniel Alvarez Sanchez -Date: Thu, 20 Jan 2022 12:58:44 +0100 -Subject: [PATCH] [ovn_central] Account for Red Hat ovn package naming - -Previous ovn packages were 'ovn2xxx' and now they have -been renamed to 'ovn-2xxx'. This causes sos tool to not -recognize that the packages are installed and it won't -collect the relevant data. - -This patch is changing the match to be compatible -with the previous and newer naming conventions. - -Signed-off-by: Daniel Alvarez Sanchez ---- - sos/report/plugins/ovn_central.py | 2 +- - sos/report/plugins/ovn_host.py | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/sos/report/plugins/ovn_central.py b/sos/report/plugins/ovn_central.py -index ddbf288da..0f947d4c5 100644 ---- a/sos/report/plugins/ovn_central.py -+++ b/sos/report/plugins/ovn_central.py -@@ -147,7 +147,7 @@ def setup(self): - - class RedHatOVNCentral(OVNCentral, RedHatPlugin): - -- packages = ('openvswitch-ovn-central', 'ovn2.*-central', ) -+ packages = ('openvswitch-ovn-central', 'ovn.*-central', ) - ovn_sbdb_sock_path = '/var/run/openvswitch/ovnsb_db.ctl' - - -diff --git a/sos/report/plugins/ovn_host.py b/sos/report/plugins/ovn_host.py -index 78604a15a..25c38cccc 100644 ---- a/sos/report/plugins/ovn_host.py -+++ b/sos/report/plugins/ovn_host.py -@@ -55,7 +55,7 @@ def check_enabled(self): - - class RedHatOVNHost(OVNHost, RedHatPlugin): - -- packages = ('openvswitch-ovn-host', 'ovn2.*-host', ) -+ packages = ('openvswitch-ovn-host', 'ovn.*-host', ) - - - class DebianOVNHost(OVNHost, DebianPlugin, UbuntuPlugin): -From 21fc376d97a5f74743e2b7cf7069349e874b979e Mon Sep 17 00:00:00 2001 -From: Hemanth Nakkina -Date: Fri, 4 Feb 2022 07:57:59 +0530 -Subject: [PATCH] [ovn-central] collect NB/SB ovsdb-server cluster status - -Add commands to collect cluster status of Northbound and -Southbound ovsdb servers. - -Resolves: #2840 - -Signed-off-by: Hemanth Nakkina hemanth.nakkina@canonical.com ---- - sos/report/plugins/ovn_central.py | 13 ++++++++++++- - 1 file changed, 12 insertions(+), 1 deletion(-) - -diff --git a/sos/report/plugins/ovn_central.py b/sos/report/plugins/ovn_central.py -index 0f947d4c5..2f0438df3 100644 ---- a/sos/report/plugins/ovn_central.py -+++ b/sos/report/plugins/ovn_central.py -@@ -84,6 +84,14 @@ def setup(self): - else: - self.add_copy_spec("/var/log/ovn/*.log") - -+ # ovsdb nb/sb cluster status commands -+ ovsdb_cmds = [ -+ 'ovs-appctl -t {} cluster/status OVN_Northbound'.format( -+ self.ovn_nbdb_sock_path), -+ 'ovs-appctl -t {} cluster/status OVN_Southbound'.format( -+ self.ovn_sbdb_sock_path), -+ ] -+ - # Some user-friendly versions of DB output - nbctl_cmds = [ - 'ovn-nbctl show', -@@ -109,7 +117,8 @@ def setup(self): - - self.add_database_output(nb_tables, nbctl_cmds, 'ovn-nbctl') - -- cmds = nbctl_cmds -+ cmds = ovsdb_cmds -+ cmds += nbctl_cmds - - # Can only run sbdb commands if we are the leader - co = {'cmd': "ovs-appctl -t {} cluster/status OVN_Southbound". -@@ -148,10 +157,12 @@ def setup(self): - class RedHatOVNCentral(OVNCentral, RedHatPlugin): - - packages = ('openvswitch-ovn-central', 'ovn.*-central', ) -+ ovn_nbdb_sock_path = '/var/run/openvswitch/ovnnb_db.ctl' - ovn_sbdb_sock_path = '/var/run/openvswitch/ovnsb_db.ctl' - - - class DebianOVNCentral(OVNCentral, DebianPlugin, UbuntuPlugin): - - packages = ('ovn-central', ) -+ ovn_nbdb_sock_path = '/var/run/ovn/ovnnb_db.ctl' - ovn_sbdb_sock_path = '/var/run/ovn/ovnsb_db.ctl' -From d0f9d507b0ec63c9e8f3e5d7b6507d9d0f97c038 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 15 Feb 2022 16:24:47 -0500 -Subject: [PATCH] [runtimes] Allow container IDs to be used with - `container_exists()` - -As container runtimes can interchange container names and container IDs, -sos should also allow the use of container IDs when checking for the -presence of a given container. - -In particular, this change unblocks the use of `Plugin.exec_cmd()` when -used in conjunction with `Plugin.get_container_by_name()` to pick a -container based on a provided regex that the container name may match. - -Related: #2856 - -Signed-off-by: Jake Hunsaker ---- - sos/policies/runtimes/__init__.py | 17 +++++++++++++++++ - sos/report/plugins/__init__.py | 6 +++--- - 2 files changed, 20 insertions(+), 3 deletions(-) - -diff --git a/sos/policies/runtimes/__init__.py b/sos/policies/runtimes/__init__.py -index 5ac673544..d28373496 100644 ---- a/sos/policies/runtimes/__init__.py -+++ b/sos/policies/runtimes/__init__.py -@@ -147,6 +147,23 @@ def get_volumes(self): - vols.append(ent[-1]) - return vols - -+ def container_exists(self, container): -+ """Check if a given container ID or name exists on the system from the -+ perspective of the container runtime. -+ -+ Note that this will only check _running_ containers -+ -+ :param container: The name or ID of the container -+ :type container: ``str`` -+ -+ :returns: True if the container exists, else False -+ :rtype: ``bool`` -+ """ -+ for _contup in self.containers: -+ if container in _contup: -+ return True -+ return False -+ - def fmt_container_cmd(self, container, cmd, quotecmd): - """Format a command to run inside a container using the runtime - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index 2988be089..cc5cb65bc 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -2593,7 +2593,7 @@ def container_exists(self, name): - """If a container runtime is present, check to see if a container with - a given name is currently running - -- :param name: The name of the container to check presence of -+ :param name: The name or ID of the container to check presence of - :type name: ``str`` - - :returns: ``True`` if `name` exists, else ``False`` -@@ -2601,8 +2601,8 @@ def container_exists(self, name): - """ - _runtime = self._get_container_runtime() - if _runtime is not None: -- con = _runtime.get_container_by_name(name) -- return con is not None -+ return (_runtime.container_exists(name) or -+ _runtime.get_container_by_name(name) is not None) - return False - - def get_all_containers_by_regex(self, regex, get_all=False): - -From de9b020a72d1ceda39587db4c6d5acf72cd90da2 Mon Sep 17 00:00:00 2001 -From: Fernando Royo -Date: Tue, 15 Feb 2022 10:00:38 +0100 -Subject: [PATCH] [ovn_central] Rename container responsable of Red Hat - ovn_central plugin - -ovn_central plugin is running by container with -name 'ovn-dbs-bundle*', a typo has been identified and -this cause plugin ovn_central not enabled by default as it -does not recognize any container responsible of this. - -This patch fix this container name match, searching schema db -keeping backward compatibility with openvswitch. ---- - sos/report/plugins/ovn_central.py | 23 ++++++++++++----------- - 1 file changed, 12 insertions(+), 11 deletions(-) - -diff --git a/sos/report/plugins/ovn_central.py b/sos/report/plugins/ovn_central.py -index 2f0438df..2f34bff0 100644 ---- a/sos/report/plugins/ovn_central.py -+++ b/sos/report/plugins/ovn_central.py -@@ -24,7 +24,7 @@ class OVNCentral(Plugin): - short_desc = 'OVN Northd' - plugin_name = "ovn_central" - profiles = ('network', 'virt') -- containers = ('ovs-db-bundle.*',) -+ containers = ('ovn-dbs-bundle.*',) - - def get_tables_from_schema(self, filename, skip=[]): - if self._container_name: -@@ -66,7 +66,7 @@ class OVNCentral(Plugin): - cmds.append('%s list %s' % (ovn_cmd, table)) - - def setup(self): -- self._container_name = self.get_container_by_name('ovs-dbs-bundle.*') -+ self._container_name = self.get_container_by_name(self.containers[0]) - - ovs_rundir = os.environ.get('OVS_RUNDIR') - for pidfile in ['ovnnb_db.pid', 'ovnsb_db.pid', 'ovn-northd.pid']: -@@ -110,12 +110,11 @@ class OVNCentral(Plugin): - 'ovn-sbctl get-connection', - ] - -- schema_dir = '/usr/share/openvswitch' -- -- nb_tables = self.get_tables_from_schema(self.path_join( -- schema_dir, 'ovn-nb.ovsschema')) -- -- self.add_database_output(nb_tables, nbctl_cmds, 'ovn-nbctl') -+ # backward compatibility -+ for path in ['/usr/share/openvswitch', '/usr/share/ovn']: -+ nb_tables = self.get_tables_from_schema(self.path_join( -+ path, 'ovn-nb.ovsschema')) -+ self.add_database_output(nb_tables, nbctl_cmds, 'ovn-nbctl') - - cmds = ovsdb_cmds - cmds += nbctl_cmds -@@ -125,9 +124,11 @@ class OVNCentral(Plugin): - format(self.ovn_sbdb_sock_path), - "output": "Leader: self"} - if self.test_predicate(self, pred=SoSPredicate(self, cmd_outputs=co)): -- sb_tables = self.get_tables_from_schema(self.path_join( -- schema_dir, 'ovn-sb.ovsschema'), ['Logical_Flow']) -- self.add_database_output(sb_tables, sbctl_cmds, 'ovn-sbctl') -+ # backward compatibility -+ for path in ['/usr/share/openvswitch', '/usr/share/ovn']: -+ sb_tables = self.get_tables_from_schema(self.path_join( -+ path, 'ovn-sb.ovsschema'), ['Logical_Flow']) -+ self.add_database_output(sb_tables, sbctl_cmds, 'ovn-sbctl') - cmds += sbctl_cmds - - # If OVN is containerized, we need to run the above commands inside --- -2.34.1 - diff --git a/SOURCES/sos-bz2054883-plugopt-logging-effective-opts.patch b/SOURCES/sos-bz2054883-plugopt-logging-effective-opts.patch deleted file mode 100644 index f8e7ed3..0000000 --- a/SOURCES/sos-bz2054883-plugopt-logging-effective-opts.patch +++ /dev/null @@ -1,94 +0,0 @@ -From 5824cd5d3bddf39e0382d568419e2453abc93d8a Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Mon, 30 Aug 2021 15:09:07 -0400 -Subject: [PATCH] [options] Fix logging on plugopts in effective sos command - -First, provide a special-case handling for plugin options specified in -sos.conf in `SoSOptions.to_args().has_value()` that allows for plugin -options to be included in the "effective options now" log message. - -Second, move the logging of said message (and thus the merging of -preset options, if used), to being _prior_ to the loading of plugin -options. - -Combined, plugin options specified in sos.conf will now be logged -properly and this logging will occur before we set (and log the setting -of) those options. - -Resolves: #2663 - -Signed-off-by: Jake Hunsaker ---- - sos/options.py | 2 ++ - sos/report/__init__.py | 30 ++++++++++++++++-------------- - 2 files changed, 18 insertions(+), 14 deletions(-) - -diff --git a/sos/options.py b/sos/options.py -index a014a022..7bea3ffc 100644 ---- a/sos/options.py -+++ b/sos/options.py -@@ -281,6 +281,8 @@ class SoSOptions(): - null_values = ("False", "None", "[]", '""', "''", "0") - if not value or value in null_values: - return False -+ if name == 'plugopts' and value: -+ return True - if name in self.arg_defaults: - if str(value) == str(self.arg_defaults[name]): - return False -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index b0159e5b..82484f1d 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -925,20 +925,6 @@ class SoSReport(SoSComponent): - self._exit(1) - - def setup(self): -- # Log command line options -- msg = "[%s:%s] executing 'sos %s'" -- self.soslog.info(msg % (__name__, "setup", " ".join(self.cmdline))) -- -- # Log active preset defaults -- preset_args = self.preset.opts.to_args() -- msg = ("[%s:%s] using '%s' preset defaults (%s)" % -- (__name__, "setup", self.preset.name, " ".join(preset_args))) -- self.soslog.info(msg) -- -- # Log effective options after applying preset defaults -- self.soslog.info("[%s:%s] effective options now: %s" % -- (__name__, "setup", " ".join(self.opts.to_args()))) -- - self.ui_log.info(_(" Setting up plugins ...")) - for plugname, plug in self.loaded_plugins: - try: -@@ -1386,11 +1372,27 @@ class SoSReport(SoSComponent): - self.report_md.add_list('disabled_plugins', self.opts.skip_plugins) - self.report_md.add_section('plugins') - -+ def _merge_preset_options(self): -+ # Log command line options -+ msg = "[%s:%s] executing 'sos %s'" -+ self.soslog.info(msg % (__name__, "setup", " ".join(self.cmdline))) -+ -+ # Log active preset defaults -+ preset_args = self.preset.opts.to_args() -+ msg = ("[%s:%s] using '%s' preset defaults (%s)" % -+ (__name__, "setup", self.preset.name, " ".join(preset_args))) -+ self.soslog.info(msg) -+ -+ # Log effective options after applying preset defaults -+ self.soslog.info("[%s:%s] effective options now: %s" % -+ (__name__, "setup", " ".join(self.opts.to_args()))) -+ - def execute(self): - try: - self.policy.set_commons(self.get_commons()) - self.load_plugins() - self._set_all_options() -+ self._merge_preset_options() - self._set_tunables() - self._check_for_unknown_plugins() - self._set_plugin_options() --- -2.34.1 - diff --git a/SOURCES/sos-bz2055003-rebase-sos-add-sos-help.patch b/SOURCES/sos-bz2055003-rebase-sos-add-sos-help.patch new file mode 100644 index 0000000..1b8af24 --- /dev/null +++ b/SOURCES/sos-bz2055003-rebase-sos-add-sos-help.patch @@ -0,0 +1,67 @@ +From b5389aa195675f473acdd22f20017a8854ff82d0 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Wed, 16 Feb 2022 08:43:32 +0100 +Subject: [PATCH] [man] Mention sos-help in main sos manpage + +Related to #2860 + +Signed-off-by: Pavel Moravec +--- + man/en/sos.1 | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/man/en/sos.1 b/man/en/sos.1 +index ce4918f99..c335b7e10 100644 +--- a/man/en/sos.1 ++++ b/man/en/sos.1 +@@ -67,6 +67,14 @@ May be invoked via either \fBsos clean\fR, \fBsos cleaner\fR, \fBsos mask\fR, + or via the \fB--clean\fR, \fB--cleaner\fR or \fB --mask\fR options + for \fBreport\fR and \fBcollect\fR. + ++.TP ++.B help ++This subcommand is used to retrieve more detailed information on the various SoS ++commands and components than is directly available in either other manpages or ++--help output. ++ ++See \fB sos help --help\fR and \fB man sos-help\fR for more information. ++ + .SH GLOBAL OPTIONS + sos components provide their own set of options, however the following are available + to be set across all components. +From ac4eb48fa35c13b99ada41540831412480babf8d Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Wed, 16 Feb 2022 08:44:16 +0100 +Subject: [PATCH] [setup] Add sos-help to build process + +Resolves: #2860 +Closes: #2861 + +Signed-off-by: Pavel Moravec +--- + setup.py | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/setup.py b/setup.py +index 25e87a71b..8db8641f0 100644 +--- a/setup.py ++++ b/setup.py +@@ -90,7 +90,7 @@ def copy_file (self, filename, dirname): + ('share/man/man1', ['man/en/sosreport.1', 'man/en/sos-report.1', + 'man/en/sos.1', 'man/en/sos-collect.1', + 'man/en/sos-collector.1', 'man/en/sos-clean.1', +- 'man/en/sos-mask.1']), ++ 'man/en/sos-mask.1', 'man/en/sos-help.1']), + ('share/man/man5', ['man/en/sos.conf.5']), + ('share/licenses/sos', ['LICENSE']), + ('share/doc/sos', ['AUTHORS', 'README.md']), +@@ -102,7 +102,8 @@ def copy_file (self, filename, dirname): + 'sos.policies.package_managers', 'sos.policies.init_systems', + 'sos.report', 'sos.report.plugins', 'sos.collector', + 'sos.collector.clusters', 'sos.collector.transports', 'sos.cleaner', +- 'sos.cleaner.mappings', 'sos.cleaner.parsers', 'sos.cleaner.archives' ++ 'sos.cleaner.mappings', 'sos.cleaner.parsers', 'sos.cleaner.archives', ++ 'sos.help' + ], + cmdclass=cmdclass, + command_options=command_options, diff --git a/SOURCES/sos-bz2055548-honour-plugins-timeout-hardcoded.patch b/SOURCES/sos-bz2055548-honour-plugins-timeout-hardcoded.patch deleted file mode 100644 index 3adde40..0000000 --- a/SOURCES/sos-bz2055548-honour-plugins-timeout-hardcoded.patch +++ /dev/null @@ -1,39 +0,0 @@ -From 7069e99d1c5c443f96a98a7ed6db67fa14683e67 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Thu, 17 Feb 2022 09:14:15 +0100 -Subject: [PATCH] [report] Honor plugins' hardcoded plugin_timeout - -Currently, plugin's plugin_timeout hardcoded default is superseded by -whatever --plugin-timeout value, even when this option is not used and -we eval it to TIMEOUT_DEFAULT. - -In this case of not setting --plugin-timeout either -k plugin.timeout, -honour plugin's plugin_timeout instead. - -Resolves: #2863 -Closes: #2864 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/__init__.py | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index cc5cb65b..336b4d22 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -636,7 +636,10 @@ class Plugin(): - if opt_timeout is None: - _timeout = own_timeout - elif opt_timeout is not None and own_timeout == -1: -- _timeout = int(opt_timeout) -+ if opt_timeout == TIMEOUT_DEFAULT: -+ _timeout = default_timeout -+ else: -+ _timeout = int(opt_timeout) - elif opt_timeout is not None and own_timeout > -1: - _timeout = own_timeout - else: --- -2.34.1 - diff --git a/SOURCES/sos-bz2065563-ocp-backports.patch b/SOURCES/sos-bz2065563-ocp-backports.patch new file mode 100644 index 0000000..7cfaa91 --- /dev/null +++ b/SOURCES/sos-bz2065563-ocp-backports.patch @@ -0,0 +1,1113 @@ +From d0f9d507b0ec63c9e8f3e5d7b6507d9d0f97c038 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Tue, 15 Feb 2022 16:24:47 -0500 +Subject: [PATCH] [runtimes] Allow container IDs to be used with + `container_exists()` + +As container runtimes can interchange container names and container IDs, +sos should also allow the use of container IDs when checking for the +presence of a given container. + +In particular, this change unblocks the use of `Plugin.exec_cmd()` when +used in conjunction with `Plugin.get_container_by_name()` to pick a +container based on a provided regex that the container name may match. + +Related: #2856 + +Signed-off-by: Jake Hunsaker +--- + sos/policies/runtimes/__init__.py | 17 +++++++++++++++++ + sos/report/plugins/__init__.py | 6 +++--- + 2 files changed, 20 insertions(+), 3 deletions(-) + +diff --git a/sos/policies/runtimes/__init__.py b/sos/policies/runtimes/__init__.py +index 5ac67354..d2837349 100644 +--- a/sos/policies/runtimes/__init__.py ++++ b/sos/policies/runtimes/__init__.py +@@ -147,6 +147,23 @@ class ContainerRuntime(): + vols.append(ent[-1]) + return vols + ++ def container_exists(self, container): ++ """Check if a given container ID or name exists on the system from the ++ perspective of the container runtime. ++ ++ Note that this will only check _running_ containers ++ ++ :param container: The name or ID of the container ++ :type container: ``str`` ++ ++ :returns: True if the container exists, else False ++ :rtype: ``bool`` ++ """ ++ for _contup in self.containers: ++ if container in _contup: ++ return True ++ return False ++ + def fmt_container_cmd(self, container, cmd, quotecmd): + """Format a command to run inside a container using the runtime + +diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py +index 2988be08..cc5cb65b 100644 +--- a/sos/report/plugins/__init__.py ++++ b/sos/report/plugins/__init__.py +@@ -2593,7 +2593,7 @@ class Plugin(): + """If a container runtime is present, check to see if a container with + a given name is currently running + +- :param name: The name of the container to check presence of ++ :param name: The name or ID of the container to check presence of + :type name: ``str`` + + :returns: ``True`` if `name` exists, else ``False`` +@@ -2601,8 +2601,8 @@ class Plugin(): + """ + _runtime = self._get_container_runtime() + if _runtime is not None: +- con = _runtime.get_container_by_name(name) +- return con is not None ++ return (_runtime.container_exists(name) or ++ _runtime.get_container_by_name(name) is not None) + return False + + def get_all_containers_by_regex(self, regex, get_all=False): +-- +2.34.3 + +From 2ae16e0245e1b01b8547e507abb69c11871a8467 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Mon, 21 Feb 2022 14:37:09 -0500 +Subject: [PATCH] [sosnode] Handle downstream versioning for runtime option + check + +First, adds parsing and formatting for an sos installation's release +version according to the loaded package manager for that node. + +Adds a fallback version check for 4.2-13 for RHEL downstreams that +backport the `container-runtime` option into sos-4.2. + +Carry this in upstream to account for use cases where a workstation used +to run `collect` from may be from a different stream than those used by +cluster nodes. + +Signed-off-by: Jake Hunsaker +--- + sos/collector/sosnode.py | 60 ++++++++++++++++++++++++++++++++++------ + 1 file changed, 51 insertions(+), 9 deletions(-) + +diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py +index 7bbe0cd1..d9b998b0 100644 +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -275,21 +275,34 @@ class SosNode(): + def _load_sos_info(self): + """Queries the node for information about the installed version of sos + """ ++ ver = None ++ rel = None + if self.host.container_version_command is None: + pkg = self.host.package_manager.pkg_version(self.host.sos_pkg_name) + if pkg is not None: + ver = '.'.join(pkg['version']) +- self.sos_info['version'] = ver ++ if pkg['release']: ++ rel = pkg['release'] ++ + else: + # use the containerized policy's command + pkgs = self.run_command(self.host.container_version_command, + use_container=True, need_root=True) + if pkgs['status'] == 0: +- ver = pkgs['output'].strip().split('-')[1] +- if ver: +- self.sos_info['version'] = ver +- else: +- self.sos_info['version'] = None ++ _, ver, rel = pkgs['output'].strip().split('-') ++ ++ if ver: ++ if len(ver.split('.')) == 2: ++ # safeguard against maintenance releases throwing off the ++ # comparison by LooseVersion ++ ver += '.0' ++ try: ++ ver += '-%s' % rel.split('.')[0] ++ except Exception as err: ++ self.log_debug("Unable to fully parse sos release: %s" % err) ++ ++ self.sos_info['version'] = ver ++ + if self.sos_info['version']: + self.log_info('sos version is %s' % self.sos_info['version']) + else: +@@ -381,9 +394,37 @@ class SosNode(): + """Checks to see if the sos installation on the node is AT LEAST the + given ver. This means that if the installed version is greater than + ver, this will still return True ++ ++ :param ver: Version number we are trying to verify is installed ++ :type ver: ``str`` ++ ++ :returns: True if installed version is at least ``ver``, else False ++ :rtype: ``bool`` + """ +- return self.sos_info['version'] is not None and \ +- LooseVersion(self.sos_info['version']) >= ver ++ def _format_version(ver): ++ # format the version we're checking to a standard form of X.Y.Z-R ++ try: ++ _fver = ver.split('-')[0] ++ _rel = '' ++ if '-' in ver: ++ _rel = '-' + ver.split('-')[-1].split('.')[0] ++ if len(_fver.split('.')) == 2: ++ _fver += '.0' ++ ++ return _fver + _rel ++ except Exception as err: ++ self.log_debug("Unable to format '%s': %s" % (ver, err)) ++ return ver ++ ++ _ver = _format_version(ver) ++ ++ try: ++ _node_ver = LooseVersion(self.sos_info['version']) ++ _test_ver = LooseVersion(_ver) ++ return _node_ver >= _test_ver ++ except Exception as err: ++ self.log_error("Error checking sos version: %s" % err) ++ return False + + def is_installed(self, pkg): + """Checks if a given package is installed on the node""" +@@ -587,7 +628,8 @@ class SosNode(): + sos_opts.append('--cmd-timeout=%s' + % quote(str(self.opts.cmd_timeout))) + +- if self.check_sos_version('4.3'): ++ # handle downstream versions that backported this option ++ if self.check_sos_version('4.3') or self.check_sos_version('4.2-13'): + if self.opts.container_runtime != 'auto': + sos_opts.append( + "--container-runtime=%s" % self.opts.container_runtime +-- +2.34.3 + +From cc60fa5ee25bffed9203a4f786256185b7fe0115 Mon Sep 17 00:00:00 2001 +From: Nadia Pinaeva +Date: Tue, 15 Mar 2022 11:49:57 +0100 +Subject: [PATCH] Add ovs datapath and groups collection commands Add + ct-zone-list command for openshift-ovn + +Signed-off-by: Nadia Pinaeva +--- + sos/report/plugins/openshift_ovn.py | 4 ++++ + sos/report/plugins/openvswitch.py | 3 +++ + 2 files changed, 7 insertions(+) + +diff --git a/sos/report/plugins/openshift_ovn.py b/sos/report/plugins/openshift_ovn.py +index 168f1dd3..b4787b8e 100644 +--- a/sos/report/plugins/openshift_ovn.py ++++ b/sos/report/plugins/openshift_ovn.py +@@ -34,6 +34,10 @@ class OpenshiftOVN(Plugin, RedHatPlugin): + 'ovn-appctl -t /var/run/ovn/ovnsb_db.ctl ' + + 'cluster/status OVN_Southbound'], + container='ovnkube-master') ++ self.add_cmd_output([ ++ 'ovs-appctl -t /var/run/ovn/ovn-controller.*.ctl ' + ++ 'ct-zone-list'], ++ container='ovnkube-node') + self.add_cmd_output([ + 'ovs-appctl -t ovs-monitor-ipsec tunnels/show', + 'ipsec status', +diff --git a/sos/report/plugins/openvswitch.py b/sos/report/plugins/openvswitch.py +index 179d1532..159b0bd2 100644 +--- a/sos/report/plugins/openvswitch.py ++++ b/sos/report/plugins/openvswitch.py +@@ -124,6 +124,8 @@ class OpenVSwitch(Plugin): + "ovs-vsctl -t 5 list interface", + # Capture OVS detailed information from all the bridges + "ovs-vsctl -t 5 list bridge", ++ # Capture OVS datapath list ++ "ovs-vsctl -t 5 list datapath", + # Capture DPDK queue to pmd mapping + "ovs-appctl dpif-netdev/pmd-rxq-show", + # Capture DPDK pmd stats +@@ -229,6 +231,7 @@ class OpenVSwitch(Plugin): + "ovs-ofctl queue-get-config %s" % br, + "ovs-ofctl queue-stats %s" % br, + "ovs-ofctl show %s" % br, ++ "ovs-ofctl dump-groups %s" % br, + ]) + + # Flow protocols currently supported +-- +2.34.3 + +From af40be92f502b35fa9d39ce4d4fea7d80c367830 Mon Sep 17 00:00:00 2001 +From: Nadia Pinaeva +Date: Tue, 15 Mar 2022 13:09:55 +0100 +Subject: [PATCH] Improve sos collect for OCP: 1. wait for sos tmp project to + be deleted (just calling delete changes project state to Terminating, and + running a new sos collect is not possible before this project is fully + deleted) 2. use --retries flag to copy sos reports from the nodes more + reliably. The flag has been recently added to kubectl, and the most reliable + way to check if it's available or not is to check command error output for + "unknown flag" substring + +Signed-off-by: Nadia Pinaeva +--- + sos/collector/clusters/ocp.py | 5 +++++ + sos/collector/transports/oc.py | 6 +++++- + 2 files changed, 10 insertions(+), 1 deletion(-) + +diff --git a/sos/collector/clusters/ocp.py b/sos/collector/clusters/ocp.py +index f1714239..9beb2f9b 100644 +--- a/sos/collector/clusters/ocp.py ++++ b/sos/collector/clusters/ocp.py +@@ -123,6 +123,11 @@ class ocp(Cluster): + if not ret['status'] == 0: + self.log_error("Error deleting temporary project: %s" + % ret['output']) ++ ret = self.exec_primary_cmd("oc wait namespace/%s --for=delete " ++ "--timeout=30s" % self.project) ++ if not ret['status'] == 0: ++ self.log_error("Error waiting for temporary project to be " ++ "deleted: %s" % ret['output']) + # don't leave the config on a non-existing project + self.exec_primary_cmd("oc project default") + self.project = None +diff --git a/sos/collector/transports/oc.py b/sos/collector/transports/oc.py +index 0fc9eee8..90a802b2 100644 +--- a/sos/collector/transports/oc.py ++++ b/sos/collector/transports/oc.py +@@ -231,5 +231,9 @@ class OCTransport(RemoteTransport): + % (self.project, self.pod_name)) + + def _retrieve_file(self, fname, dest): +- cmd = self.run_oc("cp %s:%s %s" % (self.pod_name, fname, dest)) ++ # check if --retries flag is available for given version of oc ++ result = self.run_oc("cp --retries", stderr=True) ++ flags = '' if "unknown flag" in result["output"] else '--retries=5' ++ cmd = self.run_oc("cp %s %s:%s %s" ++ % (flags, self.pod_name, fname, dest)) + return cmd['status'] == 0 +-- +2.34.3 + +From 3b0676b90ff65f20eaba3062775ff72b89386ffc Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Tue, 22 Mar 2022 14:25:24 -0400 +Subject: [PATCH] [Plugin] Allow plugins to define default command environment + vars + +Adds the ability for plugins to define a default set of environment vars +to pass to all commands executed by the plugin. This may be done either +via the new `set_default_cmd_environment()` or +`add_default_cmd_environment()` methods. The former will override any +previously set values, whereas the latter will add/update/modify any +existing values. + +Signed-off-by: Jake Hunsaker +--- + sos/report/plugins/__init__.py | 55 ++++++++++++++++++- + .../plugin_tests/plugin_environment.py | 44 +++++++++++++++ + .../fake_plugins/default_env_test.py | 28 ++++++++++ + tests/unittests/plugin_tests.py | 15 +++++ + 4 files changed, 140 insertions(+), 2 deletions(-) + create mode 100644 tests/report_tests/plugin_tests/plugin_environment.py + create mode 100644 tests/test_data/fake_plugins/default_env_test.py + +diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py +index 336b4d22..74b4f4be 100644 +--- a/sos/report/plugins/__init__.py ++++ b/sos/report/plugins/__init__.py +@@ -571,6 +571,7 @@ class Plugin(): + self.manifest = None + self.skip_files = commons['cmdlineopts'].skip_files + self.skip_commands = commons['cmdlineopts'].skip_commands ++ self.default_environment = {} + + self.soslog = self.commons['soslog'] if 'soslog' in self.commons \ + else logging.getLogger('sos') +@@ -624,6 +625,52 @@ class Plugin(): + self.manifest.add_field('strings', {}) + self.manifest.add_field('containers', {}) + ++ def set_default_cmd_environment(self, env_vars): ++ """ ++ Specify a collection of environment variables that should always be ++ passed to commands being executed by this plugin. ++ ++ :param env_vars: The environment variables and their values to set ++ :type env_vars: ``dict{ENV_VAR_NAME: ENV_VAR_VALUE}`` ++ """ ++ if not isinstance(env_vars, dict): ++ raise TypeError( ++ "Environment variables for Plugin must be specified by dict" ++ ) ++ self.default_environment = env_vars ++ self._log_debug("Default environment for all commands now set to %s" ++ % self.default_environment) ++ ++ def add_default_cmd_environment(self, env_vars): ++ """ ++ Add or modify a specific environment variable in the set of default ++ environment variables used by this Plugin. ++ ++ :param env_vars: The environment variables to add to the current ++ set of env vars in use ++ :type env_vars: ``dict`` ++ """ ++ if not isinstance(env_vars, dict): ++ raise TypeError("Environment variables must be added via dict") ++ self._log_debug("Adding %s to default environment" % env_vars) ++ self.default_environment.update(env_vars) ++ ++ def _get_cmd_environment(self, env=None): ++ """ ++ Get the merged set of environment variables for a command about to be ++ executed by this plugin. ++ ++ :returns: The set of env vars to use for a command ++ :rtype: ``dict`` ++ """ ++ if env is None: ++ return self.default_environment ++ if not isinstance(env, dict): ++ raise TypeError("Command env vars must be passed as dict") ++ _env = self.default_environment.copy() ++ _env.update(env) ++ return _env ++ + def timeout_from_options(self, optname, plugoptname, default_timeout): + """Returns either the default [plugin|cmd] timeout value, the value as + provided on the commandline via -k plugin.[|cmd-]timeout=value, or the +@@ -2258,6 +2305,8 @@ class Plugin(): + + _tags = list(set(_tags)) + ++ _env = self._get_cmd_environment(env) ++ + if chroot or self.commons['cmdlineopts'].chroot == 'always': + root = self.sysroot + else: +@@ -2282,7 +2331,7 @@ class Plugin(): + + result = sos_get_command_output( + cmd, timeout=timeout, stderr=stderr, chroot=root, +- chdir=runat, env=env, binary=binary, sizelimit=sizelimit, ++ chdir=runat, env=_env, binary=binary, sizelimit=sizelimit, + poller=self.check_timeout, foreground=foreground, + to_file=out_file + ) +@@ -2510,6 +2559,8 @@ class Plugin(): + else: + root = None + ++ _env = self._get_cmd_environment(env) ++ + if container: + if self._get_container_runtime() is None: + self._log_info("Cannot run cmd '%s' in container %s: no " +@@ -2522,7 +2573,7 @@ class Plugin(): + "container is running." % (cmd, container)) + + return sos_get_command_output(cmd, timeout=timeout, chroot=root, +- chdir=runat, binary=binary, env=env, ++ chdir=runat, binary=binary, env=_env, + foreground=foreground, stderr=stderr) + + def _add_container_file_to_manifest(self, container, path, arcpath, tags): +diff --git a/tests/report_tests/plugin_tests/plugin_environment.py b/tests/report_tests/plugin_tests/plugin_environment.py +new file mode 100644 +index 00000000..3158437a +--- /dev/null ++++ b/tests/report_tests/plugin_tests/plugin_environment.py +@@ -0,0 +1,44 @@ ++# This file is part of the sos project: https://github.com/sosreport/sos ++# ++# This copyrighted material is made available to anyone wishing to use, ++# modify, copy, or redistribute it subject to the terms and conditions of ++# version 2 of the GNU General Public License. ++# ++# See the LICENSE file in the source distribution for further information. ++ ++import os ++ ++from sos_tests import StageTwoReportTest ++ ++ ++class PluginDefaultEnvironmentTest(StageTwoReportTest): ++ """ ++ Ensure that being able to set a default set of environment variables is ++ working correctly and does not leave a lingering env var on the system ++ ++ :avocado: tags=stageone ++ """ ++ ++ install_plugins = ['default_env_test'] ++ sos_cmd = '-o default_env_test' ++ ++ def test_environment_used_in_cmd(self): ++ self.assertFileHasContent( ++ 'sos_commands/default_env_test/env_var_test', ++ 'Does Linus play hockey?' ++ ) ++ ++ def test_environment_setting_logged(self): ++ self.assertSosLogContains( ++ 'Default environment for all commands now set to' ++ ) ++ ++ def test_environment_not_set_on_host(self): ++ self.assertTrue('TORVALDS' not in os.environ) ++ self.assertTrue('GREATESTSPORT' not in os.environ) ++ ++ def test_environment_not_captured(self): ++ # we should still have an empty environment file ++ self.assertFileCollected('environment') ++ self.assertFileNotHasContent('environment', 'TORVALDS') ++ self.assertFileNotHasContent('environment', 'GREATESTSPORT') +diff --git a/tests/test_data/fake_plugins/default_env_test.py b/tests/test_data/fake_plugins/default_env_test.py +new file mode 100644 +index 00000000..d1d1fb78 +--- /dev/null ++++ b/tests/test_data/fake_plugins/default_env_test.py +@@ -0,0 +1,28 @@ ++# This file is part of the sos project: https://github.com/sosreport/sos ++# ++# This copyrighted material is made available to anyone wishing to use, ++# modify, copy, or redistribute it subject to the terms and conditions of ++# version 2 of the GNU General Public License. ++# ++# See the LICENSE file in the source distribution for further information. ++ ++from sos.report.plugins import Plugin, IndependentPlugin ++ ++ ++class DefaultEnv(Plugin, IndependentPlugin): ++ ++ plugin_name = 'default_env_test' ++ short_desc = 'Fake plugin to test default env var handling' ++ ++ def setup(self): ++ self.set_default_cmd_environment({ ++ 'TORVALDS': 'Linus', ++ 'GREATESTSPORT': 'hockey' ++ }) ++ ++ self.add_cmd_output( ++ "sh -c 'echo Does '$TORVALDS' play '$GREATESTSPORT'?'", ++ suggest_filename='env_var_test' ++ ) ++ ++ self.add_env_var(['TORVALDS', 'GREATESTSPORT']) +diff --git a/tests/unittests/plugin_tests.py b/tests/unittests/plugin_tests.py +index 0dfa243d..e469b78e 100644 +--- a/tests/unittests/plugin_tests.py ++++ b/tests/unittests/plugin_tests.py +@@ -305,6 +305,21 @@ class PluginTests(unittest.TestCase): + p.postproc() + self.assertTrue(p.did_postproc) + ++ def test_set_default_cmd_env(self): ++ p = MockPlugin({ ++ 'sysroot': self.sysroot, ++ 'policy': LinuxPolicy(init=InitSystem(), probe_runtime=False), ++ 'cmdlineopts': MockOptions(), ++ 'devices': {} ++ }) ++ e = {'TORVALDS': 'Linus'} ++ p.set_default_cmd_environment(e) ++ self.assertEquals(p.default_environment, e) ++ add_e = {'GREATESTSPORT': 'hockey'} ++ p.add_default_cmd_environment(add_e) ++ self.assertEquals(p.default_environment['GREATESTSPORT'], 'hockey') ++ self.assertEquals(p.default_environment['TORVALDS'], 'Linus') ++ + + class AddCopySpecTests(unittest.TestCase): + +-- +2.34.3 + +From 1e12325efaa500d304dcbfbeeb50e72ed0f938f5 Mon Sep 17 00:00:00 2001 +From: Vladislav Walek <22072258+vwalek@users.noreply.github.com> +Date: Thu, 17 Mar 2022 14:10:26 -0700 +Subject: [PATCH] [openshift] Adding ability to use the localhost.kubeconfig + and KUBECONFIG env to use system:admin + +Signed-off-by: Vladislav Walek <22072258+vwalek@users.noreply.github.com> +--- + sos/report/plugins/openshift.py | 45 +++++++++++++++++++++++++++++++-- + 1 file changed, 43 insertions(+), 2 deletions(-) + +diff --git a/sos/report/plugins/openshift.py b/sos/report/plugins/openshift.py +index 5ae38178..d643f04c 100644 +--- a/sos/report/plugins/openshift.py ++++ b/sos/report/plugins/openshift.py +@@ -53,12 +53,19 @@ class Openshift(Plugin, RedHatPlugin): + profiles = ('openshift',) + packages = ('openshift-hyperkube',) + ++ master_localhost_kubeconfig = ( ++ '/etc/kubernetes/static-pod-resources/' ++ 'kube-apiserver-certs/secrets/node-kubeconfigs/localhost.kubeconfig' ++ ) ++ + option_list = [ + PluginOpt('token', default=None, val_type=str, + desc='admin token to allow API queries'), ++ PluginOpt('kubeconfig', default=None, val_type=str, ++ desc='Path to a locally available kubeconfig file'), + PluginOpt('host', default='https://localhost:6443', + desc='host address to use for oc login, including port'), +- PluginOpt('no-oc', default=False, desc='do not collect `oc` output'), ++ PluginOpt('no-oc', default=True, desc='do not collect `oc` output'), + PluginOpt('podlogs', default=True, desc='collect logs from each pod'), + PluginOpt('podlogs-filter', default='', val_type=str, + desc='only collect logs from pods matching this pattern'), +@@ -73,6 +80,10 @@ class Openshift(Plugin, RedHatPlugin): + """Check to see if we can run `oc` commands""" + return self.exec_cmd('oc whoami')['status'] == 0 + ++ def _check_localhost_kubeconfig(self): ++ """Check if the localhost.kubeconfig exists with system:admin user""" ++ return self.path_exists(self.get_option('kubeconfig')) ++ + def _check_oc_logged_in(self): + """See if we're logged in to the API service, and if not attempt to do + so using provided plugin options +@@ -80,8 +91,38 @@ class Openshift(Plugin, RedHatPlugin): + if self._check_oc_function(): + return True + +- # Not logged in currently, attempt to do so ++ if self.get_option('kubeconfig') is None: ++ # If admin doesn't add the kubeconfig ++ # use default localhost.kubeconfig ++ self.set_option( ++ 'kubeconfig', ++ self.master_localhost_kubeconfig ++ ) ++ ++ # Check first if we can use the localhost.kubeconfig before ++ # using token. We don't want to use 'host' option due we use ++ # cluster url from kubeconfig. Default is localhost. ++ if self._check_localhost_kubeconfig(): ++ self.set_default_cmd_environment({ ++ 'KUBECONFIG': self.get_option('kubeconfig') ++ }) ++ ++ oc_res = self.exec_cmd( ++ "oc login -u system:admin " ++ "--insecure-skip-tls-verify=True" ++ ) ++ if oc_res['status'] == 0 and self._check_oc_function(): ++ return True ++ ++ self._log_warn( ++ "The login command failed with status: %s and error: %s" ++ % (oc_res['status'], oc_res['output']) ++ ) ++ return False ++ ++ # If kubeconfig is not defined, check if token is provided. + token = self.get_option('token') or os.getenv('SOSOCPTOKEN', None) ++ + if token: + oc_res = self.exec_cmd("oc login %s --token=%s " + "--insecure-skip-tls-verify=True" +-- +2.34.3 + +From 61765992812afb785e9552e01e3b5579118a6963 Mon Sep 17 00:00:00 2001 +From: Nadia Pinaeva +Date: Fri, 1 Apr 2022 12:05:36 +0200 +Subject: [PATCH] Add one more container for plugin enablement + +Signed-off-by: Nadia Pinaeva +--- + sos/report/plugins/openshift_ovn.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/sos/report/plugins/openshift_ovn.py b/sos/report/plugins/openshift_ovn.py +index b4787b8e..98522b1e 100644 +--- a/sos/report/plugins/openshift_ovn.py ++++ b/sos/report/plugins/openshift_ovn.py +@@ -16,7 +16,7 @@ class OpenshiftOVN(Plugin, RedHatPlugin): + """ + short_desc = 'Openshift OVN' + plugin_name = "openshift_ovn" +- containers = ('ovnkube-master', 'ovn-ipsec') ++ containers = ('ovnkube-master', 'ovnkube-node', 'ovn-ipsec') + profiles = ('openshift',) + + def setup(self): +-- +2.34.3 + +From d3aa071efc85507341cf65dd61414a734654f50a Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Mon, 28 Mar 2022 14:47:09 -0400 +Subject: [PATCH] [presets] Adjust OCP preset options + +Adjust the options used by the 'ocp' preset to better reflect the +current collection needs and approach. + +This includes disabling the `cgroups` plugin due to the large amount of +mostly irrelevant data captured due to the high number of containers +present on OCP nodes, ensuring the `--container-runtime` option is set +to `crio` to align container-based collections, disabling HTML report +generation and increasing the base log size rather than blindly enabling +all-logs. + +Signed-off-by: Jake Hunsaker +--- + sos/presets/redhat/__init__.py | 13 +++++++++---- + 1 file changed, 9 insertions(+), 4 deletions(-) + +diff --git a/sos/presets/redhat/__init__.py b/sos/presets/redhat/__init__.py +index 865c9b6b..0b9f6f11 100644 +--- a/sos/presets/redhat/__init__.py ++++ b/sos/presets/redhat/__init__.py +@@ -36,10 +36,15 @@ RHOSP_OPTS = SoSOptions(plugopts=[ + + RHOCP = "ocp" + RHOCP_DESC = "OpenShift Container Platform by Red Hat" +-RHOCP_OPTS = SoSOptions(all_logs=True, verify=True, plugopts=[ +- 'networking.timeout=600', +- 'networking.ethtool_namespaces=False', +- 'networking.namespaces=200']) ++RHOCP_OPTS = SoSOptions( ++ verify=True, skip_plugins=['cgroups'], container_runtime='crio', ++ no_report=True, log_size=100, ++ plugopts=[ ++ 'crio.timeout=600', ++ 'networking.timeout=600', ++ 'networking.ethtool_namespaces=False', ++ 'networking.namespaces=200' ++ ]) + + RH_CFME = "cfme" + RH_CFME_DESC = "Red Hat CloudForms" +-- +2.34.3 + +From f2b67ab820070063995689fed03492cdaa012d01 Mon Sep 17 00:00:00 2001 +From: Nadia Pinaeva +Date: Fri, 1 Apr 2022 17:01:35 +0200 +Subject: [PATCH] Use /etc/os-release instead of /etc/redhat-release as the + most compatible way to find host release + +Signed-off-by: Nadia Pinaeva +--- + sos/policies/distros/redhat.py | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/sos/policies/distros/redhat.py b/sos/policies/distros/redhat.py +index 0c72a5e4..2e117f37 100644 +--- a/sos/policies/distros/redhat.py ++++ b/sos/policies/distros/redhat.py +@@ -40,7 +40,6 @@ class RedHatPolicy(LinuxPolicy): + ('Distribution Website', 'https://www.redhat.com/'), + ('Commercial Support', 'https://www.access.redhat.com/') + ] +- _redhat_release = '/etc/redhat-release' + _tmp_dir = "/var/tmp" + _in_container = False + default_scl_prefix = '/opt/rh' +@@ -471,7 +470,7 @@ support representative. + atomic = False + if ENV_HOST_SYSROOT not in os.environ: + return atomic +- host_release = os.environ[ENV_HOST_SYSROOT] + cls._redhat_release ++ host_release = os.environ[ENV_HOST_SYSROOT] + OS_RELEASE + if not os.path.exists(host_release): + return False + try: +@@ -558,7 +557,7 @@ support representative. + coreos = False + if ENV_HOST_SYSROOT not in os.environ: + return coreos +- host_release = os.environ[ENV_HOST_SYSROOT] + cls._redhat_release ++ host_release = os.environ[ENV_HOST_SYSROOT] + OS_RELEASE + try: + for line in open(host_release, 'r').read().splitlines(): + coreos |= 'Red Hat Enterprise Linux CoreOS' in line +-- +2.34.3 + +From ee0dd68199a2c9296eafe64ead5b2263c8270e4a Mon Sep 17 00:00:00 2001 +From: Nadia Pinaeva +Date: Wed, 6 Apr 2022 11:56:41 +0200 +Subject: [PATCH] Use --force-pull-image option for pods created with oc. Set + --force-pull-image=True by default, can be turned off with + --force-pull-image=False + +Signed-off-by: Nadia Pinaeva +--- + man/en/sos-collect.1 | 16 +++++++++++----- + sos/collector/__init__.py | 9 +++++---- + sos/collector/transports/oc.py | 2 ++ + sos/options.py | 20 ++++++++++++++------ + 4 files changed, 32 insertions(+), 15 deletions(-) + +diff --git a/man/en/sos-collect.1 b/man/en/sos-collect.1 +index 9b0a5d7b..2f60332b 100644 +--- a/man/en/sos-collect.1 ++++ b/man/en/sos-collect.1 +@@ -28,7 +28,7 @@ sos collect \- Collect sosreports from multiple (cluster) nodes + [\-\-no\-local] + [\-\-primary PRIMARY] + [\-\-image IMAGE] +- [\-\-force-pull-image] ++ [\-\-force-pull-image TOGGLE, --pull TOGGLE] + [\-\-registry-user USER] + [\-\-registry-password PASSWORD] + [\-\-registry-authfile FILE] +@@ -262,10 +262,16 @@ Specify an image to use for the temporary container created for collections on + containerized host, if you do not want to use the default image specifed by the + host's policy. Note that this should include the registry. + .TP +-\fB\-\-force-pull-image\fR +-Use this option to force the container runtime to pull the specified image (even +-if it is the policy default image) even if the image already exists on the host. +-This may be useful to update an older container image on containerized hosts. ++\fB\-\-force-pull-image TOGGLE, \-\-pull TOGGLE\fR ++When collecting an sos report from a containerized host, force the host to always ++pull the specified image, even if that image already exists on the host. ++This is useful to ensure that the latest version of that image is always in use. ++Disabling this option will use whatever version of the image is present on the node, ++and only attempt a pull if there is no copy of the image present at all. ++ ++Enable with true/on/yes or disable with false/off/no ++ ++Default: true + .TP + \fB\-\-registry-user USER\fR + Specify the username to authenticate to the registry with in order to pull the container +diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py +index d898ca34..66c3d932 100644 +--- a/sos/collector/__init__.py ++++ b/sos/collector/__init__.py +@@ -27,7 +27,7 @@ from pipes import quote + from textwrap import fill + from sos.cleaner import SoSCleaner + from sos.collector.sosnode import SosNode +-from sos.options import ClusterOption ++from sos.options import ClusterOption, str_to_bool + from sos.component import SoSComponent + from sos.utilities import bold + from sos import __version__ +@@ -85,7 +85,7 @@ class SoSCollector(SoSComponent): + 'encrypt_pass': '', + 'group': None, + 'image': '', +- 'force_pull_image': False, ++ 'force_pull_image': True, + 'jobs': 4, + 'keywords': [], + 'keyword_file': None, +@@ -357,8 +357,9 @@ class SoSCollector(SoSComponent): + collect_grp.add_argument('--image', + help=('Specify the container image to use for' + ' containerized hosts.')) +- collect_grp.add_argument('--force-pull-image', '--pull', default=False, +- action='store_true', ++ collect_grp.add_argument('--force-pull-image', '--pull', ++ default=True, choices=(True, False), ++ type=str_to_bool, + help='Force pull the container image even if ' + 'it already exists on the host') + collect_grp.add_argument('--registry-user', default=None, +diff --git a/sos/collector/transports/oc.py b/sos/collector/transports/oc.py +index 90a802b2..8f6aa9b4 100644 +--- a/sos/collector/transports/oc.py ++++ b/sos/collector/transports/oc.py +@@ -147,6 +147,8 @@ class OCTransport(RemoteTransport): + "tty": True + } + ], ++ "imagePullPolicy": ++ "Always" if self.opts.force_pull_image else "IfNotPresent", + "restartPolicy": "Never", + "nodeName": self.address, + "hostNetwork": True, +diff --git a/sos/options.py b/sos/options.py +index 4846a509..2d5a5135 100644 +--- a/sos/options.py ++++ b/sos/options.py +@@ -18,6 +18,16 @@ def _is_seq(val): + return val_type is list or val_type is tuple + + ++def str_to_bool(val): ++ _val = val.lower() ++ if _val in ['true', 'on', 'yes']: ++ return True ++ elif _val in ['false', 'off', 'no']: ++ return False ++ else: ++ return None ++ ++ + class SoSOptions(): + + def _merge_opt(self, opt, src, is_default): +@@ -153,15 +163,13 @@ class SoSOptions(): + if isinstance(self.arg_defaults[key], list): + return [v for v in val.split(',')] + if isinstance(self.arg_defaults[key], bool): +- _val = val.lower() +- if _val in ['true', 'on', 'yes']: +- return True +- elif _val in ['false', 'off', 'no']: +- return False +- else: ++ val = str_to_bool(val) ++ if val is None: + raise Exception( + "Value of '%s' in %s must be True or False or analagous" + % (key, conf)) ++ else: ++ return val + if isinstance(self.arg_defaults[key], int): + try: + return int(val) +-- +2.34.3 + +From ce289a3ae7101a898efdb84ddfd575576ba5819b Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Tue, 5 Apr 2022 11:32:11 -0400 +Subject: [PATCH] [ocp, openshift] Re-align API collection options and rename + option + +Previously, in #2888, the `openshift` plugin was extended to allow API +collections by using a default-available kubeconfig file rather than +relying on user-provided tokens. This also included flipping the default +value of the `no-oc` plugin option to `True` (meaning do not collect API +output by default). + +This worked for the plugin, but it introduced a gap in `sos collect` +whereby the cluster profile could no longer reliably enable API +collections when trying to leverage the new functionality of not +requiring a user token. + +Fix this by updating the cluster profile to align with the new +default-off approach of API collections. + +Along with this, add a toggle to the cluster profile directly to allow +users to toggle API collections on or off (default off) directly. This +is done via a new `with-api` cluster option (e.g. `-c ocp.with-api`). +Further, rename the `openshift` plugin option from `no-oc` to +`with-api`. This change not only makes the option use case far more +obvious, it will also align the use of the option to both `collect` and +`report` so that users need only be aware of a single option for either +method. + +The cluster profile also has logic to detect which plugin option, +`no-oc` or `with-api` to use based on the (RHEL) sos version installed +on the nodes being inspected by the `ocp` cluster profile. + +Signed-off-by: Jake Hunsaker +--- + sos/collector/clusters/ocp.py | 72 +++++++++++++++++++++++++++------ + sos/report/plugins/openshift.py | 26 +++++++----- + 2 files changed, 77 insertions(+), 21 deletions(-) + +diff --git a/sos/collector/clusters/ocp.py b/sos/collector/clusters/ocp.py +index 9beb2f9b..e31d1903 100644 +--- a/sos/collector/clusters/ocp.py ++++ b/sos/collector/clusters/ocp.py +@@ -30,7 +30,11 @@ class ocp(Cluster): + clusterAdmin privileges. + + If this requires the use of a secondary configuration file, specify that +- path with the 'kubeconfig' cluster option. ++ path with the 'kubeconfig' cluster option. This config file will also be ++ used on a single master node to perform API collections if the `with-api` ++ option is enabled (default disabled). If no `kubeconfig` option is given, ++ but `with-api` is enabled, the cluster profile will attempt to use a ++ well-known default kubeconfig file if it is available on the host. + + Alternatively, provide a clusterAdmin access token either via the 'token' + cluster option or, preferably, the SOSOCPTOKEN environment variable. +@@ -45,7 +49,7 @@ class ocp(Cluster): + option mentioned above. + + To avoid redundant collections of OCP API information (e.g. 'oc get' +- commands), this profile will attempt to enable the openshift plugin on only ++ commands), this profile will attempt to enable the API collections on only + a single master node. If the none of the master nodes have a functional + 'oc' binary available, *and* the --no-local option is used, that means that + no API data will be collected. +@@ -63,7 +67,8 @@ class ocp(Cluster): + ('label', '', 'Colon delimited list of labels to select nodes with'), + ('role', 'master', 'Colon delimited list of roles to filter on'), + ('kubeconfig', '', 'Path to the kubeconfig file'), +- ('token', '', 'Service account token to use for oc authorization') ++ ('token', '', 'Service account token to use for oc authorization'), ++ ('with-api', False, 'Collect OCP API data from a master node') + ] + + def fmt_oc_cmd(self, cmd): +@@ -219,13 +224,52 @@ class ocp(Cluster): + return False + return 'master' in self.node_dict[sosnode.address]['roles'] + ++ def _toggle_api_opt(self, node, use_api): ++ """In earlier versions of sos, the openshift plugin option that is ++ used to toggle the API collections was called `no-oc` rather than ++ `with-api`. This older plugin option had the inverse logic of the ++ current `with-api` option. ++ ++ Use this to toggle the correct plugin option given the node's sos ++ version. Note that the use of version 4.2 here is tied to the RHEL ++ release (the only usecase for this cluster profile) rather than ++ the upstream version given the backports for that downstream. ++ ++ :param node: The node being inspected for API collections ++ :type node: ``SoSNode`` ++ ++ :param use_api: Should this node enable API collections? ++ :type use_api: ``bool`` ++ """ ++ if node.check_sos_version('4.2-16'): ++ _opt = 'with-api' ++ _val = 'on' if use_api else 'off' ++ else: ++ _opt = 'no-oc' ++ _val = 'off' if use_api else 'on' ++ node.plugopts.append("openshift.%s=%s" % (_opt, _val)) ++ + def set_primary_options(self, node): ++ + node.enable_plugins.append('openshift') ++ if not self.get_option('with-api'): ++ self._toggle_api_opt(node, False) ++ return + if self.api_collect_enabled: + # a primary has already been enabled for API collection, disable + # it among others +- node.plugopts.append('openshift.no-oc=on') ++ self._toggle_api_opt(node, False) + else: ++ # running in a container, so reference the /host mount point ++ master_kube = ( ++ '/host/etc/kubernetes/static-pod-resources/' ++ 'kube-apiserver-certs/secrets/node-kubeconfigs/' ++ 'localhost.kubeconfig' ++ ) ++ _optconfig = self.get_option('kubeconfig') ++ if _optconfig and not _optconfig.startswith('/host'): ++ _optconfig = '/host/' + _optconfig ++ _kubeconfig = _optconfig or master_kube + _oc_cmd = 'oc' + if node.host.containerized: + _oc_cmd = '/host/bin/oc' +@@ -244,17 +288,21 @@ class ocp(Cluster): + need_root=True) + if can_oc['status'] == 0: + # the primary node can already access the API ++ self._toggle_api_opt(node, True) + self.api_collect_enabled = True + elif self.token: + node.sos_env_vars['SOSOCPTOKEN'] = self.token ++ self._toggle_api_opt(node, True) ++ self.api_collect_enabled = True ++ elif node.file_exists(_kubeconfig): ++ # if the file exists, then the openshift sos plugin will use it ++ # if the with-api option is turned on ++ if not _kubeconfig == master_kube: ++ node.plugopts.append( ++ "openshift.kubeconfig=%s" % _kubeconfig ++ ) ++ self._toggle_api_opt(node, True) + self.api_collect_enabled = True +- elif self.get_option('kubeconfig'): +- kc = self.get_option('kubeconfig') +- if node.file_exists(kc): +- if node.host.containerized: +- kc = "/host/%s" % kc +- node.sos_env_vars['KUBECONFIG'] = kc +- self.api_collect_enabled = True + if self.api_collect_enabled: + msg = ("API collections will be performed on %s\nNote: API " + "collections may extend runtime by 10s of minutes\n" +@@ -264,6 +312,6 @@ class ocp(Cluster): + + def set_node_options(self, node): + # don't attempt OC API collections on non-primary nodes +- node.plugopts.append('openshift.no-oc=on') ++ self._toggle_api_opt(node, False) + + # vim: set et ts=4 sw=4 : +diff --git a/sos/report/plugins/openshift.py b/sos/report/plugins/openshift.py +index d643f04c..a41ab62b 100644 +--- a/sos/report/plugins/openshift.py ++++ b/sos/report/plugins/openshift.py +@@ -19,7 +19,10 @@ class Openshift(Plugin, RedHatPlugin): + further extending the kubernetes plugin (or the OCP 3.x extensions included + in the Red Hat version of the kube plugin). + +- By default, this plugin will collect cluster information and inspect the ++ This plugin may collect OCP API information when the `with-api` option is ++ enabled. This option is disabled by default. ++ ++ When enabled, this plugin will collect cluster information and inspect the + default namespaces/projects that are created during deployment - i.e. the + namespaces of the cluster projects matching openshift.* and kube.*. At the + time of this plugin's creation that number of default projects is already +@@ -34,16 +37,20 @@ class Openshift(Plugin, RedHatPlugin): + + Users will need to either: + +- 1) Provide the bearer token via the `-k openshift.token` option +- 2) Provide the bearer token via the `SOSOCPTOKEN` environment variable +- 3) Otherwise ensure that the root user can successfully run `oc` and ++ 1) Accept the use of a well-known stock kubeconfig file provided via a ++ static pod resource for the kube-apiserver ++ 2) Provide the bearer token via the `-k openshift.token` option ++ 3) Provide the bearer token via the `SOSOCPTOKEN` environment variable ++ 4) Otherwise ensure that the root user can successfully run `oc` and + get proper output prior to running this plugin + + +- It is highly suggested that option #2 be used first, as this will prevent +- the token from being recorded in output saved to the archive. Option #1 may ++ It is highly suggested that option #1 be used first, as this uses well ++ known configurations and requires the least information from the user. If ++ using a token, it is recommended to use option #3 as this will prevent ++ the token from being recorded in output saved to the archive. Option #2 may + be used if this is considered an acceptable risk. It is not recommended to +- rely on option #3, though it will provide the functionality needed. ++ rely on option #4, though it will provide the functionality needed. + """ + + short_desc = 'Openshift Container Platform 4.x' +@@ -65,7 +72,8 @@ class Openshift(Plugin, RedHatPlugin): + desc='Path to a locally available kubeconfig file'), + PluginOpt('host', default='https://localhost:6443', + desc='host address to use for oc login, including port'), +- PluginOpt('no-oc', default=True, desc='do not collect `oc` output'), ++ PluginOpt('with-api', default=False, ++ desc='collect output from the OCP API'), + PluginOpt('podlogs', default=True, desc='collect logs from each pod'), + PluginOpt('podlogs-filter', default='', val_type=str, + desc='only collect logs from pods matching this pattern'), +@@ -212,7 +220,7 @@ class Openshift(Plugin, RedHatPlugin): + self.add_copy_spec('/etc/kubernetes/*') + + # see if we run `oc` commands +- if not self.get_option('no-oc'): ++ if self.get_option('with-api'): + can_run_oc = self._check_oc_logged_in() + else: + can_run_oc = False +-- +2.34.3 + diff --git a/SOURCES/sos-bz2066181-tigervnc-update-collections.patch b/SOURCES/sos-bz2066181-tigervnc-update-collections.patch new file mode 100644 index 0000000..f2767c9 --- /dev/null +++ b/SOURCES/sos-bz2066181-tigervnc-update-collections.patch @@ -0,0 +1,67 @@ +From 4c92968ce461cdfc6a5d913748b2ce4f148ff4a9 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Thu, 10 Mar 2022 12:31:49 -0500 +Subject: [PATCH] [tigervnc] Update collections for newer versions of TigerVNC + +First, relaxes the file specifications for collection by capturing the +entire `/etc/tigervnc/` directory. + +Second, adds collection of service status and journal output for each +configured vnc server. Collection of `vncserver -list` is kept for +backwards compatibility. + +Finally, add a short docstring for the plugin for --help output. + +Signed-off-by: Jake Hunsaker +--- + sos/report/plugins/tigervnc.py | 28 +++++++++++++++++++++++----- + 1 file changed, 23 insertions(+), 5 deletions(-) + +diff --git a/sos/report/plugins/tigervnc.py b/sos/report/plugins/tigervnc.py +index 1302f6d4..e31aee25 100644 +--- a/sos/report/plugins/tigervnc.py ++++ b/sos/report/plugins/tigervnc.py +@@ -12,17 +12,35 @@ from sos.report.plugins import Plugin, RedHatPlugin + + + class TigerVNC(Plugin, RedHatPlugin): ++ """ ++ This plugin gathers information for VNC servers provided by the tigervnc ++ package. This is explicitly for server-side collections, not clients. ++ ++ By default, this plugin will capture the contents of /etc/tigervnc, which ++ may include usernames. If usernames are sensitive information for end ++ users of sos, consider using the `--clean` option to obfuscate these ++ names. ++ """ + + short_desc = 'TigerVNC server configuration' + plugin_name = 'tigervnc' + packages = ('tigervnc-server',) + + def setup(self): +- self.add_copy_spec([ +- '/etc/tigervnc/vncserver-config-defaults', +- '/etc/tigervnc/vncserver-config-mandatory', +- '/etc/tigervnc/vncserver.users' +- ]) ++ self.add_copy_spec('/etc/tigervnc/') ++ ++ # service names are 'vncserver@$port' where $port is :1,, :2, etc... ++ # however they are not reported via list-unit-files, only list-units ++ vncs = self.exec_cmd( ++ 'systemctl list-units --type=service --no-legend vncserver*' ++ ) ++ if vncs['status'] == 0: ++ for serv in vncs['output'].splitlines(): ++ vnc = serv.split() ++ if not vnc: ++ continue ++ self.add_service_status(vnc[0]) ++ self.add_journal(vnc[0]) + + self.add_cmd_output('vncserver -list') + +-- +2.34.3 + diff --git a/SOURCES/sos-bz2079188-honor-default-plugin-timeout.patch b/SOURCES/sos-bz2079188-honor-default-plugin-timeout.patch new file mode 100644 index 0000000..822565d --- /dev/null +++ b/SOURCES/sos-bz2079188-honor-default-plugin-timeout.patch @@ -0,0 +1,39 @@ +From 7069e99d1c5c443f96a98a7ed6db67fa14683e67 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Thu, 17 Feb 2022 09:14:15 +0100 +Subject: [PATCH] [report] Honor plugins' hardcoded plugin_timeout + +Currently, plugin's plugin_timeout hardcoded default is superseded by +whatever --plugin-timeout value, even when this option is not used and +we eval it to TIMEOUT_DEFAULT. + +In this case of not setting --plugin-timeout either -k plugin.timeout, +honour plugin's plugin_timeout instead. + +Resolves: #2863 +Closes: #2864 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/__init__.py | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py +index cc5cb65b..336b4d22 100644 +--- a/sos/report/plugins/__init__.py ++++ b/sos/report/plugins/__init__.py +@@ -636,7 +636,10 @@ class Plugin(): + if opt_timeout is None: + _timeout = own_timeout + elif opt_timeout is not None and own_timeout == -1: +- _timeout = int(opt_timeout) ++ if opt_timeout == TIMEOUT_DEFAULT: ++ _timeout = default_timeout ++ else: ++ _timeout = int(opt_timeout) + elif opt_timeout is not None and own_timeout > -1: + _timeout = own_timeout + else: +-- +2.34.3 + diff --git a/SOURCES/sos-bz2079490-list-plugins-ignore-options.patch b/SOURCES/sos-bz2079490-list-plugins-ignore-options.patch new file mode 100644 index 0000000..f0bda41 --- /dev/null +++ b/SOURCES/sos-bz2079490-list-plugins-ignore-options.patch @@ -0,0 +1,68 @@ +From f3dc8cd574614572d441f76c02453fd85d0c57e2 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 27 Apr 2022 10:40:55 -0400 +Subject: [PATCH] [report] --list-plugins should report used, not default, + option values + +When using `--list-plugins`, sos should report the values that will be +used in a given command, or with a given config file, not what the +default values are. + +By reporting the set value, users can be sure their configuration or +commandline settings are being honored correctly before executing a +report collection. + +Closes: #2921 + +Signed-off-by: Jake Hunsaker +--- + sos/report/__init__.py | 22 +++++++++++++++------- + 1 file changed, 15 insertions(+), 7 deletions(-) + +diff --git a/sos/report/__init__.py b/sos/report/__init__.py +index 74c7973a..8735c903 100644 +--- a/sos/report/__init__.py ++++ b/sos/report/__init__.py +@@ -868,24 +868,32 @@ class SoSReport(SoSComponent): + _defaults = self.loaded_plugins[0][1].get_default_plugin_opts() + for _opt in _defaults: + opt = _defaults[_opt] +- val = opt.default +- if opt.default == -1: +- val = TIMEOUT_DEFAULT ++ val = opt.value ++ if opt.value == -1: ++ if _opt == 'timeout': ++ val = self.opts.plugin_timeout or TIMEOUT_DEFAULT ++ elif _opt == 'cmd-timeout': ++ val = self.opts.cmd_timeout or TIMEOUT_DEFAULT ++ else: ++ val = TIMEOUT_DEFAULT ++ if opt.name == 'postproc': ++ val = not self.opts.no_postproc + self.ui_log.info(" %-25s %-15s %s" % (opt.name, val, opt.desc)) + self.ui_log.info("") + + self.ui_log.info(_("The following plugin options are available:")) + for opt in self.all_options: + if opt.name in ('timeout', 'postproc', 'cmd-timeout'): +- continue ++ if opt.value == opt.default: ++ continue + # format option value based on its type (int or bool) +- if isinstance(opt.default, bool): +- if opt.default is True: ++ if isinstance(opt.value, bool): ++ if opt.value is True: + tmpopt = "on" + else: + tmpopt = "off" + else: +- tmpopt = opt.default ++ tmpopt = opt.value + + if tmpopt is None: + tmpopt = 0 +-- +2.34.3 + diff --git a/SOURCES/sos-bz2079491-plugopts-valtype-str.patch b/SOURCES/sos-bz2079491-plugopts-valtype-str.patch new file mode 100644 index 0000000..eaf42ab --- /dev/null +++ b/SOURCES/sos-bz2079491-plugopts-valtype-str.patch @@ -0,0 +1,34 @@ +From 9b10abcdd4aaa41e2549438d5bc52ece86dcb21f Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Sat, 7 May 2022 14:23:04 +0200 +Subject: [PATCH] [plugins] Allow 'str' PlugOpt type to accept any value + +For PlugOpt type 'str', we should allow any content including e.g. +numbers, and interpret it as a string. + +Resolves: #2922 +Closes: #2935 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/__init__.py | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py +index d6be42b9..2a42e6b0 100644 +--- a/sos/report/plugins/__init__.py ++++ b/sos/report/plugins/__init__.py +@@ -452,6 +452,10 @@ class PluginOpt(): + return self.__str__() + + def set_value(self, val): ++ # 'str' type accepts any value, incl. numbers ++ if type('') in self.val_type: ++ self.value = str(val) ++ return + if not any([type(val) == _t for _t in self.val_type]): + valid = [] + for t in self.val_type: +-- +2.34.3 + diff --git a/SOURCES/sos-bz2079492-timeouted-exec-cmd-exception.patch b/SOURCES/sos-bz2079492-timeouted-exec-cmd-exception.patch new file mode 100644 index 0000000..dc58a67 --- /dev/null +++ b/SOURCES/sos-bz2079492-timeouted-exec-cmd-exception.patch @@ -0,0 +1,31 @@ +From 5e27b92a8a9f066af4c41ddd0bedc7c69187ff52 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Mon, 2 May 2022 22:13:34 +0200 +Subject: [PATCH] [utilities] Close file only when storing to file + +Call _output.close() only when to_file=true. + +Closes: #2925 + +Signed-off-by: Pavel Moravec +--- + sos/utilities.py | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/sos/utilities.py b/sos/utilities.py +index d2f73d86..1075d1d4 100644 +--- a/sos/utilities.py ++++ b/sos/utilities.py +@@ -212,7 +212,8 @@ def sos_get_command_output(command, timeout=TIMEOUT_DEFAULT, stderr=False, + p.wait(timeout if timeout else None) + except Exception: + p.terminate() +- _output.close() ++ if to_file: ++ _output.close() + # until we separate timeouts from the `timeout` command + # handle per-cmd timeouts via Plugin status checks + return {'status': 124, 'output': reader.get_contents(), +-- +2.34.3 + diff --git a/SOURCES/sos-bz2082914-collect-pacemaker-cluster.patch b/SOURCES/sos-bz2082914-collect-pacemaker-cluster.patch new file mode 100644 index 0000000..d573ea2 --- /dev/null +++ b/SOURCES/sos-bz2082914-collect-pacemaker-cluster.patch @@ -0,0 +1,230 @@ +From 3b84b4ccfa9e4924a5a3829d3810568dfb69bf63 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Fri, 18 Mar 2022 16:25:35 -0400 +Subject: [PATCH 1/2] [pacemaker] Redesign node enumeration logic + +It has been found that `pcs status` output is liable to change, which +ends up breaking our parsing of node lists when using it on newer +versions. + +Instead, first try to parse through `crm_mon` output, which is what `pcs +status` uses under the hood, but as a stable and reliable xml format. + +Failing that, for example if the `--primary` node is not functioning as +part of the cluster, source `/etc/corosync/corosync.conf` instead. + +Related: RHBZ2065805 +Related: RHBZ2065811 + +Signed-off-by: Jake Hunsaker +--- + sos/collector/clusters/pacemaker.py | 110 +++++++++++++++++++--------- + 1 file changed, 76 insertions(+), 34 deletions(-) + +diff --git a/sos/collector/clusters/pacemaker.py b/sos/collector/clusters/pacemaker.py +index 55024314..49d0ce51 100644 +--- a/sos/collector/clusters/pacemaker.py ++++ b/sos/collector/clusters/pacemaker.py +@@ -8,7 +8,11 @@ + # + # See the LICENSE file in the source distribution for further information. + ++import re ++ + from sos.collector.clusters import Cluster ++from setuptools._vendor.packaging import version ++from xml.etree import ElementTree + + + class pacemaker(Cluster): +@@ -18,42 +22,80 @@ class pacemaker(Cluster): + packages = ('pacemaker',) + option_list = [ + ('online', True, 'Collect nodes listed as online'), +- ('offline', True, 'Collect nodes listed as offline') ++ ('offline', True, 'Collect nodes listed as offline'), ++ ('only-corosync', False, 'Only use corosync.conf to enumerate nodes') + ] + + def get_nodes(self): +- self.res = self.exec_primary_cmd('pcs status') +- if self.res['status'] != 0: +- self.log_error('Cluster status could not be determined. Is the ' +- 'cluster running on this node?') +- return [] +- if 'node names do not match' in self.res['output']: +- self.log_warn('Warning: node name mismatch reported. Attempts to ' +- 'connect to some nodes may fail.\n') +- return self.parse_pcs_output() +- +- def parse_pcs_output(self): +- nodes = [] +- if self.get_option('online'): +- nodes += self.get_online_nodes() +- if self.get_option('offline'): +- nodes += self.get_offline_nodes() +- return nodes +- +- def get_online_nodes(self): +- for line in self.res['output'].splitlines(): +- if line.startswith('Online:'): +- nodes = line.split('[')[1].split(']')[0] +- return [n for n in nodes.split(' ') if n] +- +- def get_offline_nodes(self): +- offline = [] +- for line in self.res['output'].splitlines(): +- if line.startswith('Node') and line.endswith('(offline)'): +- offline.append(line.split()[1].replace(':', '')) +- if line.startswith('OFFLINE:'): +- nodes = line.split('[')[1].split(']')[0] +- offline.extend([n for n in nodes.split(' ') if n]) +- return offline ++ self.nodes = [] ++ # try crm_mon first ++ try: ++ if not self.get_option('only-corosync'): ++ try: ++ self.get_nodes_from_crm() ++ except Exception as err: ++ self.log_warn("Falling back to sourcing corosync.conf. " ++ "Could not parse crm_mon output: %s" % err) ++ if not self.nodes: ++ # fallback to corosync.conf, in case the node we're inspecting ++ # is offline from the cluster ++ self.get_nodes_from_corosync() ++ except Exception as err: ++ self.log_error("Could not determine nodes from cluster: %s" % err) ++ ++ _shorts = [n for n in self.nodes if '.' not in n] ++ if _shorts: ++ self.log_warn( ++ "WARNING: Node addresses '%s' may not resolve locally if you " ++ "are not running on a node in the cluster. Try using option " ++ "'-c pacemaker.only-corosync' if these connections fail." ++ % ','.join(_shorts) ++ ) ++ return self.nodes ++ ++ def get_nodes_from_crm(self): ++ """ ++ Try to parse crm_mon output for node list and status. ++ """ ++ xmlopt = '--output-as=xml' ++ # older pacemaker had a different option for xml output ++ _ver = self.exec_primary_cmd('crm_mon --version') ++ if _ver['status'] == 0: ++ cver = _ver['output'].split()[1].split('-')[0] ++ if not version.parse(cver) > version.parse('2.0.3'): ++ xmlopt = '--as-xml' ++ else: ++ return ++ _out = self.exec_primary_cmd( ++ "crm_mon --one-shot --inactive %s" % xmlopt, ++ need_root=True ++ ) ++ if _out['status'] == 0: ++ self.parse_crm_xml(_out['output']) ++ ++ def parse_crm_xml(self, xmlstring): ++ """ ++ Parse the xml output string provided by crm_mon ++ """ ++ _xml = ElementTree.fromstring(xmlstring) ++ nodes = _xml.find('nodes') ++ for node in nodes: ++ _node = node.attrib ++ if self.get_option('online') and _node['online'] == 'true': ++ self.nodes.append(_node['name']) ++ elif self.get_option('offline') and _node['online'] == 'false': ++ self.nodes.append(_node['name']) ++ ++ def get_nodes_from_corosync(self): ++ """ ++ As a fallback measure, read corosync.conf to get the node list. Note ++ that this prevents us from separating online nodes from offline nodes. ++ """ ++ self.log_warn("WARNING: unable to distinguish online nodes from " ++ "offline nodes when sourcing from corosync.conf") ++ cc = self.primary.read_file('/etc/corosync/corosync.conf') ++ nodes = re.findall(r'((\sring0_addr:)(.*))', cc) ++ for node in nodes: ++ self.nodes.append(node[-1].strip()) + + # vim: set et ts=4 sw=4 : +-- +2.34.3 + + +From 6701a7d77ecc998b018b54ecc00f9fd102ae9518 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Mon, 21 Mar 2022 12:05:59 -0400 +Subject: [PATCH 2/2] [clusters] Allow clusters to not add localhost to node + list + +For most of our supported clusters, we end up needing to add the +local host executing `sos collect` to the node list (unless `--no-local` +is used) as that accounts for the primary node that may otherwise be +left off. However, this is not helpful for clusters that may reports +node names as something other than resolveable names. In those cases, +such as with pacemaker, adding the local hostname may result in +duplicate collections. + +Add a toggle to cluster profiles via a new `strict_node_list` class attr +that, if True, will skip this addition. This toggle is default `False` +to preserve existing behavior, and is now enabled for `pacemaker` +specifically. + +Related: RHBZ#2065821 + +Signed-off-by: Jake Hunsaker +--- + sos/collector/__init__.py | 3 ++- + sos/collector/clusters/__init__.py | 4 ++++ + sos/collector/clusters/pacemaker.py | 1 + + 3 files changed, 7 insertions(+), 1 deletion(-) + +diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py +index a8bb0064..d898ca34 100644 +--- a/sos/collector/__init__.py ++++ b/sos/collector/__init__.py +@@ -1073,7 +1073,8 @@ class SoSCollector(SoSComponent): + for node in self.node_list: + if host == node.split('.')[0]: + self.node_list.remove(node) +- self.node_list.append(self.hostname) ++ if not self.cluster.strict_node_list: ++ self.node_list.append(self.hostname) + self.reduce_node_list() + try: + _node_max = len(max(self.node_list, key=len)) +diff --git a/sos/collector/clusters/__init__.py b/sos/collector/clusters/__init__.py +index f3f550ad..f00677b8 100644 +--- a/sos/collector/clusters/__init__.py ++++ b/sos/collector/clusters/__init__.py +@@ -57,6 +57,10 @@ class Cluster(): + sos_plugin_options = {} + sos_preset = '' + cluster_name = None ++ # set this to True if the local host running collect should *not* be ++ # forcibly added to the node list. This can be helpful in situations where ++ # the host's fqdn and the name the cluster uses are different ++ strict_node_list = False + + def __init__(self, commons): + self.primary = None +diff --git a/sos/collector/clusters/pacemaker.py b/sos/collector/clusters/pacemaker.py +index 49d0ce51..bebcb265 100644 +--- a/sos/collector/clusters/pacemaker.py ++++ b/sos/collector/clusters/pacemaker.py +@@ -20,6 +20,7 @@ class pacemaker(Cluster): + cluster_name = 'Pacemaker High Availability Cluster Manager' + sos_plugins = ['pacemaker'] + packages = ('pacemaker',) ++ strict_node_list = True + option_list = [ + ('online', True, 'Collect nodes listed as online'), + ('offline', True, 'Collect nodes listed as offline'), +-- +2.34.3 + diff --git a/SOURCES/sos-bz2095267-ovirt-answer-files-passwords.patch b/SOURCES/sos-bz2095267-ovirt-answer-files-passwords.patch new file mode 100644 index 0000000..67eb6a0 --- /dev/null +++ b/SOURCES/sos-bz2095267-ovirt-answer-files-passwords.patch @@ -0,0 +1,66 @@ +From 5fd872c64c53af37015f366295e0c2418c969757 Mon Sep 17 00:00:00 2001 +From: Yedidyah Bar David +Date: Thu, 26 May 2022 16:43:21 +0300 +Subject: [PATCH] [ovirt] answer files: Filter out all password keys + +Instead of hard-coding specific keys and having to maintain them over +time, replace the values of all keys that have 'password' in their name. +I think this covers all our current and hopefully future keys. It might +add "false positives" - keys that are not passwords but have 'password' +in their name - and I think that's a risk worth taking. + +Sadly, the engine admin password prompt's name is +'OVESETUP_CONFIG_ADMIN_SETUP', which does not include 'password', so has +to be listed specifically. + +A partial list of keys added since the replaced code was written: +- grafana-related stuff +- keycloak-related stuff +- otopi-style answer files + +Signed-off-by: Yedidyah Bar David +Change-Id: I416c6e4078e7c3638493eb271d08d73a0c22b5ba +--- + sos/report/plugins/ovirt.py | 23 +++++++++++++---------- + 1 file changed, 13 insertions(+), 10 deletions(-) + +diff --git a/sos/report/plugins/ovirt.py b/sos/report/plugins/ovirt.py +index 09647bf1..3b1bb29b 100644 +--- a/sos/report/plugins/ovirt.py ++++ b/sos/report/plugins/ovirt.py +@@ -241,19 +241,22 @@ class Ovirt(Plugin, RedHatPlugin): + r'{key}=********'.format(key=key) + ) + +- # Answer files contain passwords +- for key in ( +- 'OVESETUP_CONFIG/adminPassword', +- 'OVESETUP_CONFIG/remoteEngineHostRootPassword', +- 'OVESETUP_DWH_DB/password', +- 'OVESETUP_DB/password', +- 'OVESETUP_REPORTS_CONFIG/adminPassword', +- 'OVESETUP_REPORTS_DB/password', ++ # Answer files contain passwords. ++ # Replace all keys that have 'password' in them, instead of hard-coding ++ # here the list of keys, which changes between versions. ++ # Sadly, the engine admin password prompt name does not contain ++ # 'password'... so neither does the env key. ++ for item in ( ++ 'password', ++ 'OVESETUP_CONFIG_ADMIN_SETUP', + ): + self.do_path_regex_sub( + r'/var/lib/ovirt-engine/setup/answers/.*', +- r'{key}=(.*)'.format(key=key), +- r'{key}=********'.format(key=key) ++ re.compile( ++ r'(?P[^=]*{item}[^=]*)=.*'.format(item=item), ++ flags=re.IGNORECASE ++ ), ++ r'\g=********' + ) + + # aaa profiles contain passwords +-- +2.34.3 + diff --git a/SOURCES/sos-bz2097674-openshift-ovn-disabled.patch b/SOURCES/sos-bz2097674-openshift-ovn-disabled.patch new file mode 100644 index 0000000..29241ba --- /dev/null +++ b/SOURCES/sos-bz2097674-openshift-ovn-disabled.patch @@ -0,0 +1,73 @@ +From c2e66fa4dae51f03c7310ba5278897ddecac1aad Mon Sep 17 00:00:00 2001 +From: Nadia Pinaeva +Date: Thu, 2 Jun 2022 15:43:09 +0200 +Subject: [PATCH] crio: switch from parsing output in table format to json + +Signed-off-by: Nadia Pinaeva +--- + sos/policies/runtimes/crio.py | 30 ++++++++++++++++++++---------- + 1 file changed, 20 insertions(+), 10 deletions(-) + +diff --git a/sos/policies/runtimes/crio.py b/sos/policies/runtimes/crio.py +index 55082d07..4cae1ecc 100644 +--- a/sos/policies/runtimes/crio.py ++++ b/sos/policies/runtimes/crio.py +@@ -7,6 +7,7 @@ + # version 2 of the GNU General Public License. + # + # See the LICENSE file in the source distribution for further information. ++import json + + from sos.policies.runtimes import ContainerRuntime + from sos.utilities import sos_get_command_output +@@ -29,14 +30,15 @@ class CrioContainerRuntime(ContainerRuntime): + :type get_all: ``bool`` + """ + containers = [] +- _cmd = "%s ps %s" % (self.binary, '-a' if get_all else '') ++ _cmd = "%s ps %s -o json" % (self.binary, '-a' if get_all else '') + if self.active: + out = sos_get_command_output(_cmd, chroot=self.policy.sysroot) +- if out['status'] == 0: +- for ent in out['output'].splitlines()[1:]: +- ent = ent.split() ++ if out["status"] == 0: ++ out_json = json.loads(out["output"]) ++ for container in out_json["containers"]: + # takes the form (container_id, container_name) +- containers.append((ent[0], ent[-3])) ++ containers.append( ++ (container["id"], container["metadata"]["name"])) + return containers + + def get_images(self): +@@ -47,13 +49,21 @@ class CrioContainerRuntime(ContainerRuntime): + """ + images = [] + if self.active: +- out = sos_get_command_output("%s images" % self.binary, ++ out = sos_get_command_output("%s images -o json" % self.binary, + chroot=self.policy.sysroot) + if out['status'] == 0: +- for ent in out['output'].splitlines(): +- ent = ent.split() +- # takes the form (image_name, image_id) +- images.append((ent[0] + ':' + ent[1], ent[2])) ++ out_json = json.loads(out["output"]) ++ for image in out_json["images"]: ++ # takes the form (repository:tag, image_id) ++ if len(image["repoTags"]) > 0: ++ for repo_tag in image["repoTags"]: ++ images.append((repo_tag, image["id"])) ++ else: ++ if len(image["repoDigests"]) == 0: ++ image_name = "" ++ else: ++ image_name = image["repoDigests"][0].split("@")[0] ++ images.append((image_name + ":", image["id"])) + return images + + def fmt_container_cmd(self, container, cmd, quotecmd): +-- +2.34.3 + diff --git a/SPECS/sos.spec b/SPECS/sos.spec index e468c18..43d79ba 100644 --- a/SPECS/sos.spec +++ b/SPECS/sos.spec @@ -4,8 +4,8 @@ Summary: A set of tools to gather troubleshooting information from a system Name: sos -Version: 4.2 -Release: 15%{?dist} +Version: 4.3 +Release: 2%{?dist} Group: Applications/System Source0: https://github.com/sosreport/sos/archive/%{version}/sos-%{version}.tar.gz Source1: sos-audit-%{auditversion}.tgz @@ -24,29 +24,17 @@ Conflicts: vdsm < 4.40 Obsoletes: sos-collector <= 1.9 Recommends: python3-pexpect Recommends: python3-requests -Patch1: sos-bz1869561-cpuX-individual-sizelimits.patch -Patch2: sos-bz2011533-unpackaged-recursive-symlink.patch -Patch3: sos-bz2011534-opacapture-under-allow-system-changes.patch -Patch4: sos-bz2011535-kernel-psi.patch -Patch5: sos-bz2011538-iptables-save-under-nf_tables-kmod.patch -Patch6: sos-bz2011537-estimate-only-option.patch -Patch7: sos-bz2011536-iptables-based-on-ntf.patch -Patch8: sos-bz2011507-foreman-puma-status.patch -Patch9: sos-bz2012858-dryrun-uncaught-exception.patch -Patch10: sos-bz2019697-openvswitch-offline-analysis.patch -Patch11: sos-bz2012859-plugin-timeout-unhandled-exception.patch -Patch12: sos-bz2023481-plugin-timeouts-proper-handling.patch -Patch13: sos-bz2020778-filter-namespace-per-pattern.patch -Patch14: sos-bz2024893-cleaner-hostnames-improvements.patch -Patch15: sos-bz2025611-RHTS-api-change.patch -Patch16: sos-bz2034001-nvidia-GPU-info.patch -Patch17: sos-bz2031777-rhui-logs.patch -Patch18: sos-bz2037350-ocp-backports.patch -Patch19: sos-bz2043104-foreman-tasks-msgpack.patch -Patch20: sos-bz2041855-virsh-in-foreground.patch -Patch21: sos-bz2043488-ovn-proper-package-enablement.patch -Patch22: sos-bz2054883-plugopt-logging-effective-opts.patch -Patch23: sos-bz2055548-honour-plugins-timeout-hardcoded.patch +Patch1: sos-bz2055003-rebase-sos-add-sos-help.patch +Patch2: sos-bz2095267-ovirt-answer-files-passwords.patch +Patch3: sos-bz2079491-plugopts-valtype-str.patch +Patch4: sos-bz2066181-tigervnc-update-collections.patch +Patch5: sos-bz2082914-collect-pacemaker-cluster.patch +Patch6: sos-bz2079188-honor-default-plugin-timeout.patch +Patch7: sos-bz2079490-list-plugins-ignore-options.patch +Patch8: sos-bz2079492-timeouted-exec-cmd-exception.patch +Patch9: sos-bz2065563-ocp-backports.patch +Patch10: sos-bz2097674-openshift-ovn-disabled.patch + %description Sos is a set of tools that gathers information about system @@ -67,19 +55,6 @@ support technicians and developers. %patch8 -p1 %patch9 -p1 %patch10 -p1 -%patch11 -p1 -%patch12 -p1 -%patch13 -p1 -%patch14 -p1 -%patch15 -p1 -%patch16 -p1 -%patch17 -p1 -%patch18 -p1 -%patch19 -p1 -%patch20 -p1 -%patch21 -p1 -%patch22 -p1 -%patch23 -p1 %build %py3_build @@ -147,6 +122,44 @@ of the system. Currently storage and filesystem commands are audited. %changelog +* Thu Jun 16 2022 Pavel Moravec = 4.3-2 +- [ocp, openshift] Re-align API collection options and rename + Resolves: bz2065563 +- [utilities] Close file only when storing to file + Resolves: bz2079492 +- [report] --list-plugins should report used, not default, + Resolves: bz2079490 +- [report] Honor plugins' hardcoded plugin_timeout + Resolves: bz2079188 +- crio: switch from parsing output in table format to json + Resolves: bz2097674 +- [pacemaker] Redesign node enumeration logic + Resolves: bz2082914 +- [tigervnc] Update collections for newer versions of TigerVNC + Resolves: bz2066181 +- [plugins] Allow 'str' PlugOpt type to accept any value + Resolves: bz2079491 +- [ovirt] answer files: Filter out all password keys + Resolves: bz2095267 + +* Thu Mar 24 2022 Pavel Moravec = 4.3-1 +- Rebase on upstream 4.3 + Resolves: 2055003 +- [sapnw] Fix IndexError exception + Resolves: 2065551 +- [subscription_manager] collect syspurpose data via sub-man + Resolves: 2002333 +- [Plugin, utilities] Allow writing command output directly to disk + Resolves: 2065564 +- [Ceph] Add support for containerized Ceph setup + Resolves: 2065562 +- [unbound] Add new plugin for Unbound DNS resolver + Resolves: 2065560 +- [discovery] Add new discovery plugin + Resolves: 2065558 +- [system] Collect glibc tuning decisions + Resolves: 2032913 + * Wed Feb 23 2022 Pavel Moravec = 4.2-15 - [sosnode] Handle downstream versioning for runtime option Resolves: bz2037350