Blame SOURCES/sos-bz1873185-estimate-only-option.patch

900e1b
From 5b245b1e449c6a05d09034bcb8290bffded79327 Mon Sep 17 00:00:00 2001
900e1b
From: Pavel Moravec <pmoravec@redhat.com>
900e1b
Date: Wed, 8 Sep 2021 17:04:58 +0200
900e1b
Subject: [PATCH] [report] Implement --estimate-only
900e1b
900e1b
Add report option --estimate-only to estimate disk space requirements
900e1b
when running a sos report.
900e1b
900e1b
Resolves: #2673
900e1b
900e1b
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
900e1b
---
900e1b
 man/en/sos-report.1    | 13 +++++++-
900e1b
 sos/report/__init__.py | 74 ++++++++++++++++++++++++++++++++++++++++--
900e1b
 2 files changed, 84 insertions(+), 3 deletions(-)
900e1b
900e1b
diff --git a/man/en/sos-report.1 b/man/en/sos-report.1
900e1b
index 36b337df..e8efc8f8 100644
900e1b
--- a/man/en/sos-report.1
900e1b
+++ b/man/en/sos-report.1
900e1b
@@ -14,7 +14,7 @@ sos report \- Collect and package diagnostic and support data
900e1b
           [--preset preset] [--add-preset add_preset]\fR
900e1b
           [--del-preset del_preset] [--desc description]\fR
900e1b
           [--batch] [--build] [--debug] [--dry-run]\fR
900e1b
-          [--label label] [--case-id id]\fR
900e1b
+          [--estimate-only] [--label label] [--case-id id]\fR
900e1b
           [--threads threads]\fR
900e1b
           [--plugin-timeout TIMEOUT]\fR
900e1b
           [--cmd-timeout TIMEOUT]\fR
900e1b
@@ -317,6 +317,17 @@ output, or string data from the system. The resulting logs may be used
900e1b
 to understand the actions that sos would have taken without the dry run
900e1b
 option.
900e1b
 .TP
900e1b
+.B \--estimate-only
900e1b
+Estimate disk space requirements when running sos report. This can be valuable
900e1b
+to prevent sosreport working dir to consume all free disk space. No plugin data
900e1b
+is available at the end.
900e1b
+
900e1b
+Plugins will be collected sequentially, size of collected files and commands outputs
900e1b
+will be calculated and the plugin files will be immediatelly deleted prior execution
900e1b
+of the next plugin. This still can consume whole free disk space, though. Please note,
900e1b
+size estimations may not be accurate for highly utilized systems due to changes between
900e1b
+an estimate and a real execution.
900e1b
+.TP
900e1b
 .B \--upload
900e1b
 If specified, attempt to upload the resulting archive to a vendor defined location.
900e1b
 
900e1b
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
900e1b
index 82484f1d..b033f621 100644
900e1b
--- a/sos/report/__init__.py
900e1b
+++ b/sos/report/__init__.py
900e1b
@@ -86,6 +86,7 @@ class SoSReport(SoSComponent):
900e1b
         'desc': '',
900e1b
         'domains': [],
900e1b
         'dry_run': False,
900e1b
+        'estimate_only': False,
900e1b
         'experimental': False,
900e1b
         'enable_plugins': [],
900e1b
         'keywords': [],
900e1b
@@ -137,6 +138,7 @@ class SoSReport(SoSComponent):
900e1b
         self._args = args
900e1b
         self.sysroot = "/"
900e1b
         self.preset = None
900e1b
+        self.estimated_plugsizes = {}
900e1b
 
900e1b
         self.print_header()
900e1b
         self._set_debug()
900e1b
@@ -223,6 +225,11 @@ class SoSReport(SoSComponent):
900e1b
                                 help="Description for a new preset",)
900e1b
         report_grp.add_argument("--dry-run", action="store_true",
900e1b
                                 help="Run plugins but do not collect data")
900e1b
+        report_grp.add_argument("--estimate-only", action="store_true",
900e1b
+                                help="Approximate disk space requirements for "
900e1b
+                                     "a real sos run; disables --clean and "
900e1b
+                                     "--collect, sets --threads=1 and "
900e1b
+                                     "--no-postproc")
900e1b
         report_grp.add_argument("--experimental", action="store_true",
900e1b
                                 dest="experimental", default=False,
900e1b
                                 help="enable experimental plugins")
900e1b
@@ -700,6 +700,33 @@ class SoSReport(SoSComponent):
900e1b
                 self.all_options.append((plugin, plugin_name, optname,
900e1b
                                          optparm))
900e1b
 
900e1b
+    def _set_estimate_only(self):
900e1b
+        # set estimate-only mode by enforcing some options settings
900e1b
+        # and return a corresponding log messages string
900e1b
+        msg = "\nEstimate-only mode enabled"
900e1b
+        ext_msg = []
900e1b
+        if self.opts.threads > 1:
900e1b
+            ext_msg += ["--threads=%s overriden to 1" % self.opts.threads, ]
900e1b
+            self.opts.threads = 1
900e1b
+        if not self.opts.build:
900e1b
+            ext_msg += ["--build enabled", ]
900e1b
+            self.opts.build = True
900e1b
+        if not self.opts.no_postproc:
900e1b
+            ext_msg += ["--no-postproc enabled", ]
900e1b
+            self.opts.no_postproc = True
900e1b
+        if self.opts.clean:
900e1b
+            ext_msg += ["--clean disabled", ]
900e1b
+            self.opts.clean = False
900e1b
+        if self.opts.upload:
900e1b
+            ext_msg += ["--upload* options disabled", ]
900e1b
+            self.opts.upload = False
900e1b
+        if ext_msg:
900e1b
+            msg += ", which overrides some options:\n  " + "\n  ".join(ext_msg)
900e1b
+        else:
900e1b
+            msg += "."
900e1b
+        msg += "\n\n"
900e1b
+        return msg
900e1b
+
900e1b
     def _report_profiles_and_plugins(self):
900e1b
         self.ui_log.info("")
900e1b
         if len(self.loaded_plugins):
900e1b
@@ -875,10 +909,12 @@ class SoSReport(SoSComponent):
900e1b
         return True
900e1b
 
900e1b
     def batch(self):
900e1b
+        msg = self.policy.get_msg()
900e1b
+        if self.opts.estimate_only:
900e1b
+            msg += self._set_estimate_only()
900e1b
         if self.opts.batch:
900e1b
-            self.ui_log.info(self.policy.get_msg())
900e1b
+            self.ui_log.info(msg)
900e1b
         else:
900e1b
-            msg = self.policy.get_msg()
900e1b
             msg += _("Press ENTER to continue, or CTRL-C to quit.\n")
900e1b
             try:
900e1b
                 input(msg)
900e1b
@@ -1011,6 +1047,22 @@ class SoSReport(SoSComponent):
900e1b
                 self.running_plugs.remove(plugin[1])
900e1b
                 self.loaded_plugins[plugin[0]-1][1].set_timeout_hit()
900e1b
                 pool._threads.clear()
900e1b
+        if self.opts.estimate_only:
900e1b
+            from pathlib import Path
900e1b
+            tmpdir_path = Path(self.archive.get_tmp_dir())
900e1b
+            self.estimated_plugsizes[plugin[1]] = sum(
900e1b
+                    [f.stat().st_size for f in tmpdir_path.glob('**/*')
900e1b
+                     if (os.path.isfile(f) and not os.path.islink(f))])
900e1b
+            # remove whole tmp_dir content - including "sos_commands" and
900e1b
+            # similar dirs that will be re-created on demand by next plugin
900e1b
+            # if needed; it is less error-prone approach than skipping
900e1b
+            # deletion of some dirs but deleting their content
900e1b
+            for f in os.listdir(self.archive.get_tmp_dir()):
900e1b
+                f = os.path.join(self.archive.get_tmp_dir(), f)
900e1b
+                if os.path.isdir(f):
900e1b
+                    rmtree(f)
900e1b
+                else:
900e1b
+                    os.unlink(f)
900e1b
         return True
900e1b
 
900e1b
     def collect_plugin(self, plugin):
900e1b
@@ -1330,6 +1382,24 @@ class SoSReport(SoSComponent):
900e1b
             self.policy.display_results(archive, directory, checksum,
900e1b
                                         map_file=map_file)
900e1b
 
900e1b
+        if self.opts.estimate_only:
900e1b
+            from sos.utilities import get_human_readable
900e1b
+            _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
900e1b
+            self.ui_log.info("Estimated disk space requirement for whole "
900e1b
+                             "uncompressed sos report directory: %s" % _sum)
900e1b
+            bigplugins = sorted(self.estimated_plugsizes.items(),
900e1b
+                                key=lambda x: x[1], reverse=True)[:3]
900e1b
+            bp_out = ",  ".join("%s: %s" %
900e1b
+                                (p, get_human_readable(v, precision=0))
900e1b
+                                for p, v in bigplugins)
900e1b
+            self.ui_log.info("Three biggest plugins:  %s" % bp_out)
900e1b
+            self.ui_log.info("")
900e1b
+            self.ui_log.info("Please note the estimation is relevant to the "
900e1b
+                             "current options.")
900e1b
+            self.ui_log.info("Be aware that the real disk space requirements "
900e1b
+                             "might be different.")
900e1b
+            self.ui_log.info("")
900e1b
+
900e1b
         if self.opts.upload or self.opts.upload_url:
900e1b
             if not self.opts.build:
900e1b
                 try:
900e1b
-- 
900e1b
2.31.1
900e1b
900e1b
From 7ae47e6c0717c0b56c3368008dd99a87f7f436d5 Mon Sep 17 00:00:00 2001
900e1b
From: Pavel Moravec <pmoravec@redhat.com>
900e1b
Date: Wed, 13 Oct 2021 20:21:16 +0200
900e1b
Subject: [PATCH] [report] Count with sos_logs and sos_reports in
900e1b
 --estimate-only
900e1b
900e1b
Currently, we estimate just plugins' disk space and ignore sos_logs
900e1b
or sos_reports directories - although they can occupy nontrivial disk
900e1b
space as well.
900e1b
900e1b
Resolves: #2723
900e1b
900e1b
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
900e1b
---
900e1b
 sos/report/__init__.py | 8 ++++++++
900e1b
 1 file changed, 8 insertions(+)
900e1b
900e1b
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
900e1b
index e35c7e8d..7feb31ee 100644
900e1b
--- a/sos/report/__init__.py
900e1b
+++ b/sos/report/__init__.py
900e1b
@@ -1380,6 +1380,14 @@ class SoSReport(SoSComponent):
900e1b
 
900e1b
         if self.opts.estimate_only:
900e1b
             from sos.utilities import get_human_readable
900e1b
+            from pathlib import Path
900e1b
+            # add sos_logs, sos_reports dirs, etc., basically everything
900e1b
+            # that remained in self.tmpdir after plugins' contents removal
900e1b
+            # that still will be moved to the sos report final directory path
900e1b
+            tmpdir_path = Path(self.tmpdir)
900e1b
+            self.estimated_plugsizes['sos_logs_reports'] = sum(
900e1b
+                    [f.stat().st_size for f in tmpdir_path.glob('**/*')])
900e1b
+
900e1b
             _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
900e1b
             self.ui_log.info("Estimated disk space requirement for whole "
900e1b
                              "uncompressed sos report directory: %s" % _sum)
900e1b
-- 
900e1b
2.31.1
900e1b
900e1b
From 4293f3317505661e8f32ba94ad87310996fa1626 Mon Sep 17 00:00:00 2001
900e1b
From: Eric Desrochers <eric.desrochers@canonical.com>
900e1b
Date: Tue, 19 Oct 2021 12:18:40 -0400
900e1b
Subject: [PATCH] [report] check for symlink before rmtree when opt
900e1b
 estimate-only is use
900e1b
900e1b
Check if the dir is also symlink before performing rmtree()
900e1b
method so that unlink() method can be used instead.
900e1b
900e1b
Traceback (most recent call last):
900e1b
  File "./bin/sos", line 22, in <module>
900e1b
    sos.execute()
900e1b
  File "/tmp/sos/sos/__init__.py", line 186, in execute
900e1b
    self._component.execute()
900e1b
OSError: Cannot call rmtree on a symbolic link
900e1b
900e1b
Closes: #2727
900e1b
900e1b
Signed-off-by: Eric Desrochers <eric.desrochers@canonical.com>
900e1b
---
900e1b
 sos/report/__init__.py | 2 +-
900e1b
 1 file changed, 1 insertion(+), 1 deletion(-)
900e1b
900e1b
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
900e1b
index 7feb31ee..1b5bc97d 100644
900e1b
--- a/sos/report/__init__.py
900e1b
+++ b/sos/report/__init__.py
900e1b
@@ -1059,7 +1059,7 @@ class SoSReport(SoSComponent):
900e1b
             # deletion of some dirs but deleting their content
900e1b
             for f in os.listdir(self.archive.get_tmp_dir()):
900e1b
                 f = os.path.join(self.archive.get_tmp_dir(), f)
900e1b
-                if os.path.isdir(f):
900e1b
+                if os.path.isdir(f) and not os.path.islink(f):
900e1b
                     rmtree(f)
900e1b
                 else:
900e1b
                     os.unlink(f)
900e1b
-- 
900e1b
2.31.1
900e1b
6ebc5b
From 589d47c93257b55bc796ef6ac25b88c974ee3d72 Mon Sep 17 00:00:00 2001
6ebc5b
From: Pavel Moravec <pmoravec@redhat.com>
6ebc5b
Date: Mon, 8 Nov 2021 16:38:24 +0100
6ebc5b
Subject: [PATCH] [report] Calculate sizes of dirs, symlinks and manifest in
6ebc5b
 estimate mode
6ebc5b
6ebc5b
Enhance --estimate-mode to calculate sizes of also:
6ebc5b
- symlinks
6ebc5b
- directories themselves
6ebc5b
- manifest.json file
6ebc5b
6ebc5b
Use os.lstat() method instead of os.stat() to properly calculate the
6ebc5b
sizes (and not destinations of symlinks, e.g.).
6ebc5b
6ebc5b
Print five biggest plugins instead of three as sos logs and reports do
6ebc5b
stand as one "plugin" in the list, often.
6ebc5b
6ebc5b
Resolves: #2752
6ebc5b
6ebc5b
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
6ebc5b
---
6ebc5b
 sos/report/__init__.py | 56 +++++++++++++++++++++---------------------
6ebc5b
 1 file changed, 28 insertions(+), 28 deletions(-)
6ebc5b
6ebc5b
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
6ebc5b
index 10952566..a4c92acc 100644
6ebc5b
--- a/sos/report/__init__.py
6ebc5b
+++ b/sos/report/__init__.py
6ebc5b
@@ -1050,8 +1050,7 @@ class SoSReport(SoSComponent):
6ebc5b
             from pathlib import Path
6ebc5b
             tmpdir_path = Path(self.archive.get_tmp_dir())
6ebc5b
             self.estimated_plugsizes[plugin[1]] = sum(
6ebc5b
-                    [f.stat().st_size for f in tmpdir_path.glob('**/*')
6ebc5b
-                     if (os.path.isfile(f) and not os.path.islink(f))])
6ebc5b
+                    [f.lstat().st_size for f in tmpdir_path.glob('**/*')])
6ebc5b
             # remove whole tmp_dir content - including "sos_commands" and
6ebc5b
             # similar dirs that will be re-created on demand by next plugin
6ebc5b
             # if needed; it is less error-prone approach than skipping
6ebc5b
@@ -1273,6 +1272,33 @@ class SoSReport(SoSComponent):
6ebc5b
                 short_name='manifest.json'
6ebc5b
             )
6ebc5b
 
6ebc5b
+        # print results in estimate mode (to include also just added manifest)
6ebc5b
+        if self.opts.estimate_only:
6ebc5b
+            from sos.utilities import get_human_readable
6ebc5b
+            from pathlib import Path
6ebc5b
+            # add sos_logs, sos_reports dirs, etc., basically everything
6ebc5b
+            # that remained in self.tmpdir after plugins' contents removal
6ebc5b
+            # that still will be moved to the sos report final directory path
6ebc5b
+            tmpdir_path = Path(self.tmpdir)
6ebc5b
+            self.estimated_plugsizes['sos_logs_reports'] = sum(
6ebc5b
+                    [f.lstat().st_size for f in tmpdir_path.glob('**/*')])
6ebc5b
+
6ebc5b
+            _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
6ebc5b
+            self.ui_log.info("Estimated disk space requirement for whole "
6ebc5b
+                             "uncompressed sos report directory: %s" % _sum)
6ebc5b
+            bigplugins = sorted(self.estimated_plugsizes.items(),
6ebc5b
+                                key=lambda x: x[1], reverse=True)[:5]
6ebc5b
+            bp_out = ",  ".join("%s: %s" %
6ebc5b
+                                (p, get_human_readable(v, precision=0))
6ebc5b
+                                for p, v in bigplugins)
6ebc5b
+            self.ui_log.info("Five biggest plugins:  %s" % bp_out)
6ebc5b
+            self.ui_log.info("")
6ebc5b
+            self.ui_log.info("Please note the estimation is relevant to the "
6ebc5b
+                             "current options.")
6ebc5b
+            self.ui_log.info("Be aware that the real disk space requirements "
6ebc5b
+                             "might be different.")
6ebc5b
+            self.ui_log.info("")
6ebc5b
+
6ebc5b
         # package up and compress the results
6ebc5b
         if not self.opts.build:
6ebc5b
             old_umask = os.umask(0o077)
6ebc5b
@@ -1377,32 +1403,6 @@ class SoSReport(SoSComponent):
6ebc5b
             self.policy.display_results(archive, directory, checksum,
6ebc5b
                                         map_file=map_file)
6ebc5b
 
6ebc5b
-        if self.opts.estimate_only:
6ebc5b
-            from sos.utilities import get_human_readable
6ebc5b
-            from pathlib import Path
6ebc5b
-            # add sos_logs, sos_reports dirs, etc., basically everything
6ebc5b
-            # that remained in self.tmpdir after plugins' contents removal
6ebc5b
-            # that still will be moved to the sos report final directory path
6ebc5b
-            tmpdir_path = Path(self.tmpdir)
6ebc5b
-            self.estimated_plugsizes['sos_logs_reports'] = sum(
6ebc5b
-                    [f.stat().st_size for f in tmpdir_path.glob('**/*')])
6ebc5b
-
6ebc5b
-            _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
6ebc5b
-            self.ui_log.info("Estimated disk space requirement for whole "
6ebc5b
-                             "uncompressed sos report directory: %s" % _sum)
6ebc5b
-            bigplugins = sorted(self.estimated_plugsizes.items(),
6ebc5b
-                                key=lambda x: x[1], reverse=True)[:3]
6ebc5b
-            bp_out = ",  ".join("%s: %s" %
6ebc5b
-                                (p, get_human_readable(v, precision=0))
6ebc5b
-                                for p, v in bigplugins)
6ebc5b
-            self.ui_log.info("Three biggest plugins:  %s" % bp_out)
6ebc5b
-            self.ui_log.info("")
6ebc5b
-            self.ui_log.info("Please note the estimation is relevant to the "
6ebc5b
-                             "current options.")
6ebc5b
-            self.ui_log.info("Be aware that the real disk space requirements "
6ebc5b
-                             "might be different.")
6ebc5b
-            self.ui_log.info("")
6ebc5b
-
6ebc5b
         if self.opts.upload or self.opts.upload_url:
6ebc5b
             if not self.opts.build:
6ebc5b
                 try:
6ebc5b
-- 
6ebc5b
2.31.1
6ebc5b
15bbc2
From c6a5bbb8d75aadd5c7f76d3f469929aba2cf8060 Mon Sep 17 00:00:00 2001
15bbc2
From: Pavel Moravec <pmoravec@redhat.com>
15bbc2
Date: Wed, 5 Jan 2022 10:33:58 +0100
15bbc2
Subject: [PATCH] [report] Provide better warning about estimate-mode
15bbc2
15bbc2
As --estimate-only calculates disk usage based on `stat` data that
15bbc2
differs from outputs of other commands like `du`, enhance the warning
15bbc2
about reliability of the calculated estimation.
15bbc2
15bbc2
Also add a rule-of-thumb recommendation of real disk space requirements.
15bbc2
15bbc2
Resolves: #2815
15bbc2
15bbc2
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
15bbc2
---
15bbc2
 man/en/sos-report.1    | 10 +++++++---
15bbc2
 sos/report/__init__.py |  3 ++-
15bbc2
 2 files changed, 9 insertions(+), 4 deletions(-)
15bbc2
15bbc2
diff --git a/man/en/sos-report.1 b/man/en/sos-report.1
15bbc2
index 464a77e54..e34773986 100644
15bbc2
--- a/man/en/sos-report.1
15bbc2
+++ b/man/en/sos-report.1
15bbc2
@@ -343,9 +343,13 @@ is available at the end.
15bbc2
 
15bbc2
 Plugins will be collected sequentially, size of collected files and commands outputs
15bbc2
 will be calculated and the plugin files will be immediatelly deleted prior execution
15bbc2
-of the next plugin. This still can consume whole free disk space, though. Please note,
15bbc2
-size estimations may not be accurate for highly utilized systems due to changes between
15bbc2
-an estimate and a real execution.
15bbc2
+of the next plugin. This still can consume whole free disk space, though.
15bbc2
+
15bbc2
+Please note, size estimations may not be accurate for highly utilized systems due to
15bbc2
+changes between an estimate and a real execution. Also some difference between
15bbc2
+estimation (using `stat` command) and other commands used (i.e. `du`).
15bbc2
+
15bbc2
+A rule of thumb is to reserve at least double the estimation.
15bbc2
 .TP
15bbc2
 .B \--upload
15bbc2
 If specified, attempt to upload the resulting archive to a vendor defined location.
15bbc2
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
15bbc2
index ef61fb344..e0617b45e 100644
15bbc2
--- a/sos/report/__init__.py
15bbc2
+++ b/sos/report/__init__.py
15bbc2
@@ -1330,7 +1330,8 @@ def final_work(self):
15bbc2
             self.ui_log.info("Please note the estimation is relevant to the "
15bbc2
                              "current options.")
15bbc2
             self.ui_log.info("Be aware that the real disk space requirements "
15bbc2
-                             "might be different.")
15bbc2
+                             "might be different. A rule of thumb is to "
15bbc2
+                             "reserve at least double the estimation.")
15bbc2
             self.ui_log.info("")
15bbc2
 
15bbc2
         # package up and compress the results
203ee3
From f22efe044f1f0565b57d6aeca2081a5227e0312c Mon Sep 17 00:00:00 2001
203ee3
From: Jake Hunsaker <jhunsake@redhat.com>
203ee3
Date: Mon, 14 Feb 2022 09:37:30 -0500
203ee3
Subject: [PATCH] [utilities] Don't try to chroot to /
203ee3
203ee3
With the recent fix for sysroot being `None` to always being (correctly)
203ee3
`/`, we should guard against situations where `sos_get_command_output()`
203ee3
would now try to chroot to `/` before running any command. Incidentally,
203ee3
this would also cause our unittests to fail if they were run by a
203ee3
non-root user.
203ee3
203ee3
Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>
203ee3
---
203ee3
 sos/utilities.py | 2 +-
203ee3
 1 file changed, 1 insertion(+), 1 deletion(-)
203ee3
203ee3
diff --git a/sos/utilities.py b/sos/utilities.py
203ee3
index 6b13415b..d782123a 100644
203ee3
--- a/sos/utilities.py
203ee3
+++ b/sos/utilities.py
203ee3
@@ -120,7 +120,7 @@ def sos_get_command_output(command, timeout=TIMEOUT_DEFAULT, stderr=False,
203ee3
     # closure are caught in the parent (chroot and chdir are bound from
203ee3
     # the enclosing scope).
203ee3
     def _child_prep_fn():
203ee3
-        if (chroot):
203ee3
+        if chroot and chroot != '/':
203ee3
             os.chroot(chroot)
203ee3
         if (chdir):
203ee3
             os.chdir(chdir)
203ee3
-- 
203ee3
2.34.1
203ee3
From 3d064102f8ca6662fd9602512e1cb05cf8746dfd Mon Sep 17 00:00:00 2001
203ee3
From: Jake Hunsaker <jhunsake@redhat.com>
203ee3
Date: Mon, 27 Sep 2021 19:01:16 -0400
203ee3
Subject: [PATCH] [Systemd, Policy] Correct InitSystem chrooting when chroot is
203ee3
 needed
203ee3
203ee3
This commit resolves a situation in which `sos` is being run in a
203ee3
container but the `SystemdInit` InitSystem would not properly load
203ee3
information from the host, thus causing the `Plugin.is_service*()`
203ee3
methods to erroneously fail or return `False`.
203ee3
203ee3
Fix this scenario by pulling the `_container_init()` and related logic
203ee3
to check for a containerized host sysroot out of the Red Hat specific
203ee3
policy and into the base `LinuxPolicy` class so that the init system can
203ee3
be initialized with the correct sysroot, which is now used to chroot the
203ee3
calls to the relevant `systemctl` commands.
203ee3
203ee3
For now, this does impose the use of looking for the `container` env var
203ee3
(automatically set by docker, podman, and crio regardless of
203ee3
distribution) and the use of the `HOST` env var to read where the host's
203ee3
`/` filesystem is mounted within the container. If desired in the
203ee3
future, this can be changed to allow policy-specific overrides. For now
203ee3
however, this extends host collection via an sos container for all
203ee3
distributions currently shipping sos.
203ee3
203ee3
Note that this issue only affected the `InitSystem` abstraction for
203ee3
loading information about local services, and did not affect init system
203ee3
related commands called by plugins as part of those collections.
203ee3
203ee3
Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>
203ee3
---
203ee3
 sos/policies/distros/__init__.py      | 28 ++++++++++++++++++++++++++-
203ee3
 sos/policies/distros/redhat.py        | 27 +-------------------------
203ee3
 sos/policies/init_systems/__init__.py | 13 +++++++++++--
203ee3
 sos/policies/init_systems/systemd.py  |  7 ++++---
203ee3
 4 files changed, 43 insertions(+), 32 deletions(-)
203ee3
203ee3
diff --git a/sos/policies/distros/__init__.py b/sos/policies/distros/__init__.py
203ee3
index f5b9fd5b01..c33a356a75 100644
203ee3
--- a/sos/policies/distros/__init__.py
203ee3
+++ b/sos/policies/distros/__init__.py
203ee3
@@ -29,6 +29,10 @@
203ee3
 except ImportError:
203ee3
     REQUESTS_LOADED = False
203ee3
 
203ee3
+# Container environment variables for detecting if we're in a container
203ee3
+ENV_CONTAINER = 'container'
203ee3
+ENV_HOST_SYSROOT = 'HOST'
203ee3
+
203ee3
 
203ee3
 class LinuxPolicy(Policy):
203ee3
     """This policy is meant to be an abc class that provides common
203ee3
@@ -69,10 +73,17 @@ def __init__(self, sysroot=None, init=None, probe_runtime=True):
203ee3
                                           probe_runtime=probe_runtime)
203ee3
         self.init_kernel_modules()
203ee3
 
203ee3
+        # need to set _host_sysroot before PackageManager()
203ee3
+        if sysroot:
203ee3
+            self._container_init()
203ee3
+            self._host_sysroot = sysroot
203ee3
+        else:
203ee3
+            sysroot = self._container_init()
203ee3
+
203ee3
         if init is not None:
203ee3
             self.init_system = init
203ee3
         elif os.path.isdir("/run/systemd/system/"):
203ee3
-            self.init_system = SystemdInit()
203ee3
+            self.init_system = SystemdInit(chroot=sysroot)
203ee3
         else:
203ee3
             self.init_system = InitSystem()
203ee3
 
203ee3
@@ -130,6 +141,21 @@ def get_local_name(self):
203ee3
     def sanitize_filename(self, name):
203ee3
         return re.sub(r"[^-a-z,A-Z.0-9]", "", name)
203ee3
 
203ee3
+    def _container_init(self):
203ee3
+        """Check if sos is running in a container and perform container
203ee3
+        specific initialisation based on ENV_HOST_SYSROOT.
203ee3
+        """
203ee3
+        if ENV_CONTAINER in os.environ:
203ee3
+            if os.environ[ENV_CONTAINER] in ['docker', 'oci', 'podman']:
203ee3
+                self._in_container = True
203ee3
+        if ENV_HOST_SYSROOT in os.environ:
203ee3
+            self._host_sysroot = os.environ[ENV_HOST_SYSROOT]
203ee3
+        use_sysroot = self._in_container and self._host_sysroot is not None
203ee3
+        if use_sysroot:
203ee3
+            host_tmp_dir = os.path.abspath(self._host_sysroot + self._tmp_dir)
203ee3
+            self._tmp_dir = host_tmp_dir
203ee3
+        return self._host_sysroot if use_sysroot else None
203ee3
+
203ee3
     def init_kernel_modules(self):
203ee3
         """Obtain a list of loaded kernel modules to reference later for plugin
203ee3
         enablement and SoSPredicate checks
203ee3
diff --git a/sos/policies/distros/redhat.py b/sos/policies/distros/redhat.py
203ee3
index b3a84336be..3476e21fb2 100644
203ee3
--- a/sos/policies/distros/redhat.py
203ee3
+++ b/sos/policies/distros/redhat.py
203ee3
@@ -17,7 +17,7 @@
203ee3
 from sos.presets.redhat import (RHEL_PRESETS, ATOMIC_PRESETS, RHV, RHEL,
203ee3
                                 CB, RHOSP, RHOCP, RH_CFME, RH_SATELLITE,
203ee3
                                 ATOMIC)
203ee3
-from sos.policies.distros import LinuxPolicy
203ee3
+from sos.policies.distros import LinuxPolicy, ENV_HOST_SYSROOT
203ee3
 from sos.policies.package_managers.rpm import RpmPackageManager
203ee3
 from sos import _sos as _
203ee3
 
203ee3
@@ -56,12 +56,6 @@ def __init__(self, sysroot=None, init=None, probe_runtime=True,
203ee3
         super(RedHatPolicy, self).__init__(sysroot=sysroot, init=init,
203ee3
                                            probe_runtime=probe_runtime)
203ee3
         self.usrmove = False
203ee3
-        # need to set _host_sysroot before PackageManager()
203ee3
-        if sysroot:
203ee3
-            self._container_init()
203ee3
-            self._host_sysroot = sysroot
203ee3
-        else:
203ee3
-            sysroot = self._container_init()
203ee3
 
203ee3
         self.package_manager = RpmPackageManager(chroot=sysroot,
203ee3
                                                  remote_exec=remote_exec)
203ee3
@@ -140,21 +134,6 @@ def transform_path(path):
203ee3
         else:
203ee3
             return files
203ee3
 
203ee3
-    def _container_init(self):
203ee3
-        """Check if sos is running in a container and perform container
203ee3
-        specific initialisation based on ENV_HOST_SYSROOT.
203ee3
-        """
203ee3
-        if ENV_CONTAINER in os.environ:
203ee3
-            if os.environ[ENV_CONTAINER] in ['docker', 'oci', 'podman']:
203ee3
-                self._in_container = True
203ee3
-        if ENV_HOST_SYSROOT in os.environ:
203ee3
-            self._host_sysroot = os.environ[ENV_HOST_SYSROOT]
203ee3
-        use_sysroot = self._in_container and self._host_sysroot is not None
203ee3
-        if use_sysroot:
203ee3
-            host_tmp_dir = os.path.abspath(self._host_sysroot + self._tmp_dir)
203ee3
-            self._tmp_dir = host_tmp_dir
203ee3
-        return self._host_sysroot if use_sysroot else None
203ee3
-
203ee3
     def runlevel_by_service(self, name):
203ee3
         from subprocess import Popen, PIPE
203ee3
         ret = []
203ee3
@@ -183,10 +162,6 @@ def get_tmp_dir(self, opt_tmp_dir):
203ee3
         return opt_tmp_dir
203ee3
 
203ee3
 
203ee3
-# Container environment variables on Red Hat systems.
203ee3
-ENV_CONTAINER = 'container'
203ee3
-ENV_HOST_SYSROOT = 'HOST'
203ee3
-
203ee3
 # Legal disclaimer text for Red Hat products
203ee3
 disclaimer_text = """
203ee3
 Any information provided to %(vendor)s will be treated in \
203ee3
diff --git a/sos/policies/init_systems/__init__.py b/sos/policies/init_systems/__init__.py
203ee3
index dd663e6522..beac44cee3 100644
203ee3
--- a/sos/policies/init_systems/__init__.py
203ee3
+++ b/sos/policies/init_systems/__init__.py
203ee3
@@ -29,9 +29,14 @@ class InitSystem():
203ee3
                       status of services
203ee3
     :type query_cmd: ``str``
203ee3
 
203ee3
+    :param chroot:  Location to chroot to for any command execution, i.e. the
203ee3
+                    sysroot if we're running in a container
203ee3
+    :type chroot:   ``str`` or ``None``
203ee3
+
203ee3
     """
203ee3
 
203ee3
-    def __init__(self, init_cmd=None, list_cmd=None, query_cmd=None):
203ee3
+    def __init__(self, init_cmd=None, list_cmd=None, query_cmd=None,
203ee3
+                 chroot=None):
203ee3
         """Initialize a new InitSystem()"""
203ee3
 
203ee3
         self.services = {}
203ee3
@@ -39,6 +44,7 @@ def __init__(self, init_cmd=None, list_cmd=None, query_cmd=None):
203ee3
         self.init_cmd = init_cmd
203ee3
         self.list_cmd = "%s %s" % (self.init_cmd, list_cmd) or None
203ee3
         self.query_cmd = "%s %s" % (self.init_cmd, query_cmd) or None
203ee3
+        self.chroot = chroot
203ee3
 
203ee3
     def is_enabled(self, name):
203ee3
         """Check if given service name is enabled
203ee3
@@ -108,7 +114,10 @@ def _query_service(self, name):
203ee3
         """Query an individual service"""
203ee3
         if self.query_cmd:
203ee3
             try:
203ee3
-                return sos_get_command_output("%s %s" % (self.query_cmd, name))
203ee3
+                return sos_get_command_output(
203ee3
+                    "%s %s" % (self.query_cmd, name),
203ee3
+                    chroot=self.chroot
203ee3
+                )
203ee3
             except Exception:
203ee3
                 return None
203ee3
         return None
203ee3
diff --git a/sos/policies/init_systems/systemd.py b/sos/policies/init_systems/systemd.py
203ee3
index 1b138f97b3..76dc57e27f 100644
203ee3
--- a/sos/policies/init_systems/systemd.py
203ee3
+++ b/sos/policies/init_systems/systemd.py
203ee3
@@ -15,11 +15,12 @@
203ee3
 class SystemdInit(InitSystem):
203ee3
     """InitSystem abstraction for SystemD systems"""
203ee3
 
203ee3
-    def __init__(self):
203ee3
+    def __init__(self, chroot=None):
203ee3
         super(SystemdInit, self).__init__(
203ee3
             init_cmd='systemctl',
203ee3
             list_cmd='list-unit-files --type=service',
203ee3
-            query_cmd='status'
203ee3
+            query_cmd='status',
203ee3
+            chroot=chroot
203ee3
         )
203ee3
         self.load_all_services()
203ee3
 
203ee3
@@ -30,7 +31,7 @@ def parse_query(self, output):
203ee3
         return 'unknown'
203ee3
 
203ee3
     def load_all_services(self):
203ee3
-        svcs = shell_out(self.list_cmd).splitlines()[1:]
203ee3
+        svcs = shell_out(self.list_cmd, chroot=self.chroot).splitlines()[1:]
203ee3
         for line in svcs:
203ee3
             try:
203ee3
                 name = line.split('.service')[0]
203ee3
From e869bc84c714bfc2249bbcb84e14908049ee42c4 Mon Sep 17 00:00:00 2001
203ee3
From: Jake Hunsaker <jhunsake@redhat.com>
203ee3
Date: Mon, 27 Sep 2021 12:07:08 -0400
203ee3
Subject: [PATCH] [Plugin,utilities] Add sysroot wrapper for os.path.join
203ee3
203ee3
Adds a wrapper for `os.path.join()` which accounts for non-/ sysroots,
203ee3
like we have done previously for other `os.path` methods. Further
203ee3
updates `Plugin()` to use this wrapper where appropriate.
203ee3
203ee3
Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>
203ee3
---
203ee3
 sos/report/plugins/__init__.py | 43 +++++++++++++++++-----------------
203ee3
 sos/utilities.py               |  6 +++++
203ee3
 2 files changed, 28 insertions(+), 21 deletions(-)
203ee3
203ee3
diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py
203ee3
index c635b8de9..1f84bca49 100644
203ee3
--- a/sos/report/plugins/__init__.py
203ee3
+++ b/sos/report/plugins/__init__.py
203ee3
@@ -13,7 +13,7 @@
203ee3
 from sos.utilities import (sos_get_command_output, import_module, grep,
203ee3
                            fileobj, tail, is_executable, TIMEOUT_DEFAULT,
203ee3
                            path_exists, path_isdir, path_isfile, path_islink,
203ee3
-                           listdir)
203ee3
+                           listdir, path_join)
203ee3
 
203ee3
 import os
203ee3
 import glob
203ee3
@@ -708,19 +708,6 @@ def _log_info(self, msg):
203ee3
     def _log_debug(self, msg):
203ee3
         self.soslog.debug(self._format_msg(msg))
203ee3
 
203ee3
-    def join_sysroot(self, path):
203ee3
-        """Join a given path with the configured sysroot
203ee3
-
203ee3
-        :param path:    The filesystem path that needs to be joined
203ee3
-        :type path: ``str``
203ee3
-
203ee3
-        :returns: The joined filesystem path
203ee3
-        :rtype: ``str``
203ee3
-        """
203ee3
-        if path[0] == os.sep:
203ee3
-            path = path[1:]
203ee3
-        return os.path.join(self.sysroot, path)
203ee3
-
203ee3
     def strip_sysroot(self, path):
203ee3
         """Remove the configured sysroot from a filesystem path
203ee3
 
203ee3
@@ -1176,7 +1163,7 @@ def _copy_dir(self, srcpath):
203ee3
 
203ee3
     def _get_dest_for_srcpath(self, srcpath):
203ee3
         if self.use_sysroot():
203ee3
-            srcpath = self.join_sysroot(srcpath)
203ee3
+            srcpath = self.path_join(srcpath)
203ee3
         for copied in self.copied_files:
203ee3
             if srcpath == copied["srcpath"]:
203ee3
                 return copied["dstpath"]
203ee3
@@ -1284,7 +1271,7 @@ def add_forbidden_path(self, forbidden, recursive=False):
203ee3
             forbidden = [forbidden]
203ee3
 
203ee3
         if self.use_sysroot():
203ee3
-            forbidden = [self.join_sysroot(f) for f in forbidden]
203ee3
+            forbidden = [self.path_join(f) for f in forbidden]
203ee3
 
203ee3
         for forbid in forbidden:
203ee3
             self._log_info("adding forbidden path '%s'" % forbid)
203ee3
@@ -1438,7 +1425,7 @@ def add_copy_spec(self, copyspecs, sizelimit=None, maxage=None,
203ee3
             since = self.get_option('since')
203ee3
 
203ee3
         logarchive_pattern = re.compile(r'.*((\.(zip|gz|bz2|xz))|[-.][\d]+)$')
203ee3
-        configfile_pattern = re.compile(r"^%s/*" % self.join_sysroot("etc"))
203ee3
+        configfile_pattern = re.compile(r"^%s/*" % self.path_join("etc"))
203ee3
 
203ee3
         if not self.test_predicate(pred=pred):
203ee3
             self._log_info("skipped copy spec '%s' due to predicate (%s)" %
203ee3
@@ -1468,7 +1455,7 @@ def add_copy_spec(self, copyspecs, sizelimit=None, maxage=None,
203ee3
                 return False
203ee3
 
203ee3
             if self.use_sysroot():
203ee3
-                copyspec = self.join_sysroot(copyspec)
203ee3
+                copyspec = self.path_join(copyspec)
203ee3
 
203ee3
             files = self._expand_copy_spec(copyspec)
203ee3
 
203ee3
@@ -1683,7 +1670,7 @@ def _add_device_cmd(self, cmds, devices, timeout=None, sizelimit=None,
203ee3
                 if not _dev_ok:
203ee3
                     continue
203ee3
                 if prepend_path:
203ee3
-                    device = os.path.join(prepend_path, device)
203ee3
+                    device = self.path_join(prepend_path, device)
203ee3
                 _cmd = cmd % {'dev': device}
203ee3
                 self._add_cmd_output(cmd=_cmd, timeout=timeout,
203ee3
                                      sizelimit=sizelimit, chroot=chroot,
203ee3
@@ -2592,7 +2579,7 @@ def __expand(paths):
203ee3
                     if self.path_isfile(path) or self.path_islink(path):
203ee3
                         found_paths.append(path)
203ee3
                     elif self.path_isdir(path) and self.listdir(path):
203ee3
-                        found_paths.extend(__expand(os.path.join(path, '*')))
203ee3
+                        found_paths.extend(__expand(self.path_join(path, '*')))
203ee3
                     else:
203ee3
                         found_paths.append(path)
203ee3
                 except PermissionError:
203ee3
@@ -2608,7 +2595,7 @@ def __expand(paths):
203ee3
         if (os.access(copyspec, os.R_OK) and self.path_isdir(copyspec) and
203ee3
                 self.listdir(copyspec)):
203ee3
             # the directory exists and is non-empty, recurse through it
203ee3
-            copyspec = os.path.join(copyspec, '*')
203ee3
+            copyspec = self.path_join(copyspec, '*')
203ee3
         expanded = glob.glob(copyspec, recursive=True)
203ee3
         recursed_files = []
203ee3
         for _path in expanded:
203ee3
@@ -2877,6 +2864,20 @@ def listdir(self, path):
203ee3
         """
203ee3
         return listdir(path, self.commons['cmdlineopts'].sysroot)
203ee3
 
203ee3
+    def path_join(self, path, *p):
203ee3
+        """Helper to call the sos.utilities wrapper that allows the
203ee3
+        corresponding `os` call to account for sysroot
203ee3
+
203ee3
+        :param path:    The leading path passed to os.path.join()
203ee3
+        :type path:     ``str``
203ee3
+
203ee3
+        :param p:       Following path section(s) to be joined with ``path``,
203ee3
+                        an empty parameter will result in a path that ends with
203ee3
+                        a separator
203ee3
+        :type p:        ``str``
203ee3
+        """
203ee3
+        return path_join(path, *p, sysroot=self.sysroot)
203ee3
+
203ee3
     def postproc(self):
203ee3
         """Perform any postprocessing. To be replaced by a plugin if required.
203ee3
         """
203ee3
diff --git a/sos/utilities.py b/sos/utilities.py
203ee3
index c940e066d..b75751539 100644
203ee3
--- a/sos/utilities.py
203ee3
+++ b/sos/utilities.py
203ee3
@@ -242,6 +242,12 @@ def listdir(path, sysroot):
203ee3
     return _os_wrapper(path, sysroot, 'listdir', os)
203ee3
 
203ee3
 
203ee3
+def path_join(path, *p, sysroot=os.sep):
203ee3
+    if not path.startswith(sysroot):
203ee3
+        path = os.path.join(sysroot, path.lstrip(os.sep))
203ee3
+    return os.path.join(path, *p)
203ee3
+
203ee3
+
203ee3
 class AsyncReader(threading.Thread):
203ee3
     """Used to limit command output to a given size without deadlocking
203ee3
     sos.
203ee3
From 9596473d1779b9c48e9923c220aaf2b8d9b3bebf Mon Sep 17 00:00:00 2001
203ee3
From: Jake Hunsaker <jhunsake@redhat.com>
203ee3
Date: Thu, 18 Nov 2021 13:17:14 -0500
203ee3
Subject: [PATCH] [global] Align sysroot determination and usage across sos
203ee3
203ee3
The determination of sysroot - being automatic, user-specified, or
203ee3
controlled via environment variables in a container - has gotten muddied
203ee3
over time. This has resulted in different parts of the project;
203ee3
`Policy`, `Plugin`, `SoSComponent`, etc... to not always be in sync when
203ee3
sysroot is not `/`, thus causing varying and unexpected/unintended
203ee3
behavior.
203ee3
203ee3
Fix this by only determining sysroot within `Policy()` initialization,
203ee3
and then using that determination across all aspects of the project that
203ee3
use or reference sysroot.
203ee3
203ee3
This results in several changes:
203ee3
203ee3
- `PackageManager()` will now (again) correctly reference host package
203ee3
  lists when sos is run in a container.
203ee3
203ee3
- `ContainerRuntime()` is now able to activate when sos is running in a
203ee3
  container.
203ee3
203ee3
- Plugins will now properly use sysroot for _all_ plugin enablement
203ee3
  triggers.
203ee3
203ee3
- Plugins, Policy, and SoSComponents now all reference the
203ee3
  `self.sysroot` variable, rather than changing between `sysroot`.
203ee3
`_host_sysroot`, and `commons['sysroot']`. `_host_sysroot` has been
203ee3
removed from `Policy`.
203ee3
203ee3
Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>
203ee3
---
203ee3
 sos/archive.py                    |  2 +-
203ee3
 sos/component.py                  |  2 +-
203ee3
 sos/policies/__init__.py          | 11 +----------
203ee3
 sos/policies/distros/__init__.py  | 33 +++++++++++++++++++------------
203ee3
 sos/policies/distros/debian.py    |  2 +-
203ee3
 sos/policies/distros/redhat.py    |  3 +--
203ee3
 sos/policies/runtimes/__init__.py | 15 +++++++++-----
203ee3
 sos/policies/runtimes/docker.py   |  4 ++--
203ee3
 sos/report/__init__.py            |  6 ++----
203ee3
 sos/report/plugins/__init__.py    | 22 +++++++++++----------
203ee3
 sos/report/plugins/unpackaged.py  |  7 ++++---
203ee3
 sos/utilities.py                  | 13 ++++++++----
203ee3
 12 files changed, 64 insertions(+), 56 deletions(-)
203ee3
203ee3
diff --git a/sos/archive.py b/sos/archive.py
203ee3
index b02b247595..e3c68b7789 100644
203ee3
--- a/sos/archive.py
203ee3
+++ b/sos/archive.py
203ee3
@@ -153,7 +153,7 @@ def dest_path(self, name):
203ee3
         return (os.path.join(self._archive_root, name))
203ee3
 
203ee3
     def join_sysroot(self, path):
203ee3
-        if path.startswith(self.sysroot):
203ee3
+        if not self.sysroot or path.startswith(self.sysroot):
203ee3
             return path
203ee3
         if path[0] == os.sep:
203ee3
             path = path[1:]
203ee3
diff --git a/sos/component.py b/sos/component.py
203ee3
index 5ac6e47f4f..dba0aabf2b 100644
203ee3
--- a/sos/component.py
203ee3
+++ b/sos/component.py
203ee3
@@ -109,7 +109,7 @@ def __init__(self, parser, parsed_args, cmdline_args):
203ee3
             try:
203ee3
                 import sos.policies
203ee3
                 self.policy = sos.policies.load(sysroot=self.opts.sysroot)
203ee3
-                self.sysroot = self.policy.host_sysroot()
203ee3
+                self.sysroot = self.policy.sysroot
203ee3
             except KeyboardInterrupt:
203ee3
                 self._exit(0)
203ee3
             self._is_root = self.policy.is_root()
203ee3
diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py
203ee3
index fb8db1d724..ef9188deb4 100644
203ee3
--- a/sos/policies/__init__.py
203ee3
+++ b/sos/policies/__init__.py
203ee3
@@ -110,7 +110,6 @@ class Policy(object):
203ee3
     presets = {"": PresetDefaults()}
203ee3
     presets_path = PRESETS_PATH
203ee3
     _in_container = False
203ee3
-    _host_sysroot = '/'
203ee3
 
203ee3
     def __init__(self, sysroot=None, probe_runtime=True):
203ee3
         """Subclasses that choose to override this initializer should call
203ee3
@@ -124,7 +123,7 @@ def __init__(self, sysroot=None, probe_runtime=True):
203ee3
         self.package_manager = PackageManager()
203ee3
         self.valid_subclasses = [IndependentPlugin]
203ee3
         self.set_exec_path()
203ee3
-        self._host_sysroot = sysroot
203ee3
+        self.sysroot = sysroot
203ee3
         self.register_presets(GENERIC_PRESETS)
203ee3
 
203ee3
     def check(self, remote=''):
203ee3
@@ -177,14 +176,6 @@ def in_container(self):
203ee3
         """
203ee3
         return self._in_container
203ee3
 
203ee3
-    def host_sysroot(self):
203ee3
-        """Get the host's default sysroot
203ee3
-
203ee3
-        :returns: Host sysroot
203ee3
-        :rtype: ``str`` or ``None``
203ee3
-        """
203ee3
-        return self._host_sysroot
203ee3
-
203ee3
     def dist_version(self):
203ee3
         """
203ee3
         Return the OS version
203ee3
diff --git a/sos/policies/distros/__init__.py b/sos/policies/distros/__init__.py
203ee3
index 7bdc81b852..c69fc1e73c 100644
203ee3
--- a/sos/policies/distros/__init__.py
203ee3
+++ b/sos/policies/distros/__init__.py
203ee3
@@ -71,19 +71,18 @@ class LinuxPolicy(Policy):
203ee3
     def __init__(self, sysroot=None, init=None, probe_runtime=True):
203ee3
         super(LinuxPolicy, self).__init__(sysroot=sysroot,
203ee3
                                           probe_runtime=probe_runtime)
203ee3
-        self.init_kernel_modules()
203ee3
 
203ee3
-        # need to set _host_sysroot before PackageManager()
203ee3
         if sysroot:
203ee3
-            self._container_init()
203ee3
-            self._host_sysroot = sysroot
203ee3
+            self.sysroot = sysroot
203ee3
         else:
203ee3
-            sysroot = self._container_init()
203ee3
+            self.sysroot = self._container_init()
203ee3
+
203ee3
+        self.init_kernel_modules()
203ee3
 
203ee3
         if init is not None:
203ee3
             self.init_system = init
203ee3
         elif os.path.isdir("/run/systemd/system/"):
203ee3
-            self.init_system = SystemdInit(chroot=sysroot)
203ee3
+            self.init_system = SystemdInit(chroot=self.sysroot)
203ee3
         else:
203ee3
             self.init_system = InitSystem()
203ee3
 
203ee3
@@ -149,27 +148,30 @@ def _container_init(self):
203ee3
             if os.environ[ENV_CONTAINER] in ['docker', 'oci', 'podman']:
203ee3
                 self._in_container = True
203ee3
         if ENV_HOST_SYSROOT in os.environ:
203ee3
-            self._host_sysroot = os.environ[ENV_HOST_SYSROOT]
203ee3
-        use_sysroot = self._in_container and self._host_sysroot is not None
203ee3
+            _host_sysroot = os.environ[ENV_HOST_SYSROOT]
203ee3
+        use_sysroot = self._in_container and _host_sysroot is not None
203ee3
         if use_sysroot:
203ee3
-            host_tmp_dir = os.path.abspath(self._host_sysroot + self._tmp_dir)
203ee3
+            host_tmp_dir = os.path.abspath(_host_sysroot + self._tmp_dir)
203ee3
             self._tmp_dir = host_tmp_dir
203ee3
-        return self._host_sysroot if use_sysroot else None
203ee3
+        return _host_sysroot if use_sysroot else None
203ee3
 
203ee3
     def init_kernel_modules(self):
203ee3
         """Obtain a list of loaded kernel modules to reference later for plugin
203ee3
         enablement and SoSPredicate checks
203ee3
         """
203ee3
         self.kernel_mods = []
203ee3
+        release = os.uname().release
203ee3
 
203ee3
         # first load modules from lsmod
203ee3
-        lines = shell_out("lsmod", timeout=0).splitlines()
203ee3
+        lines = shell_out("lsmod", timeout=0, chroot=self.sysroot).splitlines()
203ee3
         self.kernel_mods.extend([
203ee3
             line.split()[0].strip() for line in lines[1:]
203ee3
         ])
203ee3
 
203ee3
         # next, include kernel builtins
203ee3
-        builtins = "/usr/lib/modules/%s/modules.builtin" % os.uname().release
203ee3
+        builtins = self.join_sysroot(
203ee3
+            "/usr/lib/modules/%s/modules.builtin" % release
203ee3
+        )
203ee3
         try:
203ee3
             with open(builtins, "r") as mfile:
203ee3
                 for line in mfile:
203ee3
@@ -186,7 +188,7 @@ def init_kernel_modules(self):
203ee3
             'dm_mod': 'CONFIG_BLK_DEV_DM'
203ee3
         }
203ee3
 
203ee3
-        booted_config = "/boot/config-%s" % os.uname().release
203ee3
+        booted_config = self.join_sysroot("/boot/config-%s" % release)
203ee3
         kconfigs = []
203ee3
         try:
203ee3
             with open(booted_config, "r") as kfile:
203ee3
@@ -200,6 +202,11 @@ def init_kernel_modules(self):
203ee3
             if config_strings[builtin] in kconfigs:
203ee3
                 self.kernel_mods.append(builtin)
203ee3
 
203ee3
+    def join_sysroot(self, path):
203ee3
+        if self.sysroot and self.sysroot != '/':
203ee3
+            path = os.path.join(self.sysroot, path.lstrip('/'))
203ee3
+        return path
203ee3
+
203ee3
     def pre_work(self):
203ee3
         # this method will be called before the gathering begins
203ee3
 
203ee3
diff --git a/sos/policies/distros/debian.py b/sos/policies/distros/debian.py
203ee3
index 95b389a65e..639fd5eba3 100644
203ee3
--- a/sos/policies/distros/debian.py
203ee3
+++ b/sos/policies/distros/debian.py
203ee3
@@ -27,7 +27,7 @@ def __init__(self, sysroot=None, init=None, probe_runtime=True,
203ee3
                  remote_exec=None):
203ee3
         super(DebianPolicy, self).__init__(sysroot=sysroot, init=init,
203ee3
                                            probe_runtime=probe_runtime)
203ee3
-        self.package_manager = DpkgPackageManager(chroot=sysroot,
203ee3
+        self.package_manager = DpkgPackageManager(chroot=self.sysroot,
203ee3
                                                   remote_exec=remote_exec)
203ee3
         self.valid_subclasses += [DebianPlugin]
203ee3
 
203ee3
diff --git a/sos/policies/distros/redhat.py b/sos/policies/distros/redhat.py
203ee3
index eb44240736..4b14abaf3a 100644
203ee3
--- a/sos/policies/distros/redhat.py
203ee3
+++ b/sos/policies/distros/redhat.py
203ee3
@@ -42,7 +42,6 @@ class RedHatPolicy(LinuxPolicy):
203ee3
     _redhat_release = '/etc/redhat-release'
203ee3
     _tmp_dir = "/var/tmp"
203ee3
     _in_container = False
203ee3
-    _host_sysroot = '/'
203ee3
     default_scl_prefix = '/opt/rh'
203ee3
     name_pattern = 'friendly'
203ee3
     upload_url = None
203ee3
@@ -57,7 +56,7 @@ def __init__(self, sysroot=None, init=None, probe_runtime=True,
203ee3
                                            probe_runtime=probe_runtime)
203ee3
         self.usrmove = False
203ee3
 
203ee3
-        self.package_manager = RpmPackageManager(chroot=sysroot,
203ee3
+        self.package_manager = RpmPackageManager(chroot=self.sysroot,
203ee3
                                                  remote_exec=remote_exec)
203ee3
 
203ee3
         self.valid_subclasses += [RedHatPlugin]
203ee3
diff --git a/sos/policies/runtimes/__init__.py b/sos/policies/runtimes/__init__.py
203ee3
index f28d6a1df3..2e60ad2361 100644
203ee3
--- a/sos/policies/runtimes/__init__.py
203ee3
+++ b/sos/policies/runtimes/__init__.py
203ee3
@@ -64,7 +64,7 @@ def check_is_active(self):
203ee3
         :returns: ``True`` if the runtime is active, else ``False``
203ee3
         :rtype: ``bool``
203ee3
         """
203ee3
-        if is_executable(self.binary):
203ee3
+        if is_executable(self.binary, self.policy.sysroot):
203ee3
             self.active = True
203ee3
             return True
203ee3
         return False
203ee3
@@ -78,7 +78,7 @@ def get_containers(self, get_all=False):
203ee3
         containers = []
203ee3
         _cmd = "%s ps %s" % (self.binary, '-a' if get_all else '')
203ee3
         if self.active:
203ee3
-            out = sos_get_command_output(_cmd)
203ee3
+            out = sos_get_command_output(_cmd, chroot=self.policy.sysroot)
203ee3
             if out['status'] == 0:
203ee3
                 for ent in out['output'].splitlines()[1:]:
203ee3
                     ent = ent.split()
203ee3
@@ -112,8 +112,10 @@ def get_images(self):
203ee3
         images = []
203ee3
         fmt = '{{lower .Repository}}:{{lower .Tag}} {{lower .ID}}'
203ee3
         if self.active:
203ee3
-            out = sos_get_command_output("%s images --format '%s'"
203ee3
-                                         % (self.binary, fmt))
203ee3
+            out = sos_get_command_output(
203ee3
+                "%s images --format '%s'" % (self.binary, fmt),
203ee3
+                chroot=self.policy.sysroot
203ee3
+            )
203ee3
             if out['status'] == 0:
203ee3
                 for ent in out['output'].splitlines():
203ee3
                     ent = ent.split()
203ee3
@@ -129,7 +131,10 @@ def get_volumes(self):
203ee3
         """
203ee3
         vols = []
203ee3
         if self.active:
203ee3
-            out = sos_get_command_output("%s volume ls" % self.binary)
203ee3
+            out = sos_get_command_output(
203ee3
+                "%s volume ls" % self.binary,
203ee3
+                chroot=self.policy.sysroot
203ee3
+            )
203ee3
             if out['status'] == 0:
203ee3
                 for ent in out['output'].splitlines()[1:]:
203ee3
                     ent = ent.split()
203ee3
diff --git a/sos/policies/runtimes/docker.py b/sos/policies/runtimes/docker.py
203ee3
index 759dfaf6a0..e81f580ec3 100644
203ee3
--- a/sos/policies/runtimes/docker.py
203ee3
+++ b/sos/policies/runtimes/docker.py
203ee3
@@ -18,9 +18,9 @@ class DockerContainerRuntime(ContainerRuntime):
203ee3
     name = 'docker'
203ee3
     binary = 'docker'
203ee3
 
203ee3
-    def check_is_active(self):
203ee3
+    def check_is_active(self, sysroot=None):
203ee3
         # the daemon must be running
203ee3
-        if (is_executable('docker') and
203ee3
+        if (is_executable('docker', sysroot) and
203ee3
                 (self.policy.init_system.is_running('docker') or
203ee3
                  self.policy.init_system.is_running('snap.docker.dockerd'))):
203ee3
             self.active = True
203ee3
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
203ee3
index a4c92accd3..a6c72778fc 100644
203ee3
--- a/sos/report/__init__.py
203ee3
+++ b/sos/report/__init__.py
203ee3
@@ -173,14 +173,12 @@ def __init__(self, parser, args, cmdline):
203ee3
         self._set_directories()
203ee3
 
203ee3
         msg = "default"
203ee3
-        host_sysroot = self.policy.host_sysroot()
203ee3
+        self.sysroot = self.policy.sysroot
203ee3
         # set alternate system root directory
203ee3
         if self.opts.sysroot:
203ee3
             msg = "cmdline"
203ee3
-            self.sysroot = self.opts.sysroot
203ee3
-        elif self.policy.in_container() and host_sysroot != os.sep:
203ee3
+        elif self.policy.in_container() and self.sysroot != os.sep:
203ee3
             msg = "policy"
203ee3
-            self.sysroot = host_sysroot
203ee3
         self.soslog.debug("set sysroot to '%s' (%s)" % (self.sysroot, msg))
203ee3
 
203ee3
         if self.opts.chroot not in chroot_modes:
203ee3
diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py
203ee3
index 46028bb124..e180ae1727 100644
203ee3
--- a/sos/report/plugins/__init__.py
203ee3
+++ b/sos/report/plugins/__init__.py
203ee3
@@ -724,7 +724,7 @@ def strip_sysroot(self, path):
203ee3
         """
203ee3
         if not self.use_sysroot():
203ee3
             return path
203ee3
-        if path.startswith(self.sysroot):
203ee3
+        if self.sysroot and path.startswith(self.sysroot):
203ee3
             return path[len(self.sysroot):]
203ee3
         return path
203ee3
 
203ee3
@@ -743,8 +743,10 @@ def tmp_in_sysroot(self):
203ee3
                   ``False``
203ee3
         :rtype: ``bool``
203ee3
         """
203ee3
-        paths = [self.sysroot, self.archive.get_tmp_dir()]
203ee3
-        return os.path.commonprefix(paths) == self.sysroot
203ee3
+        # if sysroot is still None, that implies '/'
203ee3
+        _sysroot = self.sysroot or '/'
203ee3
+        paths = [_sysroot, self.archive.get_tmp_dir()]
203ee3
+        return os.path.commonprefix(paths) == _sysroot
203ee3
 
203ee3
     def is_installed(self, package_name):
203ee3
         """Is the package $package_name installed?
203ee3
@@ -2621,7 +2623,7 @@ def __expand(paths):
203ee3
         return list(set(expanded))
203ee3
 
203ee3
     def _collect_copy_specs(self):
203ee3
-        for path in self.copy_paths:
203ee3
+        for path in sorted(self.copy_paths, reverse=True):
203ee3
             self._log_info("collecting path '%s'" % path)
203ee3
             self._do_copy_path(path)
203ee3
         self.generate_copyspec_tags()
203ee3
@@ -2749,7 +2751,7 @@ def _check_plugin_triggers(self, files, packages, commands, services,
203ee3
 
203ee3
         return ((any(self.path_exists(fname) for fname in files) or
203ee3
                 any(self.is_installed(pkg) for pkg in packages) or
203ee3
-                any(is_executable(cmd) for cmd in commands) or
203ee3
+                any(is_executable(cmd, self.sysroot) for cmd in commands) or
203ee3
                 any(self.is_module_loaded(mod) for mod in self.kernel_mods) or
203ee3
                 any(self.is_service(svc) for svc in services) or
203ee3
                 any(self.container_exists(cntr) for cntr in containers)) and
203ee3
@@ -2817,7 +2819,7 @@ def path_exists(self, path):
203ee3
         :returns:           True if the path exists in sysroot, else False
203ee3
         :rtype:             ``bool``
203ee3
         """
203ee3
-        return path_exists(path, self.commons['cmdlineopts'].sysroot)
203ee3
+        return path_exists(path, self.sysroot)
203ee3
 
203ee3
     def path_isdir(self, path):
203ee3
         """Helper to call the sos.utilities wrapper that allows the
203ee3
@@ -2830,7 +2832,7 @@ def path_isdir(self, path):
203ee3
         :returns:           True if the path is a dir, else False
203ee3
         :rtype:             ``bool``
203ee3
         """
203ee3
-        return path_isdir(path, self.commons['cmdlineopts'].sysroot)
203ee3
+        return path_isdir(path, self.sysroot)
203ee3
 
203ee3
     def path_isfile(self, path):
203ee3
         """Helper to call the sos.utilities wrapper that allows the
203ee3
@@ -2843,7 +2845,7 @@ def path_isfile(self, path):
203ee3
         :returns:           True if the path is a file, else False
203ee3
         :rtype:             ``bool``
203ee3
         """
203ee3
-        return path_isfile(path, self.commons['cmdlineopts'].sysroot)
203ee3
+        return path_isfile(path, self.sysroot)
203ee3
 
203ee3
     def path_islink(self, path):
203ee3
         """Helper to call the sos.utilities wrapper that allows the
203ee3
@@ -2856,7 +2858,7 @@ def path_islink(self, path):
203ee3
         :returns:           True if the path is a link, else False
203ee3
         :rtype:             ``bool``
203ee3
         """
203ee3
-        return path_islink(path, self.commons['cmdlineopts'].sysroot)
203ee3
+        return path_islink(path, self.sysroot)
203ee3
 
203ee3
     def listdir(self, path):
203ee3
         """Helper to call the sos.utilities wrapper that allows the
203ee3
@@ -2869,7 +2871,7 @@ def listdir(self, path):
203ee3
         :returns:           Contents of path, if it is a directory
203ee3
         :rtype:             ``list``
203ee3
         """
203ee3
-        return listdir(path, self.commons['cmdlineopts'].sysroot)
203ee3
+        return listdir(path, self.sysroot)
203ee3
 
203ee3
     def path_join(self, path, *p):
203ee3
         """Helper to call the sos.utilities wrapper that allows the
203ee3
diff --git a/sos/report/plugins/unpackaged.py b/sos/report/plugins/unpackaged.py
203ee3
index 772b1d1fbb..24203c4b13 100644
203ee3
--- a/sos/report/plugins/unpackaged.py
203ee3
+++ b/sos/report/plugins/unpackaged.py
203ee3
@@ -58,10 +58,11 @@ def format_output(files):
203ee3
             """
203ee3
             expanded = []
203ee3
             for f in files:
203ee3
-                if self.path_islink(f):
203ee3
-                    expanded.append("{} -> {}".format(f, os.readlink(f)))
203ee3
+                fp = self.path_join(f)
203ee3
+                if self.path_islink(fp):
203ee3
+                    expanded.append("{} -> {}".format(fp, os.readlink(fp)))
203ee3
                 else:
203ee3
-                    expanded.append(f)
203ee3
+                    expanded.append(fp)
203ee3
             return expanded
203ee3
 
203ee3
         # Check command predicate to avoid costly processing
203ee3
diff --git a/sos/utilities.py b/sos/utilities.py
203ee3
index b757515397..d66309334b 100644
203ee3
--- a/sos/utilities.py
203ee3
+++ b/sos/utilities.py
203ee3
@@ -96,11 +96,15 @@ def grep(pattern, *files_or_paths):
203ee3
     return matches
203ee3
 
203ee3
 
203ee3
-def is_executable(command):
203ee3
+def is_executable(command, sysroot=None):
203ee3
     """Returns if a command matches an executable on the PATH"""
203ee3
 
203ee3
     paths = os.environ.get("PATH", "").split(os.path.pathsep)
203ee3
     candidates = [command] + [os.path.join(p, command) for p in paths]
203ee3
+    if sysroot:
203ee3
+        candidates += [
203ee3
+            os.path.join(sysroot, c.lstrip('/')) for c in candidates
203ee3
+        ]
203ee3
     return any(os.access(path, os.X_OK) for path in candidates)
203ee3
 
203ee3
 
203ee3
@@ -216,8 +220,9 @@ def get_human_readable(size, precision=2):
203ee3
 
203ee3
 
203ee3
 def _os_wrapper(path, sysroot, method, module=os.path):
203ee3
-    if sysroot not in [None, '/']:
203ee3
-        path = os.path.join(sysroot, path.lstrip('/'))
203ee3
+    if sysroot and sysroot != os.sep:
203ee3
+        if not path.startswith(sysroot):
203ee3
+            path = os.path.join(sysroot, path.lstrip('/'))
203ee3
     _meth = getattr(module, method)
203ee3
     return _meth(path)
203ee3
 
203ee3
@@ -243,7 +248,7 @@ def listdir(path, sysroot):
203ee3
 
203ee3
 
203ee3
 def path_join(path, *p, sysroot=os.sep):
203ee3
-    if not path.startswith(sysroot):
203ee3
+    if sysroot and not path.startswith(sysroot):
203ee3
         path = os.path.join(sysroot, path.lstrip(os.sep))
203ee3
     return os.path.join(path, *p)
203ee3
 
203ee3
From a43124e1f6217107838eed4d70339d100cbbc77a Mon Sep 17 00:00:00 2001
203ee3
From: Pavel Moravec <pmoravec@redhat.com>
203ee3
Date: Wed, 9 Feb 2022 19:45:27 +0100
203ee3
Subject: [PATCH] [policies] Set fallback to None sysroot
203ee3
203ee3
9596473 commit added a regression allowing to set sysroot to None
203ee3
when running sos report on a regular system (outside a container). In
203ee3
such a case, we need to fallback to '/' sysroot.
203ee3
203ee3
Resolves: #2846
203ee3
203ee3
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
203ee3
---
203ee3
 sos/policies/distros/__init__.py | 2 +-
203ee3
 1 file changed, 1 insertion(+), 1 deletion(-)
203ee3
203ee3
diff --git a/sos/policies/distros/__init__.py b/sos/policies/distros/__init__.py
203ee3
index f3c1de11..9048f1c4 100644
203ee3
--- a/sos/policies/distros/__init__.py
203ee3
+++ b/sos/policies/distros/__init__.py
203ee3
@@ -78,7 +78,7 @@ class LinuxPolicy(Policy):
203ee3
         if sysroot:
203ee3
             self.sysroot = sysroot
203ee3
         else:
203ee3
-            self.sysroot = self._container_init()
203ee3
+            self.sysroot = self._container_init() or '/'
203ee3
203ee3
         self.init_kernel_modules()
203ee3
203ee3
-- 
203ee3
2.34.1
203ee3