Blame SOURCES/sos-bz2011537-estimate-only-option.patch

92cd8f
From 5b245b1e449c6a05d09034bcb8290bffded79327 Mon Sep 17 00:00:00 2001
92cd8f
From: Pavel Moravec <pmoravec@redhat.com>
92cd8f
Date: Wed, 8 Sep 2021 17:04:58 +0200
92cd8f
Subject: [PATCH] [report] Implement --estimate-only
92cd8f
92cd8f
Add report option --estimate-only to estimate disk space requirements
92cd8f
when running a sos report.
92cd8f
92cd8f
Resolves: #2673
92cd8f
92cd8f
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
92cd8f
---
92cd8f
 man/en/sos-report.1    | 13 +++++++-
92cd8f
 sos/report/__init__.py | 74 ++++++++++++++++++++++++++++++++++++++++--
92cd8f
 2 files changed, 84 insertions(+), 3 deletions(-)
92cd8f
92cd8f
diff --git a/man/en/sos-report.1 b/man/en/sos-report.1
92cd8f
index 36b337df..e8efc8f8 100644
92cd8f
--- a/man/en/sos-report.1
92cd8f
+++ b/man/en/sos-report.1
92cd8f
@@ -14,7 +14,7 @@ sos report \- Collect and package diagnostic and support data
92cd8f
           [--preset preset] [--add-preset add_preset]\fR
92cd8f
           [--del-preset del_preset] [--desc description]\fR
92cd8f
           [--batch] [--build] [--debug] [--dry-run]\fR
92cd8f
-          [--label label] [--case-id id]\fR
92cd8f
+          [--estimate-only] [--label label] [--case-id id]\fR
92cd8f
           [--threads threads]\fR
92cd8f
           [--plugin-timeout TIMEOUT]\fR
92cd8f
           [--cmd-timeout TIMEOUT]\fR
92cd8f
@@ -317,6 +317,17 @@ output, or string data from the system. The resulting logs may be used
92cd8f
 to understand the actions that sos would have taken without the dry run
92cd8f
 option.
92cd8f
 .TP
92cd8f
+.B \--estimate-only
92cd8f
+Estimate disk space requirements when running sos report. This can be valuable
92cd8f
+to prevent sosreport working dir to consume all free disk space. No plugin data
92cd8f
+is available at the end.
92cd8f
+
92cd8f
+Plugins will be collected sequentially, size of collected files and commands outputs
92cd8f
+will be calculated and the plugin files will be immediatelly deleted prior execution
92cd8f
+of the next plugin. This still can consume whole free disk space, though. Please note,
92cd8f
+size estimations may not be accurate for highly utilized systems due to changes between
92cd8f
+an estimate and a real execution.
92cd8f
+.TP
92cd8f
 .B \--upload
92cd8f
 If specified, attempt to upload the resulting archive to a vendor defined location.
92cd8f
 
92cd8f
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
92cd8f
index 82484f1d..b033f621 100644
92cd8f
--- a/sos/report/__init__.py
92cd8f
+++ b/sos/report/__init__.py
92cd8f
@@ -86,6 +86,7 @@ class SoSReport(SoSComponent):
92cd8f
         'desc': '',
92cd8f
         'domains': [],
92cd8f
         'dry_run': False,
92cd8f
+        'estimate_only': False,
92cd8f
         'experimental': False,
92cd8f
         'enable_plugins': [],
92cd8f
         'keywords': [],
92cd8f
@@ -137,6 +138,7 @@ class SoSReport(SoSComponent):
92cd8f
         self._args = args
92cd8f
         self.sysroot = "/"
92cd8f
         self.preset = None
92cd8f
+        self.estimated_plugsizes = {}
92cd8f
 
92cd8f
         self.print_header()
92cd8f
         self._set_debug()
92cd8f
@@ -223,6 +225,11 @@ class SoSReport(SoSComponent):
92cd8f
                                 help="Description for a new preset",)
92cd8f
         report_grp.add_argument("--dry-run", action="store_true",
92cd8f
                                 help="Run plugins but do not collect data")
92cd8f
+        report_grp.add_argument("--estimate-only", action="store_true",
92cd8f
+                                help="Approximate disk space requirements for "
92cd8f
+                                     "a real sos run; disables --clean and "
92cd8f
+                                     "--collect, sets --threads=1 and "
92cd8f
+                                     "--no-postproc")
92cd8f
         report_grp.add_argument("--experimental", action="store_true",
92cd8f
                                 dest="experimental", default=False,
92cd8f
                                 help="enable experimental plugins")
92cd8f
@@ -700,6 +700,33 @@ class SoSReport(SoSComponent):
92cd8f
                 self.all_options.append((plugin, plugin_name, optname,
92cd8f
                                          optparm))
92cd8f
 
92cd8f
+    def _set_estimate_only(self):
92cd8f
+        # set estimate-only mode by enforcing some options settings
92cd8f
+        # and return a corresponding log messages string
92cd8f
+        msg = "\nEstimate-only mode enabled"
92cd8f
+        ext_msg = []
92cd8f
+        if self.opts.threads > 1:
92cd8f
+            ext_msg += ["--threads=%s overriden to 1" % self.opts.threads, ]
92cd8f
+            self.opts.threads = 1
92cd8f
+        if not self.opts.build:
92cd8f
+            ext_msg += ["--build enabled", ]
92cd8f
+            self.opts.build = True
92cd8f
+        if not self.opts.no_postproc:
92cd8f
+            ext_msg += ["--no-postproc enabled", ]
92cd8f
+            self.opts.no_postproc = True
92cd8f
+        if self.opts.clean:
92cd8f
+            ext_msg += ["--clean disabled", ]
92cd8f
+            self.opts.clean = False
92cd8f
+        if self.opts.upload:
92cd8f
+            ext_msg += ["--upload* options disabled", ]
92cd8f
+            self.opts.upload = False
92cd8f
+        if ext_msg:
92cd8f
+            msg += ", which overrides some options:\n  " + "\n  ".join(ext_msg)
92cd8f
+        else:
92cd8f
+            msg += "."
92cd8f
+        msg += "\n\n"
92cd8f
+        return msg
92cd8f
+
92cd8f
     def _report_profiles_and_plugins(self):
92cd8f
         self.ui_log.info("")
92cd8f
         if len(self.loaded_plugins):
92cd8f
@@ -875,10 +909,12 @@ class SoSReport(SoSComponent):
92cd8f
         return True
92cd8f
 
92cd8f
     def batch(self):
92cd8f
+        msg = self.policy.get_msg()
92cd8f
+        if self.opts.estimate_only:
92cd8f
+            msg += self._set_estimate_only()
92cd8f
         if self.opts.batch:
92cd8f
-            self.ui_log.info(self.policy.get_msg())
92cd8f
+            self.ui_log.info(msg)
92cd8f
         else:
92cd8f
-            msg = self.policy.get_msg()
92cd8f
             msg += _("Press ENTER to continue, or CTRL-C to quit.\n")
92cd8f
             try:
92cd8f
                 input(msg)
92cd8f
@@ -1011,6 +1047,22 @@ class SoSReport(SoSComponent):
92cd8f
                 self.running_plugs.remove(plugin[1])
92cd8f
                 self.loaded_plugins[plugin[0]-1][1].set_timeout_hit()
92cd8f
                 pool._threads.clear()
92cd8f
+        if self.opts.estimate_only:
92cd8f
+            from pathlib import Path
92cd8f
+            tmpdir_path = Path(self.archive.get_tmp_dir())
92cd8f
+            self.estimated_plugsizes[plugin[1]] = sum(
92cd8f
+                    [f.stat().st_size for f in tmpdir_path.glob('**/*')
92cd8f
+                     if (os.path.isfile(f) and not os.path.islink(f))])
92cd8f
+            # remove whole tmp_dir content - including "sos_commands" and
92cd8f
+            # similar dirs that will be re-created on demand by next plugin
92cd8f
+            # if needed; it is less error-prone approach than skipping
92cd8f
+            # deletion of some dirs but deleting their content
92cd8f
+            for f in os.listdir(self.archive.get_tmp_dir()):
92cd8f
+                f = os.path.join(self.archive.get_tmp_dir(), f)
92cd8f
+                if os.path.isdir(f):
92cd8f
+                    rmtree(f)
92cd8f
+                else:
92cd8f
+                    os.unlink(f)
92cd8f
         return True
92cd8f
 
92cd8f
     def collect_plugin(self, plugin):
92cd8f
@@ -1330,6 +1382,24 @@ class SoSReport(SoSComponent):
92cd8f
             self.policy.display_results(archive, directory, checksum,
92cd8f
                                         map_file=map_file)
92cd8f
 
92cd8f
+        if self.opts.estimate_only:
92cd8f
+            from sos.utilities import get_human_readable
92cd8f
+            _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
92cd8f
+            self.ui_log.info("Estimated disk space requirement for whole "
92cd8f
+                             "uncompressed sos report directory: %s" % _sum)
92cd8f
+            bigplugins = sorted(self.estimated_plugsizes.items(),
92cd8f
+                                key=lambda x: x[1], reverse=True)[:3]
92cd8f
+            bp_out = ",  ".join("%s: %s" %
92cd8f
+                                (p, get_human_readable(v, precision=0))
92cd8f
+                                for p, v in bigplugins)
92cd8f
+            self.ui_log.info("Three biggest plugins:  %s" % bp_out)
92cd8f
+            self.ui_log.info("")
92cd8f
+            self.ui_log.info("Please note the estimation is relevant to the "
92cd8f
+                             "current options.")
92cd8f
+            self.ui_log.info("Be aware that the real disk space requirements "
92cd8f
+                             "might be different.")
92cd8f
+            self.ui_log.info("")
92cd8f
+
92cd8f
         if self.opts.upload or self.opts.upload_url:
92cd8f
             if not self.opts.build:
92cd8f
                 try:
92cd8f
-- 
92cd8f
2.31.1
92cd8f
92cd8f
From 7ae47e6c0717c0b56c3368008dd99a87f7f436d5 Mon Sep 17 00:00:00 2001
92cd8f
From: Pavel Moravec <pmoravec@redhat.com>
92cd8f
Date: Wed, 13 Oct 2021 20:21:16 +0200
92cd8f
Subject: [PATCH] [report] Count with sos_logs and sos_reports in
92cd8f
 --estimate-only
92cd8f
92cd8f
Currently, we estimate just plugins' disk space and ignore sos_logs
92cd8f
or sos_reports directories - although they can occupy nontrivial disk
92cd8f
space as well.
92cd8f
92cd8f
Resolves: #2723
92cd8f
92cd8f
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
92cd8f
---
92cd8f
 sos/report/__init__.py | 8 ++++++++
92cd8f
 1 file changed, 8 insertions(+)
92cd8f
92cd8f
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
92cd8f
index e35c7e8d..7feb31ee 100644
92cd8f
--- a/sos/report/__init__.py
92cd8f
+++ b/sos/report/__init__.py
92cd8f
@@ -1380,6 +1380,14 @@ class SoSReport(SoSComponent):
92cd8f
 
92cd8f
         if self.opts.estimate_only:
92cd8f
             from sos.utilities import get_human_readable
92cd8f
+            from pathlib import Path
92cd8f
+            # add sos_logs, sos_reports dirs, etc., basically everything
92cd8f
+            # that remained in self.tmpdir after plugins' contents removal
92cd8f
+            # that still will be moved to the sos report final directory path
92cd8f
+            tmpdir_path = Path(self.tmpdir)
92cd8f
+            self.estimated_plugsizes['sos_logs_reports'] = sum(
92cd8f
+                    [f.stat().st_size for f in tmpdir_path.glob('**/*')])
92cd8f
+
92cd8f
             _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
92cd8f
             self.ui_log.info("Estimated disk space requirement for whole "
92cd8f
                              "uncompressed sos report directory: %s" % _sum)
92cd8f
-- 
92cd8f
2.31.1
92cd8f
92cd8f
From 4293f3317505661e8f32ba94ad87310996fa1626 Mon Sep 17 00:00:00 2001
92cd8f
From: Eric Desrochers <eric.desrochers@canonical.com>
92cd8f
Date: Tue, 19 Oct 2021 12:18:40 -0400
92cd8f
Subject: [PATCH] [report] check for symlink before rmtree when opt
92cd8f
 estimate-only is use
92cd8f
92cd8f
Check if the dir is also symlink before performing rmtree()
92cd8f
method so that unlink() method can be used instead.
92cd8f
92cd8f
Traceback (most recent call last):
92cd8f
  File "./bin/sos", line 22, in <module>
92cd8f
    sos.execute()
92cd8f
  File "/tmp/sos/sos/__init__.py", line 186, in execute
92cd8f
    self._component.execute()
92cd8f
OSError: Cannot call rmtree on a symbolic link
92cd8f
92cd8f
Closes: #2727
92cd8f
92cd8f
Signed-off-by: Eric Desrochers <eric.desrochers@canonical.com>
92cd8f
---
92cd8f
 sos/report/__init__.py | 2 +-
92cd8f
 1 file changed, 1 insertion(+), 1 deletion(-)
92cd8f
92cd8f
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
92cd8f
index 7feb31ee..1b5bc97d 100644
92cd8f
--- a/sos/report/__init__.py
92cd8f
+++ b/sos/report/__init__.py
92cd8f
@@ -1059,7 +1059,7 @@ class SoSReport(SoSComponent):
92cd8f
             # deletion of some dirs but deleting their content
92cd8f
             for f in os.listdir(self.archive.get_tmp_dir()):
92cd8f
                 f = os.path.join(self.archive.get_tmp_dir(), f)
92cd8f
-                if os.path.isdir(f):
92cd8f
+                if os.path.isdir(f) and not os.path.islink(f):
92cd8f
                     rmtree(f)
92cd8f
                 else:
92cd8f
                     os.unlink(f)
92cd8f
-- 
92cd8f
2.31.1
92cd8f
92cd8f
From 589d47c93257b55bc796ef6ac25b88c974ee3d72 Mon Sep 17 00:00:00 2001
92cd8f
From: Pavel Moravec <pmoravec@redhat.com>
92cd8f
Date: Mon, 8 Nov 2021 16:38:24 +0100
92cd8f
Subject: [PATCH] [report] Calculate sizes of dirs, symlinks and manifest in
92cd8f
 estimate mode
92cd8f
92cd8f
Enhance --estimate-mode to calculate sizes of also:
92cd8f
- symlinks
92cd8f
- directories themselves
92cd8f
- manifest.json file
92cd8f
92cd8f
Use os.lstat() method instead of os.stat() to properly calculate the
92cd8f
sizes (and not destinations of symlinks, e.g.).
92cd8f
92cd8f
Print five biggest plugins instead of three as sos logs and reports do
92cd8f
stand as one "plugin" in the list, often.
92cd8f
92cd8f
Resolves: #2752
92cd8f
92cd8f
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
92cd8f
---
92cd8f
 sos/report/__init__.py | 56 +++++++++++++++++++++---------------------
92cd8f
 1 file changed, 28 insertions(+), 28 deletions(-)
92cd8f
92cd8f
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
92cd8f
index 10952566..a4c92acc 100644
92cd8f
--- a/sos/report/__init__.py
92cd8f
+++ b/sos/report/__init__.py
92cd8f
@@ -1050,8 +1050,7 @@ class SoSReport(SoSComponent):
92cd8f
             from pathlib import Path
92cd8f
             tmpdir_path = Path(self.archive.get_tmp_dir())
92cd8f
             self.estimated_plugsizes[plugin[1]] = sum(
92cd8f
-                    [f.stat().st_size for f in tmpdir_path.glob('**/*')
92cd8f
-                     if (os.path.isfile(f) and not os.path.islink(f))])
92cd8f
+                    [f.lstat().st_size for f in tmpdir_path.glob('**/*')])
92cd8f
             # remove whole tmp_dir content - including "sos_commands" and
92cd8f
             # similar dirs that will be re-created on demand by next plugin
92cd8f
             # if needed; it is less error-prone approach than skipping
92cd8f
@@ -1273,6 +1272,33 @@ class SoSReport(SoSComponent):
92cd8f
                 short_name='manifest.json'
92cd8f
             )
92cd8f
 
92cd8f
+        # print results in estimate mode (to include also just added manifest)
92cd8f
+        if self.opts.estimate_only:
92cd8f
+            from sos.utilities import get_human_readable
92cd8f
+            from pathlib import Path
92cd8f
+            # add sos_logs, sos_reports dirs, etc., basically everything
92cd8f
+            # that remained in self.tmpdir after plugins' contents removal
92cd8f
+            # that still will be moved to the sos report final directory path
92cd8f
+            tmpdir_path = Path(self.tmpdir)
92cd8f
+            self.estimated_plugsizes['sos_logs_reports'] = sum(
92cd8f
+                    [f.lstat().st_size for f in tmpdir_path.glob('**/*')])
92cd8f
+
92cd8f
+            _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
92cd8f
+            self.ui_log.info("Estimated disk space requirement for whole "
92cd8f
+                             "uncompressed sos report directory: %s" % _sum)
92cd8f
+            bigplugins = sorted(self.estimated_plugsizes.items(),
92cd8f
+                                key=lambda x: x[1], reverse=True)[:5]
92cd8f
+            bp_out = ",  ".join("%s: %s" %
92cd8f
+                                (p, get_human_readable(v, precision=0))
92cd8f
+                                for p, v in bigplugins)
92cd8f
+            self.ui_log.info("Five biggest plugins:  %s" % bp_out)
92cd8f
+            self.ui_log.info("")
92cd8f
+            self.ui_log.info("Please note the estimation is relevant to the "
92cd8f
+                             "current options.")
92cd8f
+            self.ui_log.info("Be aware that the real disk space requirements "
92cd8f
+                             "might be different.")
92cd8f
+            self.ui_log.info("")
92cd8f
+
92cd8f
         # package up and compress the results
92cd8f
         if not self.opts.build:
92cd8f
             old_umask = os.umask(0o077)
92cd8f
@@ -1377,32 +1403,6 @@ class SoSReport(SoSComponent):
92cd8f
             self.policy.display_results(archive, directory, checksum,
92cd8f
                                         map_file=map_file)
92cd8f
 
92cd8f
-        if self.opts.estimate_only:
92cd8f
-            from sos.utilities import get_human_readable
92cd8f
-            from pathlib import Path
92cd8f
-            # add sos_logs, sos_reports dirs, etc., basically everything
92cd8f
-            # that remained in self.tmpdir after plugins' contents removal
92cd8f
-            # that still will be moved to the sos report final directory path
92cd8f
-            tmpdir_path = Path(self.tmpdir)
92cd8f
-            self.estimated_plugsizes['sos_logs_reports'] = sum(
92cd8f
-                    [f.stat().st_size for f in tmpdir_path.glob('**/*')])
92cd8f
-
92cd8f
-            _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
92cd8f
-            self.ui_log.info("Estimated disk space requirement for whole "
92cd8f
-                             "uncompressed sos report directory: %s" % _sum)
92cd8f
-            bigplugins = sorted(self.estimated_plugsizes.items(),
92cd8f
-                                key=lambda x: x[1], reverse=True)[:3]
92cd8f
-            bp_out = ",  ".join("%s: %s" %
92cd8f
-                                (p, get_human_readable(v, precision=0))
92cd8f
-                                for p, v in bigplugins)
92cd8f
-            self.ui_log.info("Three biggest plugins:  %s" % bp_out)
92cd8f
-            self.ui_log.info("")
92cd8f
-            self.ui_log.info("Please note the estimation is relevant to the "
92cd8f
-                             "current options.")
92cd8f
-            self.ui_log.info("Be aware that the real disk space requirements "
92cd8f
-                             "might be different.")
92cd8f
-            self.ui_log.info("")
92cd8f
-
92cd8f
         if self.opts.upload or self.opts.upload_url:
92cd8f
             if not self.opts.build:
92cd8f
                 try:
92cd8f
-- 
92cd8f
2.31.1
92cd8f
9bf60c
From c6a5bbb8d75aadd5c7f76d3f469929aba2cf8060 Mon Sep 17 00:00:00 2001
9bf60c
From: Pavel Moravec <pmoravec@redhat.com>
9bf60c
Date: Wed, 5 Jan 2022 10:33:58 +0100
9bf60c
Subject: [PATCH] [report] Provide better warning about estimate-mode
9bf60c
9bf60c
As --estimate-only calculates disk usage based on `stat` data that
9bf60c
differs from outputs of other commands like `du`, enhance the warning
9bf60c
about reliability of the calculated estimation.
9bf60c
9bf60c
Also add a rule-of-thumb recommendation of real disk space requirements.
9bf60c
9bf60c
Resolves: #2815
9bf60c
9bf60c
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
9bf60c
---
9bf60c
 man/en/sos-report.1    | 10 +++++++---
9bf60c
 sos/report/__init__.py |  3 ++-
9bf60c
 2 files changed, 9 insertions(+), 4 deletions(-)
9bf60c
9bf60c
diff --git a/man/en/sos-report.1 b/man/en/sos-report.1
9bf60c
index 464a77e54..e34773986 100644
9bf60c
--- a/man/en/sos-report.1
9bf60c
+++ b/man/en/sos-report.1
9bf60c
@@ -343,9 +343,13 @@ is available at the end.
9bf60c
 
9bf60c
 Plugins will be collected sequentially, size of collected files and commands outputs
9bf60c
 will be calculated and the plugin files will be immediatelly deleted prior execution
9bf60c
-of the next plugin. This still can consume whole free disk space, though. Please note,
9bf60c
-size estimations may not be accurate for highly utilized systems due to changes between
9bf60c
-an estimate and a real execution.
9bf60c
+of the next plugin. This still can consume whole free disk space, though.
9bf60c
+
9bf60c
+Please note, size estimations may not be accurate for highly utilized systems due to
9bf60c
+changes between an estimate and a real execution. Also some difference between
9bf60c
+estimation (using `stat` command) and other commands used (i.e. `du`).
9bf60c
+
9bf60c
+A rule of thumb is to reserve at least double the estimation.
9bf60c
 .TP
9bf60c
 .B \--upload
9bf60c
 If specified, attempt to upload the resulting archive to a vendor defined location.
9bf60c
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
9bf60c
index ef61fb344..e0617b45e 100644
9bf60c
--- a/sos/report/__init__.py
9bf60c
+++ b/sos/report/__init__.py
9bf60c
@@ -1330,7 +1330,8 @@ def final_work(self):
9bf60c
             self.ui_log.info("Please note the estimation is relevant to the "
9bf60c
                              "current options.")
9bf60c
             self.ui_log.info("Be aware that the real disk space requirements "
9bf60c
-                             "might be different.")
9bf60c
+                             "might be different. A rule of thumb is to "
9bf60c
+                             "reserve at least double the estimation.")
9bf60c
             self.ui_log.info("")
9bf60c
 
9bf60c
         # package up and compress the results