From 5b245b1e449c6a05d09034bcb8290bffded79327 Mon Sep 17 00:00:00 2001

From: Pavel Moravec <pmoravec@redhat.com>

Date: Wed, 8 Sep 2021 17:04:58 +0200

Subject: [PATCH] [report] Implement --estimate-only

Add report option --estimate-only to estimate disk space requirements

when running a sos report.

Resolves: #2673

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

---

 man/en/sos-report.1    | 13 +++++++-

 sos/report/__init__.py | 74 ++++++++++++++++++++++++++++++++++++++++--

 2 files changed, 84 insertions(+), 3 deletions(-)

diff --git a/man/en/sos-report.1 b/man/en/sos-report.1

index 36b337df..e8efc8f8 100644

--- a/man/en/sos-report.1

+++ b/man/en/sos-report.1

@@ -14,7 +14,7 @@ sos report \- Collect and package diagnostic and support data

           [--preset preset] [--add-preset add_preset]\fR

           [--del-preset del_preset] [--desc description]\fR

           [--batch] [--build] [--debug] [--dry-run]\fR

-          [--label label] [--case-id id]\fR

+          [--estimate-only] [--label label] [--case-id id]\fR

           [--threads threads]\fR

           [--plugin-timeout TIMEOUT]\fR

           [--cmd-timeout TIMEOUT]\fR

@@ -317,6 +317,17 @@ output, or string data from the system. The resulting logs may be used

 to understand the actions that sos would have taken without the dry run

 option.

 .TP

+.B \--estimate-only

+Estimate disk space requirements when running sos report. This can be valuable

+to prevent sosreport working dir to consume all free disk space. No plugin data

+is available at the end.

+

+Plugins will be collected sequentially, size of collected files and commands outputs

+will be calculated and the plugin files will be immediatelly deleted prior execution

+of the next plugin. This still can consume whole free disk space, though. Please note,

+size estimations may not be accurate for highly utilized systems due to changes between

+an estimate and a real execution.

+.TP

 .B \--upload

 If specified, attempt to upload the resulting archive to a vendor defined location.

diff --git a/sos/report/__init__.py b/sos/report/__init__.py

index 82484f1d..b033f621 100644

--- a/sos/report/__init__.py

+++ b/sos/report/__init__.py

@@ -86,6 +86,7 @@ class SoSReport(SoSComponent):

         'desc': '',

         'domains': [],

         'dry_run': False,

+        'estimate_only': False,

         'experimental': False,

         'enable_plugins': [],

         'keywords': [],

@@ -137,6 +138,7 @@ class SoSReport(SoSComponent):

         self._args = args

         self.sysroot = "/"

         self.preset = None

+        self.estimated_plugsizes = {}

         self.print_header()

         self._set_debug()

@@ -223,6 +225,11 @@ class SoSReport(SoSComponent):

                                 help="Description for a new preset",)

         report_grp.add_argument("--dry-run", action="store_true",

                                 help="Run plugins but do not collect data")

+        report_grp.add_argument("--estimate-only", action="store_true",

+                                help="Approximate disk space requirements for "

+                                     "a real sos run; disables --clean and "

+                                     "--collect, sets --threads=1 and "

+                                     "--no-postproc")

         report_grp.add_argument("--experimental", action="store_true",

                                 dest="experimental", default=False,

                                 help="enable experimental plugins")

@@ -700,6 +700,33 @@ class SoSReport(SoSComponent):

                 self.all_options.append((plugin, plugin_name, optname,

                                          optparm))

+    def _set_estimate_only(self):

+        # set estimate-only mode by enforcing some options settings

+        # and return a corresponding log messages string

+        msg = "\nEstimate-only mode enabled"

+        ext_msg = []

+        if self.opts.threads > 1:

+            ext_msg += ["--threads=%s overriden to 1" % self.opts.threads, ]

+            self.opts.threads = 1

+        if not self.opts.build:

+            ext_msg += ["--build enabled", ]

+            self.opts.build = True

+        if not self.opts.no_postproc:

+            ext_msg += ["--no-postproc enabled", ]

+            self.opts.no_postproc = True

+        if self.opts.clean:

+            ext_msg += ["--clean disabled", ]

+            self.opts.clean = False

+        if self.opts.upload:

+            ext_msg += ["--upload* options disabled", ]

+            self.opts.upload = False

+        if ext_msg:

+            msg += ", which overrides some options:\n  " + "\n  ".join(ext_msg)

+        else:

+            msg += "."

+        msg += "\n\n"

+        return msg

+

     def _report_profiles_and_plugins(self):

         self.ui_log.info("")

         if len(self.loaded_plugins):

@@ -875,10 +909,12 @@ class SoSReport(SoSComponent):

         return True

     def batch(self):

+        msg = self.policy.get_msg()

+        if self.opts.estimate_only:

+            msg += self._set_estimate_only()

         if self.opts.batch:

-            self.ui_log.info(self.policy.get_msg())

+            self.ui_log.info(msg)

         else:

-            msg = self.policy.get_msg()

             msg += _("Press ENTER to continue, or CTRL-C to quit.\n")

             try:

                 input(msg)

@@ -1011,6 +1047,22 @@ class SoSReport(SoSComponent):

                 self.running_plugs.remove(plugin[1])

                 self.loaded_plugins[plugin[0]-1][1].set_timeout_hit()

                 pool._threads.clear()

+        if self.opts.estimate_only:

+            from pathlib import Path

+            tmpdir_path = Path(self.archive.get_tmp_dir())

+            self.estimated_plugsizes[plugin[1]] = sum(

+                    [f.stat().st_size for f in tmpdir_path.glob('**/*')

+                     if (os.path.isfile(f) and not os.path.islink(f))])

+            # remove whole tmp_dir content - including "sos_commands" and

+            # similar dirs that will be re-created on demand by next plugin

+            # if needed; it is less error-prone approach than skipping

+            # deletion of some dirs but deleting their content

+            for f in os.listdir(self.archive.get_tmp_dir()):

+                f = os.path.join(self.archive.get_tmp_dir(), f)

+                if os.path.isdir(f):

+                    rmtree(f)

+                else:

+                    os.unlink(f)

         return True

     def collect_plugin(self, plugin):

@@ -1330,6 +1382,24 @@ class SoSReport(SoSComponent):

             self.policy.display_results(archive, directory, checksum,

                                         map_file=map_file)

+        if self.opts.estimate_only:

+            from sos.utilities import get_human_readable

+            _sum = get_human_readable(sum(self.estimated_plugsizes.values()))

+            self.ui_log.info("Estimated disk space requirement for whole "

+                             "uncompressed sos report directory: %s" % _sum)

+            bigplugins = sorted(self.estimated_plugsizes.items(),

+                                key=lambda x: x[1], reverse=True)[:3]

+            bp_out = ",  ".join("%s: %s" %

+                                (p, get_human_readable(v, precision=0))

+                                for p, v in bigplugins)

+            self.ui_log.info("Three biggest plugins:  %s" % bp_out)

+            self.ui_log.info("")

+            self.ui_log.info("Please note the estimation is relevant to the "

+                             "current options.")

+            self.ui_log.info("Be aware that the real disk space requirements "

+                             "might be different.")

+            self.ui_log.info("")

+

         if self.opts.upload or self.opts.upload_url:

             if not self.opts.build:

                 try:

-- 

2.31.1

From 7ae47e6c0717c0b56c3368008dd99a87f7f436d5 Mon Sep 17 00:00:00 2001

From: Pavel Moravec <pmoravec@redhat.com>

Date: Wed, 13 Oct 2021 20:21:16 +0200

Subject: [PATCH] [report] Count with sos_logs and sos_reports in

 --estimate-only

Currently, we estimate just plugins' disk space and ignore sos_logs

or sos_reports directories - although they can occupy nontrivial disk

space as well.

Resolves: #2723

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

---

 sos/report/__init__.py | 8 ++++++++

 1 file changed, 8 insertions(+)

diff --git a/sos/report/__init__.py b/sos/report/__init__.py

index e35c7e8d..7feb31ee 100644

--- a/sos/report/__init__.py

+++ b/sos/report/__init__.py

@@ -1380,6 +1380,14 @@ class SoSReport(SoSComponent):

         if self.opts.estimate_only:

             from sos.utilities import get_human_readable

+            from pathlib import Path

+            # add sos_logs, sos_reports dirs, etc., basically everything

+            # that remained in self.tmpdir after plugins' contents removal

+            # that still will be moved to the sos report final directory path

+            tmpdir_path = Path(self.tmpdir)

+            self.estimated_plugsizes['sos_logs_reports'] = sum(

+                    [f.stat().st_size for f in tmpdir_path.glob('**/*')])

+

             _sum = get_human_readable(sum(self.estimated_plugsizes.values()))

             self.ui_log.info("Estimated disk space requirement for whole "

                              "uncompressed sos report directory: %s" % _sum)

-- 

2.31.1

From 4293f3317505661e8f32ba94ad87310996fa1626 Mon Sep 17 00:00:00 2001

From: Eric Desrochers <eric.desrochers@canonical.com>

Date: Tue, 19 Oct 2021 12:18:40 -0400

Subject: [PATCH] [report] check for symlink before rmtree when opt

 estimate-only is use

Check if the dir is also symlink before performing rmtree()

method so that unlink() method can be used instead.

Traceback (most recent call last):

  File "./bin/sos", line 22, in <module>

    sos.execute()

  File "/tmp/sos/sos/__init__.py", line 186, in execute

    self._component.execute()

OSError: Cannot call rmtree on a symbolic link

Closes: #2727

Signed-off-by: Eric Desrochers <eric.desrochers@canonical.com>

---

 sos/report/__init__.py | 2 +-

 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sos/report/__init__.py b/sos/report/__init__.py

index 7feb31ee..1b5bc97d 100644

--- a/sos/report/__init__.py

+++ b/sos/report/__init__.py

@@ -1059,7 +1059,7 @@ class SoSReport(SoSComponent):

             # deletion of some dirs but deleting their content

             for f in os.listdir(self.archive.get_tmp_dir()):

                 f = os.path.join(self.archive.get_tmp_dir(), f)

-                if os.path.isdir(f):

+                if os.path.isdir(f) and not os.path.islink(f):

                     rmtree(f)

                 else:

                     os.unlink(f)

-- 

2.31.1