From 5b245b1e449c6a05d09034bcb8290bffded79327 Mon Sep 17 00:00:00 2001

From: Pavel Moravec <pmoravec@redhat.com>

Date: Wed, 8 Sep 2021 17:04:58 +0200

Subject: [PATCH] [report] Implement --estimate-only

Add report option --estimate-only to estimate disk space requirements

when running a sos report.

Resolves: #2673

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

---

 man/en/sos-report.1    | 13 +++++++-

 sos/report/__init__.py | 74 ++++++++++++++++++++++++++++++++++++++++--

 2 files changed, 84 insertions(+), 3 deletions(-)

diff --git a/man/en/sos-report.1 b/man/en/sos-report.1

index 36b337df..e8efc8f8 100644

--- a/man/en/sos-report.1

+++ b/man/en/sos-report.1

@@ -14,7 +14,7 @@ sos report \- Collect and package diagnostic and support data

           [--preset preset] [--add-preset add_preset]\fR

           [--del-preset del_preset] [--desc description]\fR

           [--batch] [--build] [--debug] [--dry-run]\fR

-          [--label label] [--case-id id]\fR

+          [--estimate-only] [--label label] [--case-id id]\fR

           [--threads threads]\fR

           [--plugin-timeout TIMEOUT]\fR

           [--cmd-timeout TIMEOUT]\fR

@@ -317,6 +317,17 @@ output, or string data from the system. The resulting logs may be used

 to understand the actions that sos would have taken without the dry run

 option.

 .TP

+.B \--estimate-only

+Estimate disk space requirements when running sos report. This can be valuable

+to prevent sosreport working dir to consume all free disk space. No plugin data

+is available at the end.

+

+Plugins will be collected sequentially, size of collected files and commands outputs

+will be calculated and the plugin files will be immediatelly deleted prior execution

+of the next plugin. This still can consume whole free disk space, though. Please note,

+size estimations may not be accurate for highly utilized systems due to changes between

+an estimate and a real execution.

+.TP

 .B \--upload

 If specified, attempt to upload the resulting archive to a vendor defined location.

diff --git a/sos/report/__init__.py b/sos/report/__init__.py

index 82484f1d..b033f621 100644

--- a/sos/report/__init__.py

+++ b/sos/report/__init__.py

@@ -86,6 +86,7 @@ class SoSReport(SoSComponent):

         'desc': '',

         'domains': [],

         'dry_run': False,

+        'estimate_only': False,

         'experimental': False,

         'enable_plugins': [],

         'keywords': [],

@@ -137,6 +138,7 @@ class SoSReport(SoSComponent):

         self._args = args

         self.sysroot = "/"

         self.preset = None

+        self.estimated_plugsizes = {}

         self.print_header()

         self._set_debug()

@@ -223,6 +225,11 @@ class SoSReport(SoSComponent):

                                 help="Description for a new preset",)

         report_grp.add_argument("--dry-run", action="store_true",

                                 help="Run plugins but do not collect data")

+        report_grp.add_argument("--estimate-only", action="store_true",

+                                help="Approximate disk space requirements for "

+                                     "a real sos run; disables --clean and "

+                                     "--collect, sets --threads=1 and "

+                                     "--no-postproc")

         report_grp.add_argument("--experimental", action="store_true",

                                 dest="experimental", default=False,

                                 help="enable experimental plugins")

@@ -700,6 +700,33 @@ class SoSReport(SoSComponent):

                 self.all_options.append((plugin, plugin_name, optname,

                                          optparm))

+    def _set_estimate_only(self):

+        # set estimate-only mode by enforcing some options settings

+        # and return a corresponding log messages string

+        msg = "\nEstimate-only mode enabled"

+        ext_msg = []

+        if self.opts.threads > 1:

+            ext_msg += ["--threads=%s overriden to 1" % self.opts.threads, ]

+            self.opts.threads = 1

+        if not self.opts.build:

+            ext_msg += ["--build enabled", ]

+            self.opts.build = True

+        if not self.opts.no_postproc:

+            ext_msg += ["--no-postproc enabled", ]

+            self.opts.no_postproc = True

+        if self.opts.clean:

+            ext_msg += ["--clean disabled", ]

+            self.opts.clean = False

+        if self.opts.upload:

+            ext_msg += ["--upload* options disabled", ]

+            self.opts.upload = False

+        if ext_msg:

+            msg += ", which overrides some options:\n  " + "\n  ".join(ext_msg)

+        else:

+            msg += "."

+        msg += "\n\n"

+        return msg

+

     def _report_profiles_and_plugins(self):

         self.ui_log.info("")

         if len(self.loaded_plugins):

@@ -875,10 +909,12 @@ class SoSReport(SoSComponent):

         return True

     def batch(self):

+        msg = self.policy.get_msg()

+        if self.opts.estimate_only:

+            msg += self._set_estimate_only()

         if self.opts.batch:

-            self.ui_log.info(self.policy.get_msg())

+            self.ui_log.info(msg)

         else:

-            msg = self.policy.get_msg()

             msg += _("Press ENTER to continue, or CTRL-C to quit.\n")

             try:

                 input(msg)

@@ -1011,6 +1047,22 @@ class SoSReport(SoSComponent):

                 self.running_plugs.remove(plugin[1])

                 self.loaded_plugins[plugin[0]-1][1].set_timeout_hit()

                 pool._threads.clear()

+        if self.opts.estimate_only:

+            from pathlib import Path

+            tmpdir_path = Path(self.archive.get_tmp_dir())

+            self.estimated_plugsizes[plugin[1]] = sum(

+                    [f.stat().st_size for f in tmpdir_path.glob('**/*')

+                     if (os.path.isfile(f) and not os.path.islink(f))])

+            # remove whole tmp_dir content - including "sos_commands" and

+            # similar dirs that will be re-created on demand by next plugin

+            # if needed; it is less error-prone approach than skipping

+            # deletion of some dirs but deleting their content

+            for f in os.listdir(self.archive.get_tmp_dir()):

+                f = os.path.join(self.archive.get_tmp_dir(), f)

+                if os.path.isdir(f):

+                    rmtree(f)

+                else:

+                    os.unlink(f)

         return True

     def collect_plugin(self, plugin):

@@ -1330,6 +1382,24 @@ class SoSReport(SoSComponent):

             self.policy.display_results(archive, directory, checksum,

                                         map_file=map_file)

+        if self.opts.estimate_only:

+            from sos.utilities import get_human_readable

+            _sum = get_human_readable(sum(self.estimated_plugsizes.values()))

+            self.ui_log.info("Estimated disk space requirement for whole "

+                             "uncompressed sos report directory: %s" % _sum)

+            bigplugins = sorted(self.estimated_plugsizes.items(),

+                                key=lambda x: x[1], reverse=True)[:3]

+            bp_out = ",  ".join("%s: %s" %

+                                (p, get_human_readable(v, precision=0))

+                                for p, v in bigplugins)

+            self.ui_log.info("Three biggest plugins:  %s" % bp_out)

+            self.ui_log.info("")

+            self.ui_log.info("Please note the estimation is relevant to the "

+                             "current options.")

+            self.ui_log.info("Be aware that the real disk space requirements "

+                             "might be different.")

+            self.ui_log.info("")

+

         if self.opts.upload or self.opts.upload_url:

             if not self.opts.build:

                 try:

-- 

2.31.1

From 7ae47e6c0717c0b56c3368008dd99a87f7f436d5 Mon Sep 17 00:00:00 2001

From: Pavel Moravec <pmoravec@redhat.com>

Date: Wed, 13 Oct 2021 20:21:16 +0200

Subject: [PATCH] [report] Count with sos_logs and sos_reports in

 --estimate-only

Currently, we estimate just plugins' disk space and ignore sos_logs

or sos_reports directories - although they can occupy nontrivial disk

space as well.

Resolves: #2723

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

---

 sos/report/__init__.py | 8 ++++++++

 1 file changed, 8 insertions(+)

diff --git a/sos/report/__init__.py b/sos/report/__init__.py

index e35c7e8d..7feb31ee 100644

--- a/sos/report/__init__.py

+++ b/sos/report/__init__.py

@@ -1380,6 +1380,14 @@ class SoSReport(SoSComponent):

         if self.opts.estimate_only:

             from sos.utilities import get_human_readable

+            from pathlib import Path

+            # add sos_logs, sos_reports dirs, etc., basically everything

+            # that remained in self.tmpdir after plugins' contents removal

+            # that still will be moved to the sos report final directory path

+            tmpdir_path = Path(self.tmpdir)

+            self.estimated_plugsizes['sos_logs_reports'] = sum(

+                    [f.stat().st_size for f in tmpdir_path.glob('**/*')])

+

             _sum = get_human_readable(sum(self.estimated_plugsizes.values()))

             self.ui_log.info("Estimated disk space requirement for whole "

                              "uncompressed sos report directory: %s" % _sum)

-- 

2.31.1

From 4293f3317505661e8f32ba94ad87310996fa1626 Mon Sep 17 00:00:00 2001

From: Eric Desrochers <eric.desrochers@canonical.com>

Date: Tue, 19 Oct 2021 12:18:40 -0400

Subject: [PATCH] [report] check for symlink before rmtree when opt

 estimate-only is use

Check if the dir is also symlink before performing rmtree()

method so that unlink() method can be used instead.

Traceback (most recent call last):

  File "./bin/sos", line 22, in <module>

    sos.execute()

  File "/tmp/sos/sos/__init__.py", line 186, in execute

    self._component.execute()

OSError: Cannot call rmtree on a symbolic link

Closes: #2727

Signed-off-by: Eric Desrochers <eric.desrochers@canonical.com>

---

 sos/report/__init__.py | 2 +-

 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sos/report/__init__.py b/sos/report/__init__.py

index 7feb31ee..1b5bc97d 100644

--- a/sos/report/__init__.py

+++ b/sos/report/__init__.py

@@ -1059,7 +1059,7 @@ class SoSReport(SoSComponent):

             # deletion of some dirs but deleting their content

             for f in os.listdir(self.archive.get_tmp_dir()):

                 f = os.path.join(self.archive.get_tmp_dir(), f)

-                if os.path.isdir(f):

+                if os.path.isdir(f) and not os.path.islink(f):

                     rmtree(f)

                 else:

                     os.unlink(f)

-- 

2.31.1

From 589d47c93257b55bc796ef6ac25b88c974ee3d72 Mon Sep 17 00:00:00 2001

From: Pavel Moravec <pmoravec@redhat.com>

Date: Mon, 8 Nov 2021 16:38:24 +0100

Subject: [PATCH] [report] Calculate sizes of dirs, symlinks and manifest in

 estimate mode

Enhance --estimate-mode to calculate sizes of also:

- symlinks

- directories themselves

- manifest.json file

Use os.lstat() method instead of os.stat() to properly calculate the

sizes (and not destinations of symlinks, e.g.).

Print five biggest plugins instead of three as sos logs and reports do

stand as one "plugin" in the list, often.

Resolves: #2752

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

---

 sos/report/__init__.py | 56 +++++++++++++++++++++---------------------

 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/sos/report/__init__.py b/sos/report/__init__.py

index 10952566..a4c92acc 100644

--- a/sos/report/__init__.py

+++ b/sos/report/__init__.py

@@ -1050,8 +1050,7 @@ class SoSReport(SoSComponent):

             from pathlib import Path

             tmpdir_path = Path(self.archive.get_tmp_dir())

             self.estimated_plugsizes[plugin[1]] = sum(

-                    [f.stat().st_size for f in tmpdir_path.glob('**/*')

-                     if (os.path.isfile(f) and not os.path.islink(f))])

+                    [f.lstat().st_size for f in tmpdir_path.glob('**/*')])

             # remove whole tmp_dir content - including "sos_commands" and

             # similar dirs that will be re-created on demand by next plugin

             # if needed; it is less error-prone approach than skipping

@@ -1273,6 +1272,33 @@ class SoSReport(SoSComponent):

                 short_name='manifest.json'

             )

+        # print results in estimate mode (to include also just added manifest)

+        if self.opts.estimate_only:

+            from sos.utilities import get_human_readable

+            from pathlib import Path

+            # add sos_logs, sos_reports dirs, etc., basically everything

+            # that remained in self.tmpdir after plugins' contents removal

+            # that still will be moved to the sos report final directory path

+            tmpdir_path = Path(self.tmpdir)

+            self.estimated_plugsizes['sos_logs_reports'] = sum(

+                    [f.lstat().st_size for f in tmpdir_path.glob('**/*')])

+

+            _sum = get_human_readable(sum(self.estimated_plugsizes.values()))

+            self.ui_log.info("Estimated disk space requirement for whole "

+                             "uncompressed sos report directory: %s" % _sum)

+            bigplugins = sorted(self.estimated_plugsizes.items(),

+                                key=lambda x: x[1], reverse=True)[:5]

+            bp_out = ",  ".join("%s: %s" %

+                                (p, get_human_readable(v, precision=0))

+                                for p, v in bigplugins)

+            self.ui_log.info("Five biggest plugins:  %s" % bp_out)

+            self.ui_log.info("")

+            self.ui_log.info("Please note the estimation is relevant to the "

+                             "current options.")

+            self.ui_log.info("Be aware that the real disk space requirements "

+                             "might be different.")

+            self.ui_log.info("")

+

         # package up and compress the results

         if not self.opts.build:

             old_umask = os.umask(0o077)

@@ -1377,32 +1403,6 @@ class SoSReport(SoSComponent):

             self.policy.display_results(archive, directory, checksum,

                                         map_file=map_file)

-        if self.opts.estimate_only:

-            from sos.utilities import get_human_readable

-            from pathlib import Path

-            # add sos_logs, sos_reports dirs, etc., basically everything

-            # that remained in self.tmpdir after plugins' contents removal

-            # that still will be moved to the sos report final directory path

-            tmpdir_path = Path(self.tmpdir)

-            self.estimated_plugsizes['sos_logs_reports'] = sum(

-                    [f.stat().st_size for f in tmpdir_path.glob('**/*')])

-

-            _sum = get_human_readable(sum(self.estimated_plugsizes.values()))

-            self.ui_log.info("Estimated disk space requirement for whole "

-                             "uncompressed sos report directory: %s" % _sum)

-            bigplugins = sorted(self.estimated_plugsizes.items(),

-                                key=lambda x: x[1], reverse=True)[:3]

-            bp_out = ",  ".join("%s: %s" %

-                                (p, get_human_readable(v, precision=0))

-                                for p, v in bigplugins)

-            self.ui_log.info("Three biggest plugins:  %s" % bp_out)

-            self.ui_log.info("")

-            self.ui_log.info("Please note the estimation is relevant to the "

-                             "current options.")

-            self.ui_log.info("Be aware that the real disk space requirements "

-                             "might be different.")

-            self.ui_log.info("")

-

         if self.opts.upload or self.opts.upload_url:

             if not self.opts.build:

                 try:

-- 

2.31.1

From c6a5bbb8d75aadd5c7f76d3f469929aba2cf8060 Mon Sep 17 00:00:00 2001

From: Pavel Moravec <pmoravec@redhat.com>

Date: Wed, 5 Jan 2022 10:33:58 +0100

Subject: [PATCH] [report] Provide better warning about estimate-mode

As --estimate-only calculates disk usage based on `stat` data that

differs from outputs of other commands like `du`, enhance the warning

about reliability of the calculated estimation.

Also add a rule-of-thumb recommendation of real disk space requirements.

Resolves: #2815

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

---

 man/en/sos-report.1    | 10 +++++++---

 sos/report/__init__.py |  3 ++-

 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/man/en/sos-report.1 b/man/en/sos-report.1

index 464a77e54..e34773986 100644

--- a/man/en/sos-report.1

+++ b/man/en/sos-report.1

@@ -343,9 +343,13 @@ is available at the end.

 Plugins will be collected sequentially, size of collected files and commands outputs

 will be calculated and the plugin files will be immediatelly deleted prior execution

-of the next plugin. This still can consume whole free disk space, though. Please note,

-size estimations may not be accurate for highly utilized systems due to changes between

-an estimate and a real execution.

+of the next plugin. This still can consume whole free disk space, though.

+

+Please note, size estimations may not be accurate for highly utilized systems due to

+changes between an estimate and a real execution. Also some difference between

+estimation (using `stat` command) and other commands used (i.e. `du`).

+

+A rule of thumb is to reserve at least double the estimation.

 .TP

 .B \--upload

 If specified, attempt to upload the resulting archive to a vendor defined location.

diff --git a/sos/report/__init__.py b/sos/report/__init__.py

index ef61fb344..e0617b45e 100644

--- a/sos/report/__init__.py

+++ b/sos/report/__init__.py

@@ -1330,7 +1330,8 @@ def final_work(self):

             self.ui_log.info("Please note the estimation is relevant to the "

                              "current options.")

             self.ui_log.info("Be aware that the real disk space requirements "

-                             "might be different.")

+                             "might be different. A rule of thumb is to "

+                             "reserve at least double the estimation.")

             self.ui_log.info("")

         # package up and compress the results