From 5b245b1e449c6a05d09034bcb8290bffded79327 Mon Sep 17 00:00:00 2001
From: Pavel Moravec <pmoravec@redhat.com>
Date: Wed, 8 Sep 2021 17:04:58 +0200
Subject: [PATCH] [report] Implement --estimate-only
Add report option --estimate-only to estimate disk space requirements
when running a sos report.
Resolves: #2673
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
---
man/en/sos-report.1 | 13 +++++++-
sos/report/__init__.py | 74 ++++++++++++++++++++++++++++++++++++++++--
2 files changed, 84 insertions(+), 3 deletions(-)
diff --git a/man/en/sos-report.1 b/man/en/sos-report.1
index 36b337df..e8efc8f8 100644
--- a/man/en/sos-report.1
+++ b/man/en/sos-report.1
@@ -14,7 +14,7 @@ sos report \- Collect and package diagnostic and support data
[--preset preset] [--add-preset add_preset]\fR
[--del-preset del_preset] [--desc description]\fR
[--batch] [--build] [--debug] [--dry-run]\fR
- [--label label] [--case-id id]\fR
+ [--estimate-only] [--label label] [--case-id id]\fR
[--threads threads]\fR
[--plugin-timeout TIMEOUT]\fR
[--cmd-timeout TIMEOUT]\fR
@@ -317,6 +317,17 @@ output, or string data from the system. The resulting logs may be used
to understand the actions that sos would have taken without the dry run
option.
.TP
+.B \--estimate-only
+Estimate disk space requirements when running sos report. This can be valuable
+to prevent sosreport working dir to consume all free disk space. No plugin data
+is available at the end.
+
+Plugins will be collected sequentially, size of collected files and commands outputs
+will be calculated and the plugin files will be immediatelly deleted prior execution
+of the next plugin. This still can consume whole free disk space, though. Please note,
+size estimations may not be accurate for highly utilized systems due to changes between
+an estimate and a real execution.
+.TP
.B \--upload
If specified, attempt to upload the resulting archive to a vendor defined location.
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
index 82484f1d..b033f621 100644
--- a/sos/report/__init__.py
+++ b/sos/report/__init__.py
@@ -86,6 +86,7 @@ class SoSReport(SoSComponent):
'desc': '',
'domains': [],
'dry_run': False,
+ 'estimate_only': False,
'experimental': False,
'enable_plugins': [],
'keywords': [],
@@ -137,6 +138,7 @@ class SoSReport(SoSComponent):
self._args = args
self.sysroot = "/"
self.preset = None
+ self.estimated_plugsizes = {}
self.print_header()
self._set_debug()
@@ -223,6 +225,11 @@ class SoSReport(SoSComponent):
help="Description for a new preset",)
report_grp.add_argument("--dry-run", action="store_true",
help="Run plugins but do not collect data")
+ report_grp.add_argument("--estimate-only", action="store_true",
+ help="Approximate disk space requirements for "
+ "a real sos run; disables --clean and "
+ "--collect, sets --threads=1 and "
+ "--no-postproc")
report_grp.add_argument("--experimental", action="store_true",
dest="experimental", default=False,
help="enable experimental plugins")
@@ -700,6 +700,33 @@ class SoSReport(SoSComponent):
self.all_options.append((plugin, plugin_name, optname,
optparm))
+ def _set_estimate_only(self):
+ # set estimate-only mode by enforcing some options settings
+ # and return a corresponding log messages string
+ msg = "\nEstimate-only mode enabled"
+ ext_msg = []
+ if self.opts.threads > 1:
+ ext_msg += ["--threads=%s overriden to 1" % self.opts.threads, ]
+ self.opts.threads = 1
+ if not self.opts.build:
+ ext_msg += ["--build enabled", ]
+ self.opts.build = True
+ if not self.opts.no_postproc:
+ ext_msg += ["--no-postproc enabled", ]
+ self.opts.no_postproc = True
+ if self.opts.clean:
+ ext_msg += ["--clean disabled", ]
+ self.opts.clean = False
+ if self.opts.upload:
+ ext_msg += ["--upload* options disabled", ]
+ self.opts.upload = False
+ if ext_msg:
+ msg += ", which overrides some options:\n " + "\n ".join(ext_msg)
+ else:
+ msg += "."
+ msg += "\n\n"
+ return msg
+
def _report_profiles_and_plugins(self):
self.ui_log.info("")
if len(self.loaded_plugins):
@@ -875,10 +909,12 @@ class SoSReport(SoSComponent):
return True
def batch(self):
+ msg = self.policy.get_msg()
+ if self.opts.estimate_only:
+ msg += self._set_estimate_only()
if self.opts.batch:
- self.ui_log.info(self.policy.get_msg())
+ self.ui_log.info(msg)
else:
- msg = self.policy.get_msg()
msg += _("Press ENTER to continue, or CTRL-C to quit.\n")
try:
input(msg)
@@ -1011,6 +1047,22 @@ class SoSReport(SoSComponent):
self.running_plugs.remove(plugin[1])
self.loaded_plugins[plugin[0]-1][1].set_timeout_hit()
pool._threads.clear()
+ if self.opts.estimate_only:
+ from pathlib import Path
+ tmpdir_path = Path(self.archive.get_tmp_dir())
+ self.estimated_plugsizes[plugin[1]] = sum(
+ [f.stat().st_size for f in tmpdir_path.glob('**/*')
+ if (os.path.isfile(f) and not os.path.islink(f))])
+ # remove whole tmp_dir content - including "sos_commands" and
+ # similar dirs that will be re-created on demand by next plugin
+ # if needed; it is less error-prone approach than skipping
+ # deletion of some dirs but deleting their content
+ for f in os.listdir(self.archive.get_tmp_dir()):
+ f = os.path.join(self.archive.get_tmp_dir(), f)
+ if os.path.isdir(f):
+ rmtree(f)
+ else:
+ os.unlink(f)
return True
def collect_plugin(self, plugin):
@@ -1330,6 +1382,24 @@ class SoSReport(SoSComponent):
self.policy.display_results(archive, directory, checksum,
map_file=map_file)
+ if self.opts.estimate_only:
+ from sos.utilities import get_human_readable
+ _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
+ self.ui_log.info("Estimated disk space requirement for whole "
+ "uncompressed sos report directory: %s" % _sum)
+ bigplugins = sorted(self.estimated_plugsizes.items(),
+ key=lambda x: x[1], reverse=True)[:3]
+ bp_out = ", ".join("%s: %s" %
+ (p, get_human_readable(v, precision=0))
+ for p, v in bigplugins)
+ self.ui_log.info("Three biggest plugins: %s" % bp_out)
+ self.ui_log.info("")
+ self.ui_log.info("Please note the estimation is relevant to the "
+ "current options.")
+ self.ui_log.info("Be aware that the real disk space requirements "
+ "might be different.")
+ self.ui_log.info("")
+
if self.opts.upload or self.opts.upload_url:
if not self.opts.build:
try:
--
2.31.1
From 7ae47e6c0717c0b56c3368008dd99a87f7f436d5 Mon Sep 17 00:00:00 2001
From: Pavel Moravec <pmoravec@redhat.com>
Date: Wed, 13 Oct 2021 20:21:16 +0200
Subject: [PATCH] [report] Count with sos_logs and sos_reports in
--estimate-only
Currently, we estimate just plugins' disk space and ignore sos_logs
or sos_reports directories - although they can occupy nontrivial disk
space as well.
Resolves: #2723
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
---
sos/report/__init__.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
index e35c7e8d..7feb31ee 100644
--- a/sos/report/__init__.py
+++ b/sos/report/__init__.py
@@ -1380,6 +1380,14 @@ class SoSReport(SoSComponent):
if self.opts.estimate_only:
from sos.utilities import get_human_readable
+ from pathlib import Path
+ # add sos_logs, sos_reports dirs, etc., basically everything
+ # that remained in self.tmpdir after plugins' contents removal
+ # that still will be moved to the sos report final directory path
+ tmpdir_path = Path(self.tmpdir)
+ self.estimated_plugsizes['sos_logs_reports'] = sum(
+ [f.stat().st_size for f in tmpdir_path.glob('**/*')])
+
_sum = get_human_readable(sum(self.estimated_plugsizes.values()))
self.ui_log.info("Estimated disk space requirement for whole "
"uncompressed sos report directory: %s" % _sum)
--
2.31.1
From 4293f3317505661e8f32ba94ad87310996fa1626 Mon Sep 17 00:00:00 2001
From: Eric Desrochers <eric.desrochers@canonical.com>
Date: Tue, 19 Oct 2021 12:18:40 -0400
Subject: [PATCH] [report] check for symlink before rmtree when opt
estimate-only is use
Check if the dir is also symlink before performing rmtree()
method so that unlink() method can be used instead.
Traceback (most recent call last):
File "./bin/sos", line 22, in <module>
sos.execute()
File "/tmp/sos/sos/__init__.py", line 186, in execute
self._component.execute()
OSError: Cannot call rmtree on a symbolic link
Closes: #2727
Signed-off-by: Eric Desrochers <eric.desrochers@canonical.com>
---
sos/report/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
index 7feb31ee..1b5bc97d 100644
--- a/sos/report/__init__.py
+++ b/sos/report/__init__.py
@@ -1059,7 +1059,7 @@ class SoSReport(SoSComponent):
# deletion of some dirs but deleting their content
for f in os.listdir(self.archive.get_tmp_dir()):
f = os.path.join(self.archive.get_tmp_dir(), f)
- if os.path.isdir(f):
+ if os.path.isdir(f) and not os.path.islink(f):
rmtree(f)
else:
os.unlink(f)
--
2.31.1