From 5b245b1e449c6a05d09034bcb8290bffded79327 Mon Sep 17 00:00:00 2001 From: Pavel Moravec Date: Wed, 8 Sep 2021 17:04:58 +0200 Subject: [PATCH] [report] Implement --estimate-only Add report option --estimate-only to estimate disk space requirements when running a sos report. Resolves: #2673 Signed-off-by: Pavel Moravec --- man/en/sos-report.1 | 13 +++++++- sos/report/__init__.py | 74 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/man/en/sos-report.1 b/man/en/sos-report.1 index 36b337df..e8efc8f8 100644 --- a/man/en/sos-report.1 +++ b/man/en/sos-report.1 @@ -14,7 +14,7 @@ sos report \- Collect and package diagnostic and support data [--preset preset] [--add-preset add_preset]\fR [--del-preset del_preset] [--desc description]\fR [--batch] [--build] [--debug] [--dry-run]\fR - [--label label] [--case-id id]\fR + [--estimate-only] [--label label] [--case-id id]\fR [--threads threads]\fR [--plugin-timeout TIMEOUT]\fR [--cmd-timeout TIMEOUT]\fR @@ -317,6 +317,17 @@ output, or string data from the system. The resulting logs may be used to understand the actions that sos would have taken without the dry run option. .TP +.B \--estimate-only +Estimate disk space requirements when running sos report. This can be valuable +to prevent sosreport working dir to consume all free disk space. No plugin data +is available at the end. + +Plugins will be collected sequentially, size of collected files and commands outputs +will be calculated and the plugin files will be immediatelly deleted prior execution +of the next plugin. This still can consume whole free disk space, though. Please note, +size estimations may not be accurate for highly utilized systems due to changes between +an estimate and a real execution. +.TP .B \--upload If specified, attempt to upload the resulting archive to a vendor defined location. diff --git a/sos/report/__init__.py b/sos/report/__init__.py index 82484f1d..b033f621 100644 --- a/sos/report/__init__.py +++ b/sos/report/__init__.py @@ -86,6 +86,7 @@ class SoSReport(SoSComponent): 'desc': '', 'domains': [], 'dry_run': False, + 'estimate_only': False, 'experimental': False, 'enable_plugins': [], 'keywords': [], @@ -137,6 +138,7 @@ class SoSReport(SoSComponent): self._args = args self.sysroot = "/" self.preset = None + self.estimated_plugsizes = {} self.print_header() self._set_debug() @@ -223,6 +225,11 @@ class SoSReport(SoSComponent): help="Description for a new preset",) report_grp.add_argument("--dry-run", action="store_true", help="Run plugins but do not collect data") + report_grp.add_argument("--estimate-only", action="store_true", + help="Approximate disk space requirements for " + "a real sos run; disables --clean and " + "--collect, sets --threads=1 and " + "--no-postproc") report_grp.add_argument("--experimental", action="store_true", dest="experimental", default=False, help="enable experimental plugins") @@ -700,6 +700,33 @@ class SoSReport(SoSComponent): self.all_options.append((plugin, plugin_name, optname, optparm)) + def _set_estimate_only(self): + # set estimate-only mode by enforcing some options settings + # and return a corresponding log messages string + msg = "\nEstimate-only mode enabled" + ext_msg = [] + if self.opts.threads > 1: + ext_msg += ["--threads=%s overriden to 1" % self.opts.threads, ] + self.opts.threads = 1 + if not self.opts.build: + ext_msg += ["--build enabled", ] + self.opts.build = True + if not self.opts.no_postproc: + ext_msg += ["--no-postproc enabled", ] + self.opts.no_postproc = True + if self.opts.clean: + ext_msg += ["--clean disabled", ] + self.opts.clean = False + if self.opts.upload: + ext_msg += ["--upload* options disabled", ] + self.opts.upload = False + if ext_msg: + msg += ", which overrides some options:\n " + "\n ".join(ext_msg) + else: + msg += "." + msg += "\n\n" + return msg + def _report_profiles_and_plugins(self): self.ui_log.info("") if len(self.loaded_plugins): @@ -875,10 +909,12 @@ class SoSReport(SoSComponent): return True def batch(self): + msg = self.policy.get_msg() + if self.opts.estimate_only: + msg += self._set_estimate_only() if self.opts.batch: - self.ui_log.info(self.policy.get_msg()) + self.ui_log.info(msg) else: - msg = self.policy.get_msg() msg += _("Press ENTER to continue, or CTRL-C to quit.\n") try: input(msg) @@ -1011,6 +1047,22 @@ class SoSReport(SoSComponent): self.running_plugs.remove(plugin[1]) self.loaded_plugins[plugin[0]-1][1].set_timeout_hit() pool._threads.clear() + if self.opts.estimate_only: + from pathlib import Path + tmpdir_path = Path(self.archive.get_tmp_dir()) + self.estimated_plugsizes[plugin[1]] = sum( + [f.stat().st_size for f in tmpdir_path.glob('**/*') + if (os.path.isfile(f) and not os.path.islink(f))]) + # remove whole tmp_dir content - including "sos_commands" and + # similar dirs that will be re-created on demand by next plugin + # if needed; it is less error-prone approach than skipping + # deletion of some dirs but deleting their content + for f in os.listdir(self.archive.get_tmp_dir()): + f = os.path.join(self.archive.get_tmp_dir(), f) + if os.path.isdir(f): + rmtree(f) + else: + os.unlink(f) return True def collect_plugin(self, plugin): @@ -1330,6 +1382,24 @@ class SoSReport(SoSComponent): self.policy.display_results(archive, directory, checksum, map_file=map_file) + if self.opts.estimate_only: + from sos.utilities import get_human_readable + _sum = get_human_readable(sum(self.estimated_plugsizes.values())) + self.ui_log.info("Estimated disk space requirement for whole " + "uncompressed sos report directory: %s" % _sum) + bigplugins = sorted(self.estimated_plugsizes.items(), + key=lambda x: x[1], reverse=True)[:3] + bp_out = ", ".join("%s: %s" % + (p, get_human_readable(v, precision=0)) + for p, v in bigplugins) + self.ui_log.info("Three biggest plugins: %s" % bp_out) + self.ui_log.info("") + self.ui_log.info("Please note the estimation is relevant to the " + "current options.") + self.ui_log.info("Be aware that the real disk space requirements " + "might be different.") + self.ui_log.info("") + if self.opts.upload or self.opts.upload_url: if not self.opts.build: try: -- 2.31.1 From 7ae47e6c0717c0b56c3368008dd99a87f7f436d5 Mon Sep 17 00:00:00 2001 From: Pavel Moravec Date: Wed, 13 Oct 2021 20:21:16 +0200 Subject: [PATCH] [report] Count with sos_logs and sos_reports in --estimate-only Currently, we estimate just plugins' disk space and ignore sos_logs or sos_reports directories - although they can occupy nontrivial disk space as well. Resolves: #2723 Signed-off-by: Pavel Moravec --- sos/report/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sos/report/__init__.py b/sos/report/__init__.py index e35c7e8d..7feb31ee 100644 --- a/sos/report/__init__.py +++ b/sos/report/__init__.py @@ -1380,6 +1380,14 @@ class SoSReport(SoSComponent): if self.opts.estimate_only: from sos.utilities import get_human_readable + from pathlib import Path + # add sos_logs, sos_reports dirs, etc., basically everything + # that remained in self.tmpdir after plugins' contents removal + # that still will be moved to the sos report final directory path + tmpdir_path = Path(self.tmpdir) + self.estimated_plugsizes['sos_logs_reports'] = sum( + [f.stat().st_size for f in tmpdir_path.glob('**/*')]) + _sum = get_human_readable(sum(self.estimated_plugsizes.values())) self.ui_log.info("Estimated disk space requirement for whole " "uncompressed sos report directory: %s" % _sum) -- 2.31.1 From 4293f3317505661e8f32ba94ad87310996fa1626 Mon Sep 17 00:00:00 2001 From: Eric Desrochers Date: Tue, 19 Oct 2021 12:18:40 -0400 Subject: [PATCH] [report] check for symlink before rmtree when opt estimate-only is use Check if the dir is also symlink before performing rmtree() method so that unlink() method can be used instead. Traceback (most recent call last): File "./bin/sos", line 22, in sos.execute() File "/tmp/sos/sos/__init__.py", line 186, in execute self._component.execute() OSError: Cannot call rmtree on a symbolic link Closes: #2727 Signed-off-by: Eric Desrochers --- sos/report/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sos/report/__init__.py b/sos/report/__init__.py index 7feb31ee..1b5bc97d 100644 --- a/sos/report/__init__.py +++ b/sos/report/__init__.py @@ -1059,7 +1059,7 @@ class SoSReport(SoSComponent): # deletion of some dirs but deleting their content for f in os.listdir(self.archive.get_tmp_dir()): f = os.path.join(self.archive.get_tmp_dir(), f) - if os.path.isdir(f): + if os.path.isdir(f) and not os.path.islink(f): rmtree(f) else: os.unlink(f) -- 2.31.1