From 348ed379209aacfe6c1f870a3a68e06b39a51f06 Mon Sep 17 00:00:00 2001 From: "Bryn M. Reeves" Date: Tue, 9 Dec 2014 17:30:06 +0000 Subject: [PATCH 21/93] [global] make all utf-8 handling use errors='ignore' Stop playing whack-a-mole with unicode handling and ignore all invalid characters. It's not really possible to ensure that we always get strict utf-8 data from the system - e.g. dmesg on systems with broken BIOS strings will often spit undecodable byte sequences. Signed-off-by: Bryn M. Reeves --- sos/archive.py | 4 ++-- sos/plugins/__init__.py | 5 +++-- sos/utilities.py | 7 +++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/sos/archive.py b/sos/archive.py index 6063625..0e019bf 100644 --- a/sos/archive.py +++ b/sos/archive.py @@ -406,9 +406,9 @@ class TarFileArchive(FileCacheArchive): close_fds=True) stdout, stderr = p.communicate() if stdout: - self.log_info(stdout.decode('utf-8')) + self.log_info(stdout.decode('utf-8', 'ignore')) if stderr: - self.log_error(stderr.decode('utf-8')) + self.log_error(stderr.decode('utf-8', 'ignore')) self._suffix += suffix return self.name() except Exception as e: diff --git a/sos/plugins/__init__.py b/sos/plugins/__init__.py index b289144..4fd85be 100644 --- a/sos/plugins/__init__.py +++ b/sos/plugins/__init__.py @@ -542,7 +542,7 @@ class Plugin(object): def add_string_as_file(self, content, filename): """Add a string to the archive as a file named `filename`""" self.copy_strings.append((content, filename)) - content = "..." + (content.splitlines()[0]).decode('utf8') + content = "..." + (content.splitlines()[0]).decode('utf8', 'ignore') self._log_debug("added string '%s' as '%s'" % (content, filename)) def get_cmd_output_now(self, exe, suggest_filename=None, @@ -611,7 +611,8 @@ class Plugin(object): def _collect_strings(self): for string, file_name in self.copy_strings: - content = "..." + (string.splitlines()[0]).decode('utf8') + content = "..." + content += (string.splitlines()[0]).decode('utf8', 'ignore') self._log_info("collecting string '%s' as '%s'" % (content, file_name)) try: diff --git a/sos/utilities.py b/sos/utilities.py index 7e8cd7e..8cb4ed6 100644 --- a/sos/utilities.py +++ b/sos/utilities.py @@ -140,7 +140,7 @@ def sos_get_command_output(command, timeout=300, runat=None): # shlex.split() reacts badly to unicode on older python runtimes. if not six.PY3: - command = command.encode('utf-8') + command = command.encode('utf-8', 'ignore') args = shlex.split(command) try: p = Popen(args, shell=False, stdout=PIPE, stderr=STDOUT, @@ -159,7 +159,10 @@ def sos_get_command_output(command, timeout=300, runat=None): if p.returncode == 126 or p.returncode == 127: stdout = six.binary_type(b"") - return {'status': p.returncode, 'output': stdout.decode('utf-8')} + return { + 'status': p.returncode, + 'output': stdout.decode('utf-8', 'ignore') + } def import_module(module_fqname, superclasses=None): -- 1.9.3