From 348ed379209aacfe6c1f870a3a68e06b39a51f06 Mon Sep 17 00:00:00 2001

From: "Bryn M. Reeves" <bmr@redhat.com>

Date: Tue, 9 Dec 2014 17:30:06 +0000

Subject: [PATCH 21/93] [global] make all utf-8 handling use errors='ignore'

Stop playing whack-a-mole with unicode handling and ignore all

invalid characters. It's not really possible to ensure that we

always get strict utf-8 data from the system - e.g. dmesg on

systems with broken BIOS strings will often spit undecodable

byte sequences.

Signed-off-by: Bryn M. Reeves <bmr@redhat.com>

---

 sos/archive.py          | 4 ++--

 sos/plugins/__init__.py | 5 +++--

 sos/utilities.py        | 7 +++++--

 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/sos/archive.py b/sos/archive.py

index 6063625..0e019bf 100644

--- a/sos/archive.py

+++ b/sos/archive.py

@@ -406,9 +406,9 @@ class TarFileArchive(FileCacheArchive):

                           close_fds=True)

                 stdout, stderr = p.communicate()

                 if stdout:

-                    self.log_info(stdout.decode('utf-8'))

+                    self.log_info(stdout.decode('utf-8', 'ignore'))

                 if stderr:

-                    self.log_error(stderr.decode('utf-8'))

+                    self.log_error(stderr.decode('utf-8', 'ignore'))

                 self._suffix += suffix

                 return self.name()

             except Exception as e:

diff --git a/sos/plugins/__init__.py b/sos/plugins/__init__.py

index b289144..4fd85be 100644

--- a/sos/plugins/__init__.py

+++ b/sos/plugins/__init__.py

@@ -542,7 +542,7 @@ class Plugin(object):

     def add_string_as_file(self, content, filename):

         """Add a string to the archive as a file named `filename`"""

         self.copy_strings.append((content, filename))

-        content = "..." + (content.splitlines()[0]).decode('utf8')

+        content = "..." + (content.splitlines()[0]).decode('utf8', 'ignore')

         self._log_debug("added string '%s' as '%s'" % (content, filename))

     def get_cmd_output_now(self, exe, suggest_filename=None,

@@ -611,7 +611,8 @@ class Plugin(object):

     def _collect_strings(self):

         for string, file_name in self.copy_strings:

-            content = "..." + (string.splitlines()[0]).decode('utf8')

+            content = "..."

+            content += (string.splitlines()[0]).decode('utf8', 'ignore')

             self._log_info("collecting string '%s' as '%s'"

                            % (content, file_name))

             try:

diff --git a/sos/utilities.py b/sos/utilities.py

index 7e8cd7e..8cb4ed6 100644

--- a/sos/utilities.py

+++ b/sos/utilities.py

@@ -140,7 +140,7 @@ def sos_get_command_output(command, timeout=300, runat=None):

     # shlex.split() reacts badly to unicode on older python runtimes.

     if not six.PY3:

-        command = command.encode('utf-8')

+        command = command.encode('utf-8', 'ignore')

     args = shlex.split(command)

     try:

         p = Popen(args, shell=False, stdout=PIPE, stderr=STDOUT,

@@ -159,7 +159,10 @@ def sos_get_command_output(command, timeout=300, runat=None):

     if p.returncode == 126 or p.returncode == 127:

         stdout = six.binary_type(b"")

-    return {'status': p.returncode, 'output': stdout.decode('utf-8')}

+    return {

+        'status': p.returncode,

+        'output': stdout.decode('utf-8', 'ignore')

+    }

 def import_module(module_fqname, superclasses=None):

-- 

1.9.3