From 0b93d1f69ccfcc76e1896ea0e5ff7854be69be13 Mon Sep 17 00:00:00 2001

From: Pavel Moravec <pmoravec@redhat.com>

Date: Sat, 25 Nov 2017 12:47:35 +0100

Subject: [PATCH] [plugins] set proper PATH for SCL commands

As SCL packages are deployed under /opt/${provider}/${scl}/,

calling a SCL command needs that prefix in any path in PATH.

Consequently, distro-specific SCL default path prefix of the provider must be

defined in sos policies.

Relevant to: #1154

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

---

 sos/plugins/__init__.py  | 37 ++++++++++++++++++++++++++++++-------

 sos/policies/__init__.py |  4 ++++

 sos/policies/redhat.py   |  1 +

 3 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/sos/plugins/__init__.py b/sos/plugins/__init__.py

index aa69b19d..2a8bc516 100644

--- a/sos/plugins/__init__.py

+++ b/sos/plugins/__init__.py

@@ -1066,25 +1066,48 @@ class SCLPlugin(RedHatPlugin):

         output = sos_get_command_output("scl -l")["output"]

         return [scl.strip() for scl in output.splitlines()]

+    def convert_cmd_scl(self, scl, cmd):

+        """wrapping command in "scl enable" call and adds proper PATH

+        """

+        # load default SCL prefix to PATH

+        prefix = self.policy().get_default_scl_prefix()

+        # read prefix from /etc/scl/prefixes/${scl} and strip trailing '\n'

+        try:

+            prefix = open('/etc/scl/prefixes/%s' % scl, 'r').read()\

+                     .rstrip('\n')

+        except Exception as e:

+            self._log_error("Failed to find prefix for SCL %s, using %s"

+                            % (scl, prefix))

+

+        # expand PATH by equivalent prefixes under the SCL tree

+        path = os.environ["PATH"]

+        for p in path.split(':'):

+            path = '%s/%s%s:%s' % (prefix, scl, p, path)

+

+        scl_cmd = "scl enable %s \"PATH=%s %s\"" % (scl, path, cmd)

+        return scl_cmd

+

     def add_cmd_output_scl(self, scl, cmds, **kwargs):

         """Same as add_cmd_output, except that it wraps command in

-        "scl enable" call.

+        "scl enable" call and sets proper PATH.

         """

         if isinstance(cmds, six.string_types):

             cmds = [cmds]

         scl_cmds = []

-        scl_cmd_tpl = "scl enable %s \"%s\""

         for cmd in cmds:

-            scl_cmds.append(scl_cmd_tpl % (scl, cmd))

+            scl_cmds.append(convert_cmd_scl(scl, cmd))

         self.add_cmd_output(scl_cmds, **kwargs)

-    # config files for Software Collections are under /etc/opt/rh/${scl} and

-    # var files are under /var/opt/rh/${scl}. So we need to insert the paths

-    # after the appropriate root dir.

+    # config files for Software Collections are under /etc/${prefix}/${scl} and

+    # var files are under /var/${prefix}/${scl} where the ${prefix} is distro

+    # specific path. So we need to insert the paths after the appropriate root

+    # dir.

     def convert_copyspec_scl(self, scl, copyspec):

+        scl_prefix = self.policy().get_default_scl_prefix()

         for rootdir in ['etc', 'var']:

             p = re.compile('^/%s/' % rootdir)

-            copyspec = p.sub('/%s/opt/rh/%s/' % (rootdir, scl), copyspec)

+            copyspec = p.sub('/%s/%s/%s/' % (rootdir, scl_prefix, scl),

+                             copyspec)

         return copyspec

     def add_copy_spec_scl(self, scl, copyspecs):

diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py

index dffd801c..dc043105 100644

--- a/sos/policies/__init__.py

+++ b/sos/policies/__init__.py

@@ -194,6 +194,7 @@ No changes will be made to system configuration.

     vendor_url = "http://www.example.com/"

     vendor_text = ""

     PATH = ""

+    default_scl_prefix = ""

     _in_container = False

     _host_sysroot = '/'

@@ -271,6 +272,9 @@ No changes will be made to system configuration.

             return tempfile.gettempdir()

         return opt_tmp_dir

+    def get_default_scl_prefix(self):

+        return self.default_scl_prefix

+

     def match_plugin(self, plugin_classes):

         if len(plugin_classes) > 1:

             for p in plugin_classes:

diff --git a/sos/policies/redhat.py b/sos/policies/redhat.py

index c7449439..2dfe0589 100644

--- a/sos/policies/redhat.py

+++ b/sos/policies/redhat.py

@@ -44,6 +44,7 @@ class RedHatPolicy(LinuxPolicy):

     _rpmv_filter = ["debuginfo", "-devel"]

     _in_container = False

     _host_sysroot = '/'

+    default_scl_prefix = '/opt/rh'

     def __init__(self, sysroot=None):

         super(RedHatPolicy, self).__init__(sysroot=sysroot)

-- 

2.13.6

From 419ebe48ea408b6596ff4d7d9837079dc3057fcf Mon Sep 17 00:00:00 2001

From: Pavel Moravec <pmoravec@redhat.com>

Date: Sat, 25 Nov 2017 12:58:16 +0100

Subject: [PATCH] [postgresql] Call SCL pg_dump with proper path

Also stop storing pg_dump in an auxiliary tempdir but under regular

sos_commands/postgresql directory.

Resolves: #1154

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

---

 sos/plugins/postgresql.py | 43 ++++++++-----------------------------------

 1 file changed, 8 insertions(+), 35 deletions(-)

diff --git a/sos/plugins/postgresql.py b/sos/plugins/postgresql.py

index 45c87e89..9ba696be 100644

--- a/sos/plugins/postgresql.py

+++ b/sos/plugins/postgresql.py

@@ -34,8 +34,6 @@ class PostgreSQL(Plugin):

     packages = ('postgresql',)

-    tmp_dir = None

-

     password_warn_text = " (password visible in process listings)"

     option_list = [

@@ -47,11 +45,9 @@ class PostgreSQL(Plugin):

         ('dbport', 'database server port number', '', '5432')

     ]

-    def pg_dump(self, pg_dump_command="pg_dump", filename="sos_pgdump.tar"):

+    def do_pg_dump(self, scl=None, filename="pgdump.tar"):

         if self.get_option("dbname"):

             if self.get_option("password") or "PGPASSWORD" in os.environ:

-                self.tmp_dir = tempfile.mkdtemp()

-                dest_file = os.path.join(self.tmp_dir, filename)

                 # We're only modifying this for ourself and our children so

                 # there is no need to save and restore environment variables if

                 # the user decided to pass the password on the command line.

@@ -59,30 +55,21 @@ class PostgreSQL(Plugin):

                     os.environ["PGPASSWORD"] = str(self.get_option("password"))

                 if self.get_option("dbhost"):

-                    cmd = "%s -U %s -h %s -p %s -w -f %s -F t %s" % (

-                        pg_dump_command,

+                    cmd = "pg_dump -U %s -h %s -p %s -w -F t %s" % (

                         self.get_option("username"),

                         self.get_option("dbhost"),

                         self.get_option("dbport"),

-                        dest_file,

                         self.get_option("dbname")

                     )

                 else:

-                    cmd = "%s -C -U %s -w -f %s -F t %s " % (

-                        pg_dump_command,

+                    cmd = "pg_dump -C -U %s -w -F t %s " % (

                         self.get_option("username"),

-                        dest_file,

                         self.get_option("dbname")

                     )

-                result = self.call_ext_prog(cmd)

-                if (result['status'] == 0):

-                    self.add_copy_spec(dest_file)

-                else:

-                    self._log_info(

-                        "Unable to execute pg_dump. Error(%s)" %

-                        (result['output'])

-                    )

+                if scl is not None:

+                    cmd = self.convert_cmd_scl(scl, cmd)

+                self.add_cmd_output(cmd, suggest_filename=filename)

             else:  # no password in env or options

                 self.soslog.warning(

                     "password must be supplied to dump a database."

@@ -92,18 +79,7 @@ class PostgreSQL(Plugin):

                 )

     def setup(self):

-        self.pg_dump()

-

-    def postproc(self):

-        import shutil

-        if self.tmp_dir:

-            try:

-                shutil.rmtree(self.tmp_dir)

-            except shutil.Error:

-                self.soslog.exception(

-                    "Unable to remove %s." % (self.tmp_dir)

-                )

-                self.add_alert("ERROR: Unable to remove %s." % (self.tmp_dir))

+        self.do_pg_dump()

 class RedHatPostgreSQL(PostgreSQL, SCLPlugin):

@@ -140,10 +116,7 @@ class RedHatPostgreSQL(PostgreSQL, SCLPlugin):

         )

         if scl in self.scls_matched:

-            self.pg_dump(

-                pg_dump_command="scl enable rh-postgresql95 -- pg_dump",

-                filename="sos_scl_pgdump.tar"

-            )

+            self.do_pg_dump(scl=scl, filename="pgdump-scl-%s.tar" % scl)

 class DebianPostgreSQL(PostgreSQL, DebianPlugin, UbuntuPlugin):

-- 

2.13.6

From 3f0fa8ef20bcc8ec2fb1ff54815141813d07b033 Mon Sep 17 00:00:00 2001

From: Pavel Moravec <pmoravec@redhat.com>

Date: Wed, 20 Dec 2017 11:47:33 +0100

Subject: [PATCH] [plugins] allow add_cmd_output to collect binary output

If a command output is a true binary data, allow add_cmd_output to

collect the raw content and dont try to decode it as UTF-8.

Resolves: #1169

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

---

 sos/archive.py            | 11 ++++++++++

 sos/plugins/__init__.py   | 51 +++++++++++++++++++++++++++++++----------------

 sos/plugins/postgresql.py |  4 +++-

 sos/utilities.py          |  5 +++--

 4 files changed, 51 insertions(+), 20 deletions(-)

diff --git a/sos/archive.py b/sos/archive.py

index 607312a71..80e27b846 100644

--- a/sos/archive.py

+++ b/sos/archive.py

@@ -85,6 +85,9 @@ def add_file(self, src, dest=None):

     def add_string(self, content, dest):

         raise NotImplementedError

+    def add_binary(self, content, dest):

+        raise NotImplementedError

+

     def add_link(self, source, link_name):

         raise NotImplementedError

@@ -215,6 +218,14 @@ def add_string(self, content, dest):

         self.log_debug("added string at '%s' to FileCacheArchive '%s'"

                        % (src, self._archive_root))

+    def add_binary(self, content, dest):

+        dest = self.dest_path(dest)

+        self._check_path(dest)

+        f = codecs.open(dest, 'wb', encoding=None)

+        f.write(content)

+        self.log_debug("added binary content at '%s' to FileCacheArchive '%s'"

+                       % (dest, self._archive_root))

+

     def add_link(self, source, link_name):

         dest = self.dest_path(link_name)

         self._check_path(dest)

diff --git a/sos/plugins/__init__.py b/sos/plugins/__init__.py

index 2a8bc516e..0eccd40a1 100644

--- a/sos/plugins/__init__.py

+++ b/sos/plugins/__init__.py

@@ -222,6 +222,11 @@ def do_cmd_private_sub(self, cmd):

             for called in self.executed_commands:

                 if called['file'] is None:

                     continue

+                if called['binary'] == 'yes':

+                    self._log_warn(("Command output '%s' collected as " +

+                                    "binary, output isn't scrubbed despite " +

+                                    "asked for") % called['exe'])

+                    continue

                 if fnmatch.fnmatch(called['exe'], globstr):

                     path = os.path.join(self.commons['cmddir'], called['file'])

                     readable = self.archive.open_file(path)

@@ -260,6 +265,11 @@ def do_cmd_output_sub(self, cmd, regexp, subst):

                 # was anything collected?

                 if called['file'] is None:

                     continue

+                if called['binary'] == 'yes':

+                    self._log_warn(("Command output '%s' collected as " +

+                                    "binary, output isn't scrubbed despite " +

+                                    "asked for") % called['exe'])

+                    continue

                 if fnmatch.fnmatch(called['exe'], globstr):

                     path = os.path.join(self.commons['cmddir'], called['file'])

                     self._log_debug("applying substitution to '%s'" % path)

@@ -587,7 +597,8 @@ def getmtime(path):

                 self.archive.add_link(link_path, _file)

     def get_command_output(self, prog, timeout=300, stderr=True,

-                           chroot=True, runat=None, env=None):

+                           chroot=True, runat=None, env=None,

+                           binary=False):

         if chroot or self.commons['cmdlineopts'].chroot == 'always':

             root = self.sysroot

         else:

@@ -595,7 +606,7 @@ def get_command_output(self, prog, timeout=300, stderr=True,

         result = sos_get_command_output(prog, timeout=timeout, stderr=stderr,

                                         chroot=root, chdir=runat,

-                                        env=env)

+                                        env=env, binary=binary)

         if result['status'] == 124:

             self._log_warn("command '%s' timed out after %ds"

@@ -611,7 +622,8 @@ def get_command_output(self, prog, timeout=300, stderr=True,

                                    % (prog.split()[0], root))

                     return self.get_command_output(prog, timeout=timeout,

                                                    chroot=False, runat=runat,

-                                                   env=env)

+                                                   env=env,

+                                                   binary=binary)

             self._log_debug("could not run '%s': command not found" % prog)

         return result

@@ -632,14 +644,14 @@ def check_ext_prog(self, prog):

     def _add_cmd_output(self, cmd, suggest_filename=None,

                         root_symlink=None, timeout=300, stderr=True,

-                        chroot=True, runat=None, env=None):

+                        chroot=True, runat=None, env=None, binary=False):

         """Internal helper to add a single command to the collection list."""

         cmdt = (

             cmd, suggest_filename,

             root_symlink, timeout, stderr,

-            chroot, runat, env

+            chroot, runat, env, binary

         )

-        _tuplefmt = "('%s', '%s', '%s', %s, '%s', '%s', '%s', '%s')"

+        _tuplefmt = "('%s', '%s', '%s', %s, '%s', '%s', '%s', '%s', '%s')"

         _logstr = "packed command tuple: " + _tuplefmt

         self._log_debug(_logstr % cmdt)

         self.collect_cmds.append(cmdt)

@@ -647,7 +659,7 @@ def _add_cmd_output(self, cmd, suggest_filename=None,

     def add_cmd_output(self, cmds, suggest_filename=None,

                        root_symlink=None, timeout=300, stderr=True,

-                       chroot=True, runat=None, env=None):

+                       chroot=True, runat=None, env=None, binary=False):

         """Run a program or a list of programs and collect the output"""

         if isinstance(cmds, six.string_types):

             cmds = [cmds]

@@ -656,7 +668,7 @@ def add_cmd_output(self, cmds, suggest_filename=None,

         for cmd in cmds:

             self._add_cmd_output(cmd, suggest_filename,

                                  root_symlink, timeout, stderr,

-                                 chroot, runat, env)

+                                 chroot, runat, env, binary)

     def get_cmd_output_path(self, name=None, make=True):

         """Return a path into which this module should store collected

@@ -712,14 +724,15 @@ def add_string_as_file(self, content, filename):

     def get_cmd_output_now(self, exe, suggest_filename=None,

                            root_symlink=False, timeout=300, stderr=True,

-                           chroot=True, runat=None, env=None):

+                           chroot=True, runat=None, env=None,

+                           binary=False):

         """Execute a command and save the output to a file for inclusion in the

         report.

         """

         start = time()

         result = self.get_command_output(exe, timeout=timeout, stderr=stderr,

                                          chroot=chroot, runat=runat,

-                                         env=env)

+                                         env=env, binary=binary)

         self._log_debug("collected output of '%s' in %s"

                         % (exe.split()[0], time() - start))

@@ -729,13 +742,17 @@ def get_cmd_output_now(self, exe, suggest_filename=None,

             outfn = self._make_command_filename(exe)

         outfn_strip = outfn[len(self.commons['cmddir'])+1:]

-        self.archive.add_string(result['output'], outfn)

+        if binary:

+            self.archive.add_binary(result['output'], outfn)

+        else:

+            self.archive.add_string(result['output'], outfn)

         if root_symlink:

             self.archive.add_link(outfn, root_symlink)

         # save info for later

         # save in our list

-        self.executed_commands.append({'exe': exe, 'file': outfn_strip})

+        self.executed_commands.append({'exe': exe, 'file': outfn_strip,

+                                       'binary': 'yes' if binary else 'no'})

         self.commons['xmlreport'].add_command(cmdline=exe,

                                               exitcode=result['status'],

                                               f_stdout=outfn_strip)

@@ -839,16 +856,16 @@ def _collect_cmd_output(self):

                 timeout,

                 stderr,

                 chroot, runat,

-                env

+                env, binary

             ) = progs[0]

-            self._log_debug("unpacked command tuple: " +

-                            "('%s', '%s', '%s', %s, '%s', '%s', '%s', '%s')" %

-                            progs[0])

+            self._log_debug(("unpacked command tuple: " +

+                             "('%s', '%s', '%s', %s, '%s', '%s', '%s', '%s'," +

+                             "'%s')") % progs[0])

             self._log_info("collecting output of '%s'" % prog)

             self.get_cmd_output_now(prog, suggest_filename=suggest_filename,

                                     root_symlink=root_symlink, timeout=timeout,

                                     stderr=stderr, chroot=chroot, runat=runat,

-                                    env=env)

+                                    env=env, binary=binary)

     def _collect_strings(self):

         for string, file_name in self.copy_strings:

diff --git a/sos/plugins/postgresql.py b/sos/plugins/postgresql.py

index 9ba696be2..2e330c9b5 100644

--- a/sos/plugins/postgresql.py

+++ b/sos/plugins/postgresql.py

@@ -69,7 +69,9 @@ def do_pg_dump(self, scl=None, filename="pgdump.tar"):

                 if scl is not None:

                     cmd = self.convert_cmd_scl(scl, cmd)

-                self.add_cmd_output(cmd, suggest_filename=filename)

+                self.add_cmd_output(cmd, suggest_filename=filename,

+                                    binary=True)

+

             else:  # no password in env or options

                 self.soslog.warning(

                     "password must be supplied to dump a database."

diff --git a/sos/utilities.py b/sos/utilities.py

index 55bb1dc96..b5aa571b7 100644

--- a/sos/utilities.py

+++ b/sos/utilities.py

@@ -110,7 +110,8 @@ def is_executable(command):

 def sos_get_command_output(command, timeout=300, stderr=False,

-                           chroot=None, chdir=None, env=None):

+                           chroot=None, chdir=None, env=None,

+                           binary=False):

     """Execute a command and return a dictionary of status and output,

     optionally changing root or current working directory before

     executing command.

@@ -164,7 +165,7 @@ def _child_prep_fn():

     return {

         'status': p.returncode,

-        'output': stdout.decode('utf-8', 'ignore')

+        'output': stdout if binary else stdout.decode('utf-8', 'ignore')

     }