From a0d7fc6560439fc6de7ae7d246604c7b8f605182 Mon Sep 17 00:00:00 2001

From: Pavel Moravec <pmoravec@redhat.com>

Date: Wed, 6 Oct 2021 14:05:53 +0200

Subject: [PATCH] [Red Hat] Update policy to use SFTP instead of legacy FTP

 dropbox

This is a backport of #2552 / #2467 to reflect on python2 systems the

planned FTP dropbox replacement by SFTP system.

Related: #2552

Resolves: #2715

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

---

 man/en/sosreport.1       |  14 +++++

 sos/__init__.py          |   6 +-

 sos/policies/__init__.py | 115 ++++++++++++++++++++++++++++++++++---

 sos/policies/redhat.py   | 121 ++++++++++++++++++++++++++++-----------

 sos/sosreport.py         |   3 +

 5 files changed, 215 insertions(+), 44 deletions(-)

diff --git a/man/en/sosreport.1 b/man/en/sosreport.1

index 4f7185f1..210f073f 100644

--- a/man/en/sosreport.1

+++ b/man/en/sosreport.1

@@ -32,6 +32,7 @@ sosreport \- Collect and package diagnostic and support data

           [--encrypt-pass PASS]\fR

           [--upload] [--upload-url url] [--upload-user user]\fR

           [--upload-directory dir] [--upload-pass pass]\fR

+          [--upload-protocol protocol]\fR

           [--experimental]\fR

           [-h|--help]\fR

@@ -314,6 +315,19 @@ when prompted rather than using this option.

 Specify a directory to upload to, if one is not specified by a vendor default location

 or if your destination server does not allow writes to '/'.

 .TP

+.B \--upload-protocol PROTO

+Manually specify the protocol to use for uploading to the target \fBupload-url\fR.

+

+Normally this is determined via the upload address, assuming that the protocol is part

+of the address provided, e.g. 'https://example.com'. By using this option, sos will skip

+the protocol check and use the method defined for the specified PROTO.

+

+For RHEL systems, setting this option to \fBsftp\fR will skip the initial attempt to

+upload to the Red Hat Customer Portal, and only attempt an upload to Red Hat's SFTP server,

+which is typically used as a fallback target.

+

+Valid values for PROTO are: 'auto' (default), 'https', 'ftp', 'sftp'.

+.TP

 .B \--experimental

 Enable plugins marked as experimental. Experimental plugins may not have

 been tested for this port or may still be under active development.

diff --git a/sos/__init__.py b/sos/__init__.py

index 111e066e..b4abf533 100644

--- a/sos/__init__.py

+++ b/sos/__init__.py

@@ -58,7 +58,7 @@ _arg_names = [

     'no_postproc', 'note', 'onlyplugins', 'plugin_timeout', 'plugopts',

     'preset', 'profiles', 'quiet', 'since', 'sysroot', 'threads', 'tmp_dir',

     'upload', 'upload_url', 'upload_directory', 'upload_user', 'upload_pass',

-    'verbosity', 'verify'

+    'upload_protocol', 'verbosity', 'verify'

 ]

 #: Arguments with non-zero default values

@@ -69,7 +69,8 @@ _arg_defaults = {

     "preset": "auto",

     # Verbosity has an explicit zero default since the ArgumentParser

     # count action default is None.

-    "verbosity": 0

+    "verbosity": 0,

+    "upload_protocol": "auto"

 }

@@ -200,6 +201,7 @@ class SoSOptions(object):

         self.upload_directory = ""

         self.upload_user = ""

         self.upload_pass = ""

+        self.upload_protocol = _arg_defaults["upload_protocol"]

         self.verbosity = _arg_defaults["verbosity"]

         self.verify = False

         self._nondefault = set()

diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py

index 1579ff5d..acda6379 100644

--- a/sos/policies/__init__.py

+++ b/sos/policies/__init__.py

@@ -15,7 +15,8 @@ from pwd import getpwuid

 from sos.utilities import (ImporterHelper,

                            import_module,

                            shell_out,

-                           sos_get_command_output)

+                           sos_get_command_output,

+                           is_executable)

 from sos.plugins import IndependentPlugin, ExperimentalPlugin

 from sos import _sos as _

 from sos import SoSOptions, _arg_names

@@ -944,6 +945,7 @@ class LinuxPolicy(Policy):

         self.upload_user = cmdline_opts.upload_user

         self.upload_directory = cmdline_opts.upload_directory

         self.upload_password = cmdline_opts.upload_pass

+        self.upload_archive_name = ''

         if not cmdline_opts.batch and not \

                 cmdline_opts.quiet:

@@ -1029,7 +1031,7 @@ class LinuxPolicy(Policy):

                                             this method

         """

-        self.upload_archive = archive

+        self.upload_archive_name = archive

         if not self.upload_url:

             self.upload_url = self.get_upload_url()

         if not self.upload_url:

@@ -1051,7 +1053,9 @@ class LinuxPolicy(Policy):

             'sftp': self.upload_sftp,

             'https': self.upload_https

         }

-        if '://' not in self.upload_url:

+        if self.commons['cmdlineopts'].upload_protocol in prots.keys():

+            return prots[self.commons['cmdlineopts'].upload_protocol]

+        elif '://' not in self.upload_url:

             raise Exception("Must provide protocol in upload URL")

         prot, url = self.upload_url.split('://')

         if prot not in prots.keys():

@@ -1092,7 +1096,7 @@ class LinuxPolicy(Policy):

         """

         return self.upload_password or self._upload_password

-    def upload_sftp(self):

+    def upload_sftp(self, user=None, password=None):

         """Attempts to upload the archive to an SFTP location.

         Due to the lack of well maintained, secure, and generally widespread

@@ -1102,7 +1106,102 @@ class LinuxPolicy(Policy):

         Do not override this method with one that uses python-paramiko, as the

         upstream sos team will reject any PR that includes that dependency.

         """

-        raise NotImplementedError("SFTP support is not yet implemented")

+        # if we somehow don't have sftp available locally, fail early

+        if not is_executable('sftp'):

+            raise Exception('SFTP is not locally supported')

+

+        # soft dependency on python-pexpect, which we need to use to control

+        # sftp login since as of this writing we don't have a viable solution

+        # via ssh python bindings commonly available among downstreams

+        try:

+            import pexpect

+        except ImportError:

+            raise Exception('SFTP upload requires python-pexpect, which is '

+                            'not currently installed')

+

+        sftp_connected = False

+

+        if not user:

+            user = self.get_upload_user()

+        if not password:

+            password = self.get_upload_password()

+

+        # need to strip the protocol prefix here

+        sftp_url = self.get_upload_url().replace('sftp://', '')

+        sftp_cmd = "sftp -oStrictHostKeyChecking=no %s@%s" % (user, sftp_url)

+        ret = pexpect.spawn(sftp_cmd, encoding='utf-8')

+

+        sftp_expects = [

+            u'sftp>',

+            u'password:',

+            u'Connection refused',

+            pexpect.TIMEOUT,

+            pexpect.EOF

+        ]

+

+        idx = ret.expect(sftp_expects, timeout=15)

+

+        if idx == 0:

+            sftp_connected = True

+        elif idx == 1:

+            ret.sendline(password)

+            pass_expects = [

+                u'sftp>',

+                u'Permission denied',

+                pexpect.TIMEOUT,

+                pexpect.EOF

+            ]

+            sftp_connected = ret.expect(pass_expects, timeout=10) == 0

+            if not sftp_connected:

+                ret.close()

+                raise Exception("Incorrect username or password for %s"

+                                % self.get_upload_url_string())

+        elif idx == 2:

+            raise Exception("Connection refused by %s. Incorrect port?"

+                            % self.get_upload_url_string())

+        elif idx == 3:

+            raise Exception("Timeout hit trying to connect to %s"

+                            % self.get_upload_url_string())

+        elif idx == 4:

+            raise Exception("Unexpected error trying to connect to sftp: %s"

+                            % ret.before)

+

+        if not sftp_connected:

+            ret.close()

+            raise Exception("Unable to connect via SFTP to %s"

+                            % self.get_upload_url_string())

+

+        put_cmd = 'put %s %s' % (self.upload_archive_name,

+                                 self._get_sftp_upload_name())

+        ret.sendline(put_cmd)

+

+        put_expects = [

+            u'100%',

+            pexpect.TIMEOUT,

+            pexpect.EOF

+        ]

+

+        put_success = ret.expect(put_expects, timeout=180)

+

+        if put_success == 0:

+            ret.sendline('bye')

+            return True

+        elif put_success == 1:

+            raise Exception("Timeout expired while uploading")

+        elif put_success == 2:

+            raise Exception("Unknown error during upload: %s" % ret.before)

+        else:

+            raise Exception("Unexpected response from server: %s" % ret.before)

+

+    def _get_sftp_upload_name(self):

+        """If a specific file name pattern is required by the SFTP server,

+        override this method in the relevant Policy. Otherwise the archive's

+        name on disk will be used

+

+        :returns:       Filename as it will exist on the SFTP server

+        :rtype:         ``str``

+        """

+        return self.upload_archive_name.split('/')[-1]

     def _upload_https_streaming(self, archive):

         """If upload_https() needs to use requests.put(), this method is used

@@ -1148,7 +1247,7 @@ class LinuxPolicy(Policy):

             raise Exception("Unable to upload due to missing python requests "

                             "library")

-        with open(self.upload_archive, 'rb') as arc:

+        with open(self.upload_archive_name, 'rb') as arc:

             if not self._use_https_streaming:

                 r = self._upload_https_no_stream(arc)

             else:

@@ -1214,9 +1313,9 @@ class LinuxPolicy(Policy):

                                 % directory)

         try:

-            with open(self.upload_archive, 'rb') as _arcfile:

+            with open(self.upload_archive_name, 'rb') as _arcfile:

                 session.storbinary(

-                    "STOR %s" % self.upload_archive.split('/')[-1],

+                    "STOR %s" % self.upload_archive_name.split('/')[-1],

                     _arcfile

                 )

             session.quit()

diff --git a/sos/policies/redhat.py b/sos/policies/redhat.py

index 3412f445..4da96894 100644

--- a/sos/policies/redhat.py

+++ b/sos/policies/redhat.py

@@ -10,6 +10,7 @@

 # This enables the use of with syntax in python 2.5 (e.g. jython)

 from __future__ import print_function

+import json

 import os

 import sys

 import re

@@ -19,6 +20,12 @@ from sos.policies import LinuxPolicy, PackageManager, PresetDefaults

 from sos import _sos as _

 from sos import SoSOptions

+try:

+    import requests

+    REQUESTS_LOADED = True

+except ImportError:

+    REQUESTS_LOADED = False

+

 OS_RELEASE = "/etc/os-release"

 # In python2.7, input() will not properly return strings, and on python3.x

@@ -44,9 +51,8 @@ class RedHatPolicy(LinuxPolicy):

     _host_sysroot = '/'

     default_scl_prefix = '/opt/rh'

     name_pattern = 'friendly'

-    upload_url = 'dropbox.redhat.com'

-    upload_user = 'anonymous'

-    upload_directory = '/incoming'

+    upload_url = None

+    upload_user = None

     def __init__(self, sysroot=None):

         super(RedHatPolicy, self).__init__(sysroot=sysroot)

@@ -252,7 +258,7 @@ No changes will be made to system configuration.

 """

 RH_API_HOST = "https://access.redhat.com"

-RH_FTP_HOST = "ftp://dropbox.redhat.com"

+RH_SFTP_HOST = "sftp://sftp.access.redhat.com"

 class RHELPolicy(RedHatPolicy):

@@ -268,9 +274,7 @@ An archive containing the collected information will be \

 generated in %(tmpdir)s and may be provided to a %(vendor)s \

 support representative.

 """ + disclaimer_text + "%(vendor_text)s\n")

-    _upload_url = RH_FTP_HOST

-    _upload_user = 'anonymous'

-    _upload_directory = '/incoming'

+    _upload_url = RH_SFTP_HOST

     def __init__(self, sysroot=None):

         super(RHELPolicy, self).__init__(sysroot=sysroot)

@@ -309,33 +313,17 @@ support representative.

             return

         if self.case_id:

             self.upload_user = input(_(

-                "Enter your Red Hat Customer Portal username (empty to use "

-                "public dropbox): ")

+                "Enter your Red Hat Customer Portal username for uploading ["

+                "empty for anonymous SFTP]: ")

             )

-            if not self.upload_user:

-                self.upload_url = RH_FTP_HOST

-                self.upload_user = self._upload_user

-

-    def _upload_user_set(self):

-        user = self.get_upload_user()

-        return user and (user != 'anonymous')

     def get_upload_url(self):

         if self.upload_url:

             return self.upload_url

-        if self.commons['cmdlineopts'].upload_url:

+        elif self.commons['cmdlineopts'].upload_url:

             return self.commons['cmdlineopts'].upload_url

-        # anonymous FTP server should be used as fallback when either:

-        # - case id is not set, or

-        # - upload user isn't set AND batch mode prevents to prompt for it

-        if (not self.case_id) or \

-           ((not self._upload_user_set()) and

-               self.commons['cmdlineopts'].batch):

-            self.upload_user = self._upload_user

-            if self.upload_directory is None:

-                self.upload_directory = self._upload_directory

-            self.upload_password = None

-            return RH_FTP_HOST

+        elif self.commons['cmdlineopts'].upload_protocol == 'sftp':

+            return RH_SFTP_HOST

         else:

             rh_case_api = "/hydra/rest/cases/%s/attachments"

             return RH_API_HOST + rh_case_api % self.case_id

@@ -348,13 +336,78 @@ support representative.

     def get_upload_url_string(self):

         if self.get_upload_url().startswith(RH_API_HOST):

             return "Red Hat Customer Portal"

-        return self.upload_url or RH_FTP_HOST

+        elif self.get_upload_url().startswith(RH_SFTP_HOST):

+            return "Red Hat Secure FTP"

+        return self.upload_url

-    def get_upload_user(self):

-        # if this is anything other than dropbox, annonymous won't work

-        if self.upload_url != RH_FTP_HOST:

-            return self.upload_user

-        return self._upload_user

+    def _get_sftp_upload_name(self):

+        """The RH SFTP server will only automatically connect file uploads to

+        cases if the filename _starts_ with the case number

+        """

+        if self.case_id:

+            return "%s_%s" % (self.case_id,

+                              self.upload_archive_name.split('/')[-1])

+        return self.upload_archive_name

+

+    def upload_sftp(self):

+        """Override the base upload_sftp to allow for setting an on-demand

+        generated anonymous login for the RH SFTP server if a username and

+        password are not given

+        """

+        if RH_SFTP_HOST.split('//')[1] not in self.get_upload_url():

+            return super(RHELPolicy, self).upload_sftp()

+

+        if not REQUESTS_LOADED:

+            raise Exception("python-requests is not installed and is required"

+                            " for obtaining SFTP auth token.")

+        _token = None

+        _user = None

+        # we have a username and password, but we need to reset the password

+        # to be the token returned from the auth endpoint

+        if self.get_upload_user() and self.get_upload_password():

+            url = RH_API_HOST + '/hydra/rest/v1/sftp/token'

+            auth = self.get_upload_https_auth()

+            ret = requests.get(url, auth=auth, timeout=10)

+            if ret.status_code == 200:

+                # credentials are valid

+                _user = self.get_upload_user()

+                _token = json.loads(ret.text)['token']

+            else:

+                print("Unable to retrieve Red Hat auth token using provided "

+                      "credentials. Will try anonymous.")

+        # we either do not have a username or password/token, or both

+        if not _token:

+            aurl = RH_API_HOST + '/hydra/rest/v1/sftp/token?isAnonymous=true'

+            anon = requests.get(aurl, timeout=10)

+            if anon.status_code == 200:

+                resp = json.loads(anon.text)

+                _user = resp['username']

+                _token = resp['token']

+                print("Using anonymous user %s for upload. Please inform your "

+                      "support engineer." % _user)

+        if _user and _token:

+            return super(RHELPolicy, self).upload_sftp(user=_user,

+                                                       password=_token)

+        raise Exception("Could not retrieve valid or anonymous credentials")

+

+    def upload_archive(self, archive):

+        """Override the base upload_archive to provide for automatic failover

+        from RHCP failures to the public RH dropbox

+        """

+        try:

+            if not self.get_upload_user() or not self.get_upload_password():

+                self.upload_url = RH_SFTP_HOST

+            uploaded = super(RHELPolicy, self).upload_archive(archive)

+        except Exception:

+            uploaded = False

+            if not self.upload_url.startswith(RH_API_HOST):

+                raise

+            else:

+                print("Upload to Red Hat Customer Portal failed. Trying %s"

+                      % RH_SFTP_HOST)

+                self.upload_url = RH_SFTP_HOST

+                uploaded = super(RHELPolicy, self).upload_archive(archive)

+        return uploaded

     def dist_version(self):

         try:

diff --git a/sos/sosreport.py b/sos/sosreport.py

index 5f3fb411..170129ea 100644

--- a/sos/sosreport.py

+++ b/sos/sosreport.py

@@ -255,6 +255,9 @@ def _get_parser():

                         help="Username to authenticate to upload server with")

     parser.add_argument("--upload-pass", default=None,

                         help="Password to authenticate to upload server with")

+    parser.add_argument("--upload-protocol", default='auto',

+                        choices=['auto', 'https', 'ftp', 'sftp'],

+                        help="Manually specify the upload protocol")

     # Group to make add/del preset exclusive

     preset_grp = parser.add_mutually_exclusive_group()

-- 

2.31.1

From 97de66bc3228b29ff33e5ba67733f15065705d89 Mon Sep 17 00:00:00 2001

From: root <root@bvassova-satellite69.gsslab.brq.redhat.com>

Date: Tue, 23 Nov 2021 11:44:33 +0100

Subject: [PATCH] [redhat] SFTP api change Backporting SFTP related changes

 from #2764 and #2772.

Related to: #2764, #2772

Resolves: #2771

Signed-off-by: Barbora Vassova <bvassova@redhat.com>

---

 sos/policies/__init__.py |  7 +++++--

 sos/policies/redhat.py   | 18 +++++++++---------

 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py

index acda63795..d78e79cfc 100644

--- a/sos/policies/__init__.py

+++ b/sos/policies/__init__.py

@@ -1178,7 +1178,8 @@ def upload_sftp(self, user=None, password=None):

         put_expects = [

             u'100%',

             pexpect.TIMEOUT,

-            pexpect.EOF

+            pexpect.EOF,

+            u'No such file or directory'

         ]

         put_success = ret.expect(put_expects, timeout=180)

@@ -1190,6 +1191,8 @@ def upload_sftp(self, user=None, password=None):

             raise Exception("Timeout expired while uploading")

         elif put_success == 2:

             raise Exception("Unknown error during upload: %s" % ret.before)

+        elif put_success == 3:

+            raise Exception("Unable to write archive to destination")

         else:

             raise Exception("Unexpected response from server: %s" % ret.before)

@@ -1252,7 +1255,7 @@ def upload_https(self):

                 r = self._upload_https_no_stream(arc)

             else:

                 r = self._upload_https_streaming(arc)

-            if r.status_code != 201:

+            if r.status_code != 200 and r.status_code != 201:

                 if r.status_code == 401:

                     raise Exception(

                         "Authentication failed: invalid user credentials"

diff --git a/sos/policies/redhat.py b/sos/policies/redhat.py

index 4da968945..960e7cd2a 100644

--- a/sos/policies/redhat.py

+++ b/sos/policies/redhat.py

@@ -257,7 +257,7 @@ def get_tmp_dir(self, opt_tmp_dir):

 No changes will be made to system configuration.

 """

-RH_API_HOST = "https://access.redhat.com"

+RH_API_HOST = "https://api.access.redhat.com"

 RH_SFTP_HOST = "sftp://sftp.access.redhat.com"

@@ -325,7 +325,7 @@ def get_upload_url(self):

         elif self.commons['cmdlineopts'].upload_protocol == 'sftp':

             return RH_SFTP_HOST

         else:

-            rh_case_api = "/hydra/rest/cases/%s/attachments"

+            rh_case_api = "/support/v1/cases/%s/attachments"

             return RH_API_HOST + rh_case_api % self.case_id

     def _get_upload_headers(self):

@@ -344,10 +344,10 @@ def _get_sftp_upload_name(self):

         """The RH SFTP server will only automatically connect file uploads to

         cases if the filename _starts_ with the case number

         """

+        fname = self.upload_archive_name.split('/')[-1]

         if self.case_id:

-            return "%s_%s" % (self.case_id,

-                              self.upload_archive_name.split('/')[-1])

-        return self.upload_archive_name

+            return "%s_%s" % (self.case_id, fname)

+        return fname

     def upload_sftp(self):

         """Override the base upload_sftp to allow for setting an on-demand

@@ -362,12 +362,12 @@ def upload_sftp(self):

                             " for obtaining SFTP auth token.")

         _token = None

         _user = None

+        url = RH_API_HOST + '/support/v2/sftp/token'

         # we have a username and password, but we need to reset the password

         # to be the token returned from the auth endpoint

         if self.get_upload_user() and self.get_upload_password():

-            url = RH_API_HOST + '/hydra/rest/v1/sftp/token'

             auth = self.get_upload_https_auth()

-            ret = requests.get(url, auth=auth, timeout=10)

+            ret = requests.post(url, auth=auth, timeout=10)

             if ret.status_code == 200:

                 # credentials are valid

                 _user = self.get_upload_user()

@@ -377,8 +377,8 @@ def upload_sftp(self):

                       "credentials. Will try anonymous.")

         # we either do not have a username or password/token, or both

         if not _token:

-            aurl = RH_API_HOST + '/hydra/rest/v1/sftp/token?isAnonymous=true'

-            anon = requests.get(aurl, timeout=10)

+            adata = {"isAnonymous": True}

+            anon = requests.post(url, data=json.dumps(adata), timeout=10)

             if anon.status_code == 200:

                 resp = json.loads(anon.text)

                 _user = resp['username']

From f231f9e502b2f98910c864c6c9000ae499280051 Mon Sep 17 00:00:00 2001

From: Jake Hunsaker <jhunsake@redhat.com>

Date: Tue, 4 Jan 2022 11:16:57 -0500

Subject: [PATCH] [report] Handle exceptionally old pexpect versions

Depending on configuration, certain downstreams may provide pexpect 2.3

or 4.6 (or later). The backport for SFTP upload support assumed a 4.x

version pexpect, however 2.x does not support the `encoding` parameter

to `pexpect.spawn()`.

Add a version check to determine if that parameter needs to be used or

not. Note that this does not need to be implemented against `main`, as

all supported downstreams for `main` support a minimum version of 4.x.

Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>

---

 sos/policies/__init__.py | 8 +++++++-

 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py

index d78e79cfc..562ccad25 100644

--- a/sos/policies/__init__.py

+++ b/sos/policies/__init__.py

@@ -1129,7 +1129,13 @@ def upload_sftp(self, user=None, password=None):

         # need to strip the protocol prefix here

         sftp_url = self.get_upload_url().replace('sftp://', '')

         sftp_cmd = "sftp -oStrictHostKeyChecking=no %s@%s" % (user, sftp_url)

-        ret = pexpect.spawn(sftp_cmd, encoding='utf-8')

+

+        if int(pexpect.__version__[0]) >= 4:

+            # newer expect requires decoding from subprocess

+            ret = pexpect.spawn(sftp_cmd, encoding='utf-8')

+        else:

+            # older pexpect does not

+            ret = pexpect.spawn(sftp_cmd)

         sftp_expects = [

             u'sftp>',