diff --git a/SOURCES/00189-add-rewheel-module.patch b/SOURCES/00189-add-rewheel-module.patch deleted file mode 100644 index 419d7e4..0000000 --- a/SOURCES/00189-add-rewheel-module.patch +++ /dev/null @@ -1,245 +0,0 @@ -diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py -index 4748ba4..986d5e9 100644 ---- a/Lib/ensurepip/__init__.py -+++ b/Lib/ensurepip/__init__.py -@@ -1,8 +1,10 @@ - import os - import os.path - import pkgutil -+import shutil - import sys - import tempfile -+from ensurepip import rewheel - - - __all__ = ["version", "bootstrap"] -@@ -24,8 +26,15 @@ def _run_pip(args, additional_paths=None): - sys.path = additional_paths + sys.path - - # Install the bundled software -- import pip._internal -- return pip._internal.main(args) -+ try: -+ # pip 10 -+ from pip._internal import main -+ except ImportError: -+ # pip 9 -+ from pip import main -+ if args[0] in ["install", "list", "wheel"]: -+ args.append('--pre') -+ return main(args) - - - def version(): -@@ -88,20 +97,39 @@ def _bootstrap(*, root=None, upgrade=False, user=False, - # omit pip and easy_install - os.environ["ENSUREPIP_OPTIONS"] = "install" - -+ whls = [] -+ rewheel_dir = None -+ # try to see if we have system-wide versions of _PROJECTS -+ dep_records = rewheel.find_system_records([p[0] for p in _PROJECTS]) -+ # TODO: check if system-wide versions are the newest ones -+ # if --upgrade is used? -+ if all(dep_records): -+ # if we have all _PROJECTS installed system-wide, we'll recreate -+ # wheels from them and install those -+ rewheel_dir = tempfile.TemporaryDirectory() -+ for dr in dep_records: -+ new_whl = rewheel.rewheel_from_record(dr, rewheel_dir.name) -+ whls.append(os.path.join(rewheel_dir.name, new_whl)) -+ else: -+ # if we don't have all the _PROJECTS installed system-wide, -+ # let's just fall back to bundled wheels -+ for project, version in _PROJECTS: -+ whl = os.path.join( -+ os.path.dirname(__file__), -+ "_bundled", -+ "{}-{}-py2.py3-none-any.whl".format(project, version) -+ ) -+ whls.append(whl) -+ - with tempfile.TemporaryDirectory() as tmpdir: - # Put our bundled wheels into a temporary directory and construct the - # additional paths that need added to sys.path - additional_paths = [] -- for project, version in _PROJECTS: -- wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) -- whl = pkgutil.get_data( -- "ensurepip", -- "_bundled/{}".format(wheel_name), -- ) -- with open(os.path.join(tmpdir, wheel_name), "wb") as fp: -- fp.write(whl) -- -- additional_paths.append(os.path.join(tmpdir, wheel_name)) -+ for whl in whls: -+ shutil.copy(whl, tmpdir) -+ additional_paths.append(os.path.join(tmpdir, os.path.basename(whl))) -+ if rewheel_dir: -+ rewheel_dir.cleanup() - - # Construct the arguments to be passed to the pip command - args = ["install", "--no-index", "--find-links", tmpdir] -diff --git a/Lib/ensurepip/rewheel/__init__.py b/Lib/ensurepip/rewheel/__init__.py -new file mode 100644 -index 0000000..753c764 ---- /dev/null -+++ b/Lib/ensurepip/rewheel/__init__.py -@@ -0,0 +1,143 @@ -+import argparse -+import codecs -+import csv -+import email.parser -+import os -+import io -+import re -+import site -+import subprocess -+import sys -+import zipfile -+ -+def run(): -+ parser = argparse.ArgumentParser(description='Recreate wheel of package with given RECORD.') -+ parser.add_argument('record_path', -+ help='Path to RECORD file') -+ parser.add_argument('-o', '--output-dir', -+ help='Dir where to place the wheel, defaults to current working dir.', -+ dest='outdir', -+ default=os.path.curdir) -+ -+ ns = parser.parse_args() -+ retcode = 0 -+ try: -+ print(rewheel_from_record(**vars(ns))) -+ except BaseException as e: -+ print('Failed: {}'.format(e)) -+ retcode = 1 -+ sys.exit(1) -+ -+def find_system_records(projects): -+ """Return list of paths to RECORD files for system-installed projects. -+ -+ If a project is not installed, the resulting list contains None instead -+ of a path to its RECORD -+ """ -+ records = [] -+ # get system site-packages dirs -+ sys_sitepack = site.getsitepackages([sys.base_prefix, sys.base_exec_prefix]) -+ sys_sitepack = [sp for sp in sys_sitepack if os.path.exists(sp)] -+ # try to find all projects in all system site-packages -+ for project in projects: -+ path = None -+ for sp in sys_sitepack: -+ dist_info_re = os.path.join(sp, project) + r'-[^\{0}]+\.dist-info'.format(os.sep) -+ candidates = [os.path.join(sp, p) for p in os.listdir(sp)] -+ # filter out candidate dirs based on the above regexp -+ filtered = [c for c in candidates if re.match(dist_info_re, c)] -+ # if we have 0 or 2 or more dirs, something is wrong... -+ if len(filtered) == 1: -+ path = filtered[0] -+ if path is not None: -+ records.append(os.path.join(path, 'RECORD')) -+ else: -+ records.append(None) -+ return records -+ -+def rewheel_from_record(record_path, outdir): -+ """Recreates a whee of package with given record_path and returns path -+ to the newly created wheel.""" -+ site_dir = os.path.dirname(os.path.dirname(record_path)) -+ record_relpath = record_path[len(site_dir):].strip(os.path.sep) -+ to_write, to_omit = get_records_to_pack(site_dir, record_relpath) -+ new_wheel_name = get_wheel_name(record_path) -+ new_wheel_path = os.path.join(outdir, new_wheel_name + '.whl') -+ -+ new_wheel = zipfile.ZipFile(new_wheel_path, mode='w', compression=zipfile.ZIP_DEFLATED) -+ # we need to write a new record with just the files that we will write, -+ # e.g. not binaries and *.pyc/*.pyo files -+ new_record = io.StringIO() -+ writer = csv.writer(new_record) -+ -+ # handle files that we can write straight away -+ for f, sha_hash, size in to_write: -+ new_wheel.write(os.path.join(site_dir, f), arcname=f) -+ writer.writerow([f, sha_hash,size]) -+ -+ # rewrite the old wheel file with a new computed one -+ writer.writerow([record_relpath, '', '']) -+ new_wheel.writestr(record_relpath, new_record.getvalue()) -+ -+ new_wheel.close() -+ -+ return new_wheel.filename -+ -+def get_wheel_name(record_path): -+ """Return proper name of the wheel, without .whl.""" -+ -+ wheel_info_path = os.path.join(os.path.dirname(record_path), 'WHEEL') -+ with codecs.open(wheel_info_path, encoding='utf-8') as wheel_info_file: -+ wheel_info = email.parser.Parser().parsestr(wheel_info_file.read()) -+ -+ metadata_path = os.path.join(os.path.dirname(record_path), 'METADATA') -+ with codecs.open(metadata_path, encoding='utf-8') as metadata_file: -+ metadata = email.parser.Parser().parsestr(metadata_file.read()) -+ -+ # construct name parts according to wheel spec -+ distribution = metadata.get('Name') -+ version = metadata.get('Version') -+ build_tag = '' # nothing for now -+ lang_tag = [] -+ for t in wheel_info.get_all('Tag'): -+ lang_tag.append(t.split('-')[0]) -+ lang_tag = '.'.join(lang_tag) -+ abi_tag, plat_tag = wheel_info.get('Tag').split('-')[1:3] -+ # leave out build tag, if it is empty -+ to_join = filter(None, [distribution, version, build_tag, lang_tag, abi_tag, plat_tag]) -+ return '-'.join(list(to_join)) -+ -+def get_records_to_pack(site_dir, record_relpath): -+ """Accepts path of sitedir and path of RECORD file relative to it. -+ Returns two lists: -+ - list of files that can be written to new RECORD straight away -+ - list of files that shouldn't be written or need some processing -+ (pyc and pyo files, scripts) -+ """ -+ record_file_path = os.path.join(site_dir, record_relpath) -+ with codecs.open(record_file_path, encoding='utf-8') as record_file: -+ record_contents = record_file.read() -+ # temporary fix for https://github.com/pypa/pip/issues/1376 -+ # we need to ignore files under ".data" directory -+ data_dir = os.path.dirname(record_relpath).strip(os.path.sep) -+ data_dir = data_dir[:-len('dist-info')] + 'data' -+ -+ to_write = [] -+ to_omit = [] -+ for l in record_contents.splitlines(): -+ spl = l.split(',') -+ if len(spl) == 3: -+ # new record will omit (or write differently): -+ # - abs paths, paths with ".." (entry points), -+ # - pyc+pyo files -+ # - the old RECORD file -+ # TODO: is there any better way to recognize an entry point? -+ if os.path.isabs(spl[0]) or spl[0].startswith('..') or \ -+ spl[0].endswith('.pyc') or spl[0].endswith('.pyo') or \ -+ spl[0] == record_relpath or spl[0].startswith(data_dir): -+ to_omit.append(spl) -+ else: -+ to_write.append(spl) -+ else: -+ pass # bad RECORD or empty line -+ return to_write, to_omit -diff --git a/Makefile.pre.in b/Makefile.pre.in -index 85e2ee3..4d34130 100644 ---- a/Makefile.pre.in -+++ b/Makefile.pre.in -@@ -1256,7 +1256,7 @@ LIBSUBDIRS= tkinter tkinter/test tkinter/test/test_tkinter \ - test/test_asyncio \ - collections concurrent concurrent/futures encodings \ - email email/mime test/test_email test/test_email/data \ -- ensurepip ensurepip/_bundled \ -+ ensurepip ensurepip/_bundled ensurepip/rewheel \ - html json test/test_json http dbm xmlrpc \ - sqlite3 sqlite3/test \ - logging csv wsgiref urllib \ diff --git a/SOURCES/00189-use-rpm-wheels.patch b/SOURCES/00189-use-rpm-wheels.patch new file mode 100644 index 0000000..6e12814 --- /dev/null +++ b/SOURCES/00189-use-rpm-wheels.patch @@ -0,0 +1,70 @@ +diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py +index 09c572d..167d27b 100644 +--- a/Lib/ensurepip/__init__.py ++++ b/Lib/ensurepip/__init__.py +@@ -1,16 +1,27 @@ ++import distutils.version ++import glob + import os + import os.path +-import pkgutil + import sys + import tempfile + + + __all__ = ["version", "bootstrap"] + ++_WHEEL_DIR = "/usr/share/python{}-wheels/".format(sys.version_info[0]) + +-_SETUPTOOLS_VERSION = "40.6.2" ++def _get_most_recent_wheel_version(pkg): ++ prefix = os.path.join(_WHEEL_DIR, "{}-".format(pkg)) ++ suffix = "-py2.py3-none-any.whl" ++ pattern = "{}*{}".format(prefix, suffix) ++ versions = (p[len(prefix):-len(suffix)] for p in glob.glob(pattern)) ++ return str(max(versions, key=distutils.version.LooseVersion)) ++ ++ ++_SETUPTOOLS_VERSION = _get_most_recent_wheel_version("setuptools") ++ ++_PIP_VERSION = _get_most_recent_wheel_version("pip") + +-_PIP_VERSION = "18.1" + + _PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION), +@@ -23,9 +34,15 @@ def _run_pip(args, additional_paths=None): + if additional_paths is not None: + sys.path = additional_paths + sys.path + +- # Install the bundled software +- import pip._internal +- return pip._internal.main(args) ++ try: ++ # pip 10 ++ from pip._internal import main ++ except ImportError: ++ # pip 9 ++ from pip import main ++ if args[0] in ["install", "list", "wheel"]: ++ args.append('--pre') ++ return main(args) + + + def version(): +@@ -94,12 +111,9 @@ def _bootstrap(*, root=None, upgrade=False, user=False, + additional_paths = [] + for project, version in _PROJECTS: + wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) +- whl = pkgutil.get_data( +- "ensurepip", +- "_bundled/{}".format(wheel_name), +- ) +- with open(os.path.join(tmpdir, wheel_name), "wb") as fp: +- fp.write(whl) ++ with open(os.path.join(_WHEEL_DIR, wheel_name), "rb") as sfp: ++ with open(os.path.join(tmpdir, wheel_name), "wb") as fp: ++ fp.write(sfp.read()) + + additional_paths.append(os.path.join(tmpdir, wheel_name)) + diff --git a/SOURCES/00317-CVE-2019-5010.patch b/SOURCES/00317-CVE-2019-5010.patch new file mode 100644 index 0000000..62e931e --- /dev/null +++ b/SOURCES/00317-CVE-2019-5010.patch @@ -0,0 +1,111 @@ +From c660debb97f4f422255a82fef2d77804552c043a Mon Sep 17 00:00:00 2001 +From: Christian Heimes +Date: Tue, 15 Jan 2019 18:16:30 +0100 +Subject: [PATCH] bpo-35746: Fix segfault in ssl's cert parser + +CVE-2019-5010, Fix a NULL pointer deref in ssl module. The cert parser did +not handle CRL distribution points with empty DP or URI correctly. A +malicious or buggy certificate can result into segfault. + +Signed-off-by: Christian Heimes +--- + Lib/test/talos-2019-0758.pem | 22 +++++++++++++++++++ + Lib/test/test_ssl.py | 22 +++++++++++++++++++ + .../2019-01-15-18-16-05.bpo-35746.nMSd0j.rst | 3 +++ + Modules/_ssl.c | 4 ++++ + 4 files changed, 51 insertions(+) + create mode 100644 Lib/test/talos-2019-0758.pem + create mode 100644 Misc/NEWS.d/next/Security/2019-01-15-18-16-05.bpo-35746.nMSd0j.rst + +diff --git a/Lib/test/talos-2019-0758.pem b/Lib/test/talos-2019-0758.pem +new file mode 100644 +index 000000000000..13b95a77fd8a +--- /dev/null ++++ b/Lib/test/talos-2019-0758.pem +@@ -0,0 +1,22 @@ ++-----BEGIN CERTIFICATE----- ++MIIDqDCCApKgAwIBAgIBAjALBgkqhkiG9w0BAQswHzELMAkGA1UEBhMCVUsxEDAO ++BgNVBAMTB2NvZHktY2EwHhcNMTgwNjE4MTgwMDU4WhcNMjgwNjE0MTgwMDU4WjA7 ++MQswCQYDVQQGEwJVSzEsMCoGA1UEAxMjY29kZW5vbWljb24tdm0tMi50ZXN0Lmxh ++bC5jaXNjby5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC63fGB ++J80A9Av1GB0bptslKRIUtJm8EeEu34HkDWbL6AJY0P8WfDtlXjlPaLqFa6sqH6ES ++V48prSm1ZUbDSVL8R6BYVYpOlK8/48xk4pGTgRzv69gf5SGtQLwHy8UPBKgjSZoD ++5a5k5wJXGswhKFFNqyyxqCvWmMnJWxXTt2XDCiWc4g4YAWi4O4+6SeeHVAV9rV7C ++1wxqjzKovVe2uZOHjKEzJbbIU6JBPb6TRfMdRdYOw98n1VXDcKVgdX2DuuqjCzHP ++WhU4Tw050M9NaK3eXp4Mh69VuiKoBGOLSOcS8reqHIU46Reg0hqeL8LIL6OhFHIF ++j7HR6V1X6F+BfRS/AgMBAAGjgdYwgdMwCQYDVR0TBAIwADAdBgNVHQ4EFgQUOktp ++HQjxDXXUg8prleY9jeLKeQ4wTwYDVR0jBEgwRoAUx6zgPygZ0ZErF9sPC4+5e2Io ++UU+hI6QhMB8xCzAJBgNVBAYTAlVLMRAwDgYDVQQDEwdjb2R5LWNhggkA1QEAuwb7 ++2s0wCQYDVR0SBAIwADAuBgNVHREEJzAlgiNjb2Rlbm9taWNvbi12bS0yLnRlc3Qu ++bGFsLmNpc2NvLmNvbTAOBgNVHQ8BAf8EBAMCBaAwCwYDVR0fBAQwAjAAMAsGCSqG ++SIb3DQEBCwOCAQEAvqantx2yBlM11RoFiCfi+AfSblXPdrIrHvccepV4pYc/yO6p ++t1f2dxHQb8rWH3i6cWag/EgIZx+HJQvo0rgPY1BFJsX1WnYf1/znZpkUBGbVmlJr ++t/dW1gSkNS6sPsM0Q+7HPgEv8CPDNK5eo7vU2seE0iWOkxSyVUuiCEY9ZVGaLVit ++p0C78nZ35Pdv4I+1cosmHl28+es1WI22rrnmdBpH8J1eY6WvUw2xuZHLeNVN0TzV ++Q3qq53AaCWuLOD1AjESWuUCxMZTK9DPS4JKXTK8RLyDeqOvJGjsSWp3kL0y3GaQ+ ++10T1rfkKJub2+m9A9duin1fn6tHc2wSvB7m3DA== ++-----END CERTIFICATE----- +diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py +index 7f6b93148f45..1fc657f4d867 100644 +--- a/Lib/test/test_ssl.py ++++ b/Lib/test/test_ssl.py +@@ -115,6 +115,7 @@ def data_file(*name): + BADKEY = data_file("badkey.pem") + NOKIACERT = data_file("nokia.pem") + NULLBYTECERT = data_file("nullbytecert.pem") ++TALOS_INVALID_CRLDP = data_file("talos-2019-0758.pem") + + DHFILE = data_file("ffdh3072.pem") + BYTES_DHFILE = os.fsencode(DHFILE) +@@ -348,6 +349,27 @@ def test_parse_cert(self): + self.assertEqual(p['crlDistributionPoints'], + ('http://SVRIntl-G3-crl.verisign.com/SVRIntlG3.crl',)) + ++ def test_parse_cert_CVE_2019_5010(self): ++ p = ssl._ssl._test_decode_cert(TALOS_INVALID_CRLDP) ++ if support.verbose: ++ sys.stdout.write("\n" + pprint.pformat(p) + "\n") ++ self.assertEqual( ++ p, ++ { ++ 'issuer': ( ++ (('countryName', 'UK'),), (('commonName', 'cody-ca'),)), ++ 'notAfter': 'Jun 14 18:00:58 2028 GMT', ++ 'notBefore': 'Jun 18 18:00:58 2018 GMT', ++ 'serialNumber': '02', ++ 'subject': ((('countryName', 'UK'),), ++ (('commonName', ++ 'codenomicon-vm-2.test.lal.cisco.com'),)), ++ 'subjectAltName': ( ++ ('DNS', 'codenomicon-vm-2.test.lal.cisco.com'),), ++ 'version': 3 ++ } ++ ) ++ + def test_parse_cert_CVE_2013_4238(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if support.verbose: +diff --git a/Misc/NEWS.d/next/Security/2019-01-15-18-16-05.bpo-35746.nMSd0j.rst b/Misc/NEWS.d/next/Security/2019-01-15-18-16-05.bpo-35746.nMSd0j.rst +new file mode 100644 +index 000000000000..dffe347eec84 +--- /dev/null ++++ b/Misc/NEWS.d/next/Security/2019-01-15-18-16-05.bpo-35746.nMSd0j.rst +@@ -0,0 +1,3 @@ ++[CVE-2019-5010] Fix a NULL pointer deref in ssl module. The cert parser did ++not handle CRL distribution points with empty DP or URI correctly. A ++malicious or buggy certificate can result into segfault. +diff --git a/Modules/_ssl.c b/Modules/_ssl.c +index 4e3352d9e661..0e720e268d93 100644 +--- a/Modules/_ssl.c ++++ b/Modules/_ssl.c +@@ -1515,6 +1515,10 @@ _get_crl_dp(X509 *certificate) { + STACK_OF(GENERAL_NAME) *gns; + + dp = sk_DIST_POINT_value(dps, i); ++ if (dp->distpoint == NULL) { ++ /* Ignore empty DP value, CVE-2019-5010 */ ++ continue; ++ } + gns = dp->distpoint->name.fullname; + + for (j=0; j < sk_GENERAL_NAME_num(gns); j++) { diff --git a/SOURCES/00318-test-ssl-fix-for-tls-13.patch b/SOURCES/00318-test-ssl-fix-for-tls-13.patch new file mode 100644 index 0000000..7b00d75 --- /dev/null +++ b/SOURCES/00318-test-ssl-fix-for-tls-13.patch @@ -0,0 +1,44 @@ +bpo-32947: test_ssl fixes for TLS 1.3 and OpenSSL 1.1.1 + +Backport partially commit 529525fb5a8fd9b96ab4021311a598c77588b918: +complete the previous partial backport (commit +2a4ee8aa01d61b6a9c8e9c65c211e61bdb471826. + +Reported upstream: + +* https://bugs.python.org/issue32947#msg333990 +* https://github.com/python/cpython/pull/11612 + +diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py +index 7f8f636..05c09a6 100644 +--- a/Lib/test/test_ssl.py ++++ b/Lib/test/test_ssl.py +@@ -2021,6 +2021,16 @@ if _have_threads: + sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" + % (msg, ctype, msg.lower(), ctype)) + self.write(msg.lower()) ++ except ConnectionResetError: ++ # XXX: OpenSSL 1.1.1 sometimes raises ConnectionResetError ++ # when connection is not shut down gracefully. ++ if self.server.chatty and support.verbose: ++ sys.stdout.write( ++ " Connection reset by peer: {}\n".format( ++ self.addr) ++ ) ++ self.close() ++ self.running = False + except OSError: + if self.server.chatty: + handle_error("Test server failure:\n") +@@ -2100,6 +2110,11 @@ if _have_threads: + pass + except KeyboardInterrupt: + self.stop() ++ except BaseException as e: ++ if support.verbose and self.chatty: ++ sys.stdout.write( ++ ' connection handling failed: ' + repr(e) + '\n') ++ + self.sock.close() + + def stop(self): diff --git a/SOURCES/00319-test_tarfile_ppc64.patch b/SOURCES/00319-test_tarfile_ppc64.patch new file mode 100644 index 0000000..132b9b3 --- /dev/null +++ b/SOURCES/00319-test_tarfile_ppc64.patch @@ -0,0 +1,41 @@ +commit 86ed41792d394f804d2c9e695ac8b257220fbdee +Author: Victor Stinner +Date: Tue Mar 12 17:17:13 2019 +0100 + + Fix test_tarfile on ppc64 + + Fix sparse file tests of test_tarfile on ppc64le with the tmpfs + filesystem. + + * https://bugzilla.redhat.com/show_bug.cgi?id=1639490 + * https://bugs.python.org/issue35772 + * https://github.com/python/cpython/commit/d1dd6be613381b996b9071443ef081de8e5f3aff + +diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py +index 4cd7d53..bd8b05f 100644 +--- a/Lib/test/test_tarfile.py ++++ b/Lib/test/test_tarfile.py +@@ -973,16 +973,21 @@ class GNUReadTest(LongnameTest, ReadTest, unittest.TestCase): + def _fs_supports_holes(): + # Return True if the platform knows the st_blocks stat attribute and + # uses st_blocks units of 512 bytes, and if the filesystem is able to +- # store holes in files. ++ # store holes of 4 KiB in files. ++ # ++ # The function returns False if page size is larger than 4 KiB. ++ # For example, ppc64 uses pages of 64 KiB. + if sys.platform.startswith("linux"): + # Linux evidentially has 512 byte st_blocks units. + name = os.path.join(TEMPDIR, "sparse-test") + with open(name, "wb") as fobj: ++ # Seek to "punch a hole" of 4 KiB + fobj.seek(4096) ++ fobj.write(b'x' * 4096) + fobj.truncate() + s = os.stat(name) + support.unlink(name) +- return s.st_blocks == 0 ++ return (s.st_blocks * 512 < s.st_size) + else: + return False + diff --git a/SOURCES/00324-disallow-control-chars-in-http-urls.patch b/SOURCES/00324-disallow-control-chars-in-http-urls.patch new file mode 100644 index 0000000..9e4317a --- /dev/null +++ b/SOURCES/00324-disallow-control-chars-in-http-urls.patch @@ -0,0 +1,150 @@ +From 7e200e0763f5b71c199aaf98bd5588f291585619 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= +Date: Tue, 7 May 2019 17:28:47 +0200 +Subject: [PATCH] bpo-30458: Disallow control chars in http URLs. (GH-12755) + (GH-13154) +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Disallow control chars in http URLs in urllib.urlopen. This addresses a potential security problem for applications that do not sanity check their URLs where http request headers could be injected. + +Disable https related urllib tests on a build without ssl (GH-13032) +These tests require an SSL enabled build. Skip these tests when python is built without SSL to fix test failures. + +Use http.client.InvalidURL instead of ValueError as the new error case's exception. (GH-13044) + +Backport Co-Authored-By: Miro Hrončok +--- + Lib/http/client.py | 15 ++++++ + Lib/test/test_urllib.py | 53 +++++++++++++++++++ + Lib/test/test_xmlrpc.py | 7 ++- + .../2019-04-10-08-53-30.bpo-30458.51E-DA.rst | 1 + + 4 files changed, 75 insertions(+), 1 deletion(-) + create mode 100644 Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst + +diff --git a/Lib/http/client.py b/Lib/http/client.py +index 1de151c38e..2afd452fe3 100644 +--- a/Lib/http/client.py ++++ b/Lib/http/client.py +@@ -140,6 +140,16 @@ _MAXHEADERS = 100 + _is_legal_header_name = re.compile(rb'[^:\s][^:\r\n]*').fullmatch + _is_illegal_header_value = re.compile(rb'\n(?![ \t])|\r(?![ \t\n])').search + ++# These characters are not allowed within HTTP URL paths. ++# See https://tools.ietf.org/html/rfc3986#section-3.3 and the ++# https://tools.ietf.org/html/rfc3986#appendix-A pchar definition. ++# Prevents CVE-2019-9740. Includes control characters such as \r\n. ++# We don't restrict chars above \x7f as putrequest() limits us to ASCII. ++_contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]') ++# Arguably only these _should_ allowed: ++# _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$") ++# We are more lenient for assumed real world compatibility purposes. ++ + # We always set the Content-Length header for these methods because some + # servers will otherwise respond with a 411 + _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} +@@ -1101,6 +1111,11 @@ class HTTPConnection: + self._method = method + if not url: + url = '/' ++ # Prevent CVE-2019-9740. ++ match = _contains_disallowed_url_pchar_re.search(url) ++ if match: ++ raise InvalidURL(f"URL can't contain control characters. {url!r} " ++ f"(found at least {match.group()!r})") + request = '%s %s %s' % (method, url, self._http_vsn_str) + + # Non-ASCII characters should have been eliminated earlier +diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py +index 2ac73b58d8..7214492eca 100644 +--- a/Lib/test/test_urllib.py ++++ b/Lib/test/test_urllib.py +@@ -329,6 +329,59 @@ class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): + finally: + self.unfakehttp() + ++ @unittest.skipUnless(ssl, "ssl module required") ++ def test_url_with_control_char_rejected(self): ++ for char_no in list(range(0, 0x21)) + [0x7f]: ++ char = chr(char_no) ++ schemeless_url = f"//localhost:7777/test{char}/" ++ self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.") ++ try: ++ # We explicitly test urllib.request.urlopen() instead of the top ++ # level 'def urlopen()' function defined in this... (quite ugly) ++ # test suite. They use different url opening codepaths. Plain ++ # urlopen uses FancyURLOpener which goes via a codepath that ++ # calls urllib.parse.quote() on the URL which makes all of the ++ # above attempts at injection within the url _path_ safe. ++ escaped_char_repr = repr(char).replace('\\', r'\\') ++ InvalidURL = http.client.InvalidURL ++ with self.assertRaisesRegex( ++ InvalidURL, f"contain control.*{escaped_char_repr}"): ++ urllib.request.urlopen(f"http:{schemeless_url}") ++ with self.assertRaisesRegex( ++ InvalidURL, f"contain control.*{escaped_char_repr}"): ++ urllib.request.urlopen(f"https:{schemeless_url}") ++ # This code path quotes the URL so there is no injection. ++ resp = urlopen(f"http:{schemeless_url}") ++ self.assertNotIn(char, resp.geturl()) ++ finally: ++ self.unfakehttp() ++ ++ @unittest.skipUnless(ssl, "ssl module required") ++ def test_url_with_newline_header_injection_rejected(self): ++ self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.") ++ host = "localhost:7777?a=1 HTTP/1.1\r\nX-injected: header\r\nTEST: 123" ++ schemeless_url = "//" + host + ":8080/test/?test=a" ++ try: ++ # We explicitly test urllib.request.urlopen() instead of the top ++ # level 'def urlopen()' function defined in this... (quite ugly) ++ # test suite. They use different url opening codepaths. Plain ++ # urlopen uses FancyURLOpener which goes via a codepath that ++ # calls urllib.parse.quote() on the URL which makes all of the ++ # above attempts at injection within the url _path_ safe. ++ InvalidURL = http.client.InvalidURL ++ with self.assertRaisesRegex( ++ InvalidURL, r"contain control.*\\r.*(found at least . .)"): ++ urllib.request.urlopen(f"http:{schemeless_url}") ++ with self.assertRaisesRegex(InvalidURL, r"contain control.*\\n"): ++ urllib.request.urlopen(f"https:{schemeless_url}") ++ # This code path quotes the URL so there is no injection. ++ resp = urlopen(f"http:{schemeless_url}") ++ self.assertNotIn(' ', resp.geturl()) ++ self.assertNotIn('\r', resp.geturl()) ++ self.assertNotIn('\n', resp.geturl()) ++ finally: ++ self.unfakehttp() ++ + def test_read_0_9(self): + # "0.9" response accepted (but not "simple responses" without + # a status line) +diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py +index 32263f7f0b..0e002ec4ef 100644 +--- a/Lib/test/test_xmlrpc.py ++++ b/Lib/test/test_xmlrpc.py +@@ -945,7 +945,12 @@ class SimpleServerTestCase(BaseServerTestCase): + def test_partial_post(self): + # Check that a partial POST doesn't make the server loop: issue #14001. + conn = http.client.HTTPConnection(ADDR, PORT) +- conn.request('POST', '/RPC2 HTTP/1.0\r\nContent-Length: 100\r\n\r\nbye') ++ conn.send('POST /RPC2 HTTP/1.0\r\n' ++ 'Content-Length: 100\r\n\r\n' ++ 'bye HTTP/1.1\r\n' ++ f'Host: {ADDR}:{PORT}\r\n' ++ 'Accept-Encoding: identity\r\n' ++ 'Content-Length: 0\r\n\r\n'.encode('ascii')) + conn.close() + + def test_context_manager(self): +diff --git a/Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst b/Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst +new file mode 100644 +index 0000000000..ed8027fb4d +--- /dev/null ++++ b/Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst +@@ -0,0 +1 @@ ++Address CVE-2019-9740 by disallowing URL paths with embedded whitespace or control characters through into the underlying http client request. Such potentially malicious header injection URLs now cause an http.client.InvalidURL exception to be raised. +-- +2.21.0 + diff --git a/SOURCES/00325-CVE-2019-9948.patch b/SOURCES/00325-CVE-2019-9948.patch new file mode 100644 index 0000000..4bf81ca --- /dev/null +++ b/SOURCES/00325-CVE-2019-9948.patch @@ -0,0 +1,49 @@ +diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py +index 649a5b8..0061a52 100644 +--- a/Lib/test/test_urllib.py ++++ b/Lib/test/test_urllib.py +@@ -16,6 +16,7 @@ except ImportError: + ssl = None + import sys + import tempfile ++import warnings + from nturl2path import url2pathname, pathname2url + + from base64 import b64encode +@@ -1463,6 +1464,23 @@ class URLopener_Tests(unittest.TestCase): + "spam://c:|windows%/:=&?~#+!$,;'@()*[]|/path/"), + "//c:|windows%/:=&?~#+!$,;'@()*[]|/path/") + ++ def test_local_file_open(self): ++ # bpo-35907, CVE-2019-9948: urllib must reject local_file:// scheme ++ class DummyURLopener(urllib.request.URLopener): ++ def open_local_file(self, url): ++ return url ++ ++ with warnings.catch_warnings(record=True): ++ warnings.simplefilter("ignore", DeprecationWarning) ++ ++ for url in ('local_file://example', 'local-file://example'): ++ self.assertRaises(OSError, urllib.request.urlopen, url) ++ self.assertRaises(OSError, urllib.request.URLopener().open, url) ++ self.assertRaises(OSError, urllib.request.URLopener().retrieve, url) ++ self.assertRaises(OSError, DummyURLopener().open, url) ++ self.assertRaises(OSError, DummyURLopener().retrieve, url) ++ ++ + # Just commented them out. + # Can't really tell why keep failing in windows and sparc. + # Everywhere else they work ok, but on those machines, sometimes +diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py +index d28f2f8..c9945d9 100644 +--- a/Lib/urllib/request.py ++++ b/Lib/urllib/request.py +@@ -1747,7 +1747,7 @@ class URLopener: + name = 'open_' + urltype + self.type = urltype + name = name.replace('-', '_') +- if not hasattr(self, name): ++ if not hasattr(self, name) or name == 'open_local_file': + if proxy: + return self.open_unknown_proxy(proxy, fullurl, data) + else: diff --git a/SOURCES/00326-do-not-set-PHA-verify-flag-on-client-side.patch b/SOURCES/00326-do-not-set-PHA-verify-flag-on-client-side.patch new file mode 100644 index 0000000..4584ce1 --- /dev/null +++ b/SOURCES/00326-do-not-set-PHA-verify-flag-on-client-side.patch @@ -0,0 +1,117 @@ +diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py +index 883201f..cf4d84d 100644 +--- a/Lib/test/test_ssl.py ++++ b/Lib/test/test_ssl.py +@@ -3891,6 +3891,37 @@ class TestPostHandshakeAuth(unittest.TestCase): + s.write(b'PHA') + self.assertIn(b'WRONG_SSL_VERSION', s.recv(1024)) + ++ def test_bpo37428_pha_cert_none(self): ++ # verify that post_handshake_auth does not implicitly enable cert ++ # validation. ++ hostname = 'localhost' ++ client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ++ client_context.post_handshake_auth = True ++ client_context.load_cert_chain(SIGNED_CERTFILE) ++ # no cert validation and CA on client side ++ client_context.check_hostname = False ++ client_context.verify_mode = ssl.CERT_NONE ++ ++ server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ++ server_context.load_cert_chain(SIGNED_CERTFILE) ++ server_context.load_verify_locations(SIGNING_CA) ++ server_context.post_handshake_auth = True ++ server_context.verify_mode = ssl.CERT_REQUIRED ++ ++ server = ThreadedEchoServer(context=server_context, chatty=False) ++ with server: ++ with client_context.wrap_socket(socket.socket(), ++ server_hostname=hostname) as s: ++ s.connect((HOST, server.port)) ++ s.write(b'HASCERT') ++ self.assertEqual(s.recv(1024), b'FALSE\n') ++ s.write(b'PHA') ++ self.assertEqual(s.recv(1024), b'OK\n') ++ s.write(b'HASCERT') ++ self.assertEqual(s.recv(1024), b'TRUE\n') ++ # server cert has not been validated ++ self.assertEqual(s.getpeercert(), {}) ++ + + def test_main(verbose=False): + if support.verbose: +diff --git a/Modules/_ssl.c b/Modules/_ssl.c +index ec366f0..9bf1cde 100644 +--- a/Modules/_ssl.c ++++ b/Modules/_ssl.c +@@ -732,6 +732,26 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, + #endif + SSL_set_mode(self->ssl, mode); + ++#ifdef TLS1_3_VERSION ++ if (sslctx->post_handshake_auth == 1) { ++ if (socket_type == PY_SSL_SERVER) { ++ /* bpo-37428: OpenSSL does not ignore SSL_VERIFY_POST_HANDSHAKE. ++ * Set SSL_VERIFY_POST_HANDSHAKE flag only for server sockets and ++ * only in combination with SSL_VERIFY_PEER flag. */ ++ int mode = SSL_get_verify_mode(self->ssl); ++ if (mode & SSL_VERIFY_PEER) { ++ int (*verify_cb)(int, X509_STORE_CTX *) = NULL; ++ verify_cb = SSL_get_verify_callback(self->ssl); ++ mode |= SSL_VERIFY_POST_HANDSHAKE; ++ SSL_set_verify(self->ssl, mode, verify_cb); ++ } ++ } else { ++ /* client socket */ ++ SSL_set_post_handshake_auth(self->ssl, 1); ++ } ++ } ++#endif ++ + #if HAVE_SNI + if (server_hostname != NULL) { + /* Don't send SNI for IP addresses. We cannot simply use inet_aton() and +@@ -2765,10 +2785,10 @@ _set_verify_mode(PySSLContext *self, enum py_ssl_cert_requirements n) + "invalid value for verify_mode"); + return -1; + } +-#ifdef TLS1_3_VERSION +- if (self->post_handshake_auth) +- mode |= SSL_VERIFY_POST_HANDSHAKE; +-#endif ++ ++ /* bpo-37428: newPySSLSocket() sets SSL_VERIFY_POST_HANDSHAKE flag for ++ * server sockets and SSL_set_post_handshake_auth() for client. */ ++ + /* keep current verify cb */ + verify_cb = SSL_CTX_get_verify_callback(self->ctx); + SSL_CTX_set_verify(self->ctx, mode, verify_cb); +@@ -3346,8 +3366,6 @@ get_post_handshake_auth(PySSLContext *self, void *c) { + #if TLS1_3_VERSION + static int + set_post_handshake_auth(PySSLContext *self, PyObject *arg, void *c) { +- int (*verify_cb)(int, X509_STORE_CTX *) = NULL; +- int mode = SSL_CTX_get_verify_mode(self->ctx); + int pha = PyObject_IsTrue(arg); + + if (pha == -1) { +@@ -3355,17 +3373,8 @@ set_post_handshake_auth(PySSLContext *self, PyObject *arg, void *c) { + } + self->post_handshake_auth = pha; + +- /* client-side socket setting, ignored by server-side */ +- SSL_CTX_set_post_handshake_auth(self->ctx, pha); +- +- /* server-side socket setting, ignored by client-side */ +- verify_cb = SSL_CTX_get_verify_callback(self->ctx); +- if (pha) { +- mode |= SSL_VERIFY_POST_HANDSHAKE; +- } else { +- mode ^= SSL_VERIFY_POST_HANDSHAKE; +- } +- SSL_CTX_set_verify(self->ctx, mode, verify_cb); ++ /* bpo-37428: newPySSLSocket() sets SSL_VERIFY_POST_HANDSHAKE flag for ++ * server sockets and SSL_set_post_handshake_auth() for client. */ + + return 0; + } diff --git a/SOURCES/00327-enable-tls-1.3-PHA-in-http.client.patch b/SOURCES/00327-enable-tls-1.3-PHA-in-http.client.patch new file mode 100644 index 0000000..bf92e84 --- /dev/null +++ b/SOURCES/00327-enable-tls-1.3-PHA-in-http.client.patch @@ -0,0 +1,70 @@ +diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst +index 2f59ece..d756916 100644 +--- a/Doc/library/http.client.rst ++++ b/Doc/library/http.client.rst +@@ -88,6 +88,11 @@ The module provides the following classes: + :func:`ssl._create_unverified_context` can be passed to the *context* + parameter. + ++ .. versionchanged:: 3.7.4 ++ This class now enables TLS 1.3 ++ :attr:`ssl.SSLContext.post_handshake_auth` for the default *context* or ++ when *cert_file* is passed with a custom *context*. ++ + .. deprecated:: 3.6 + + *key_file* and *cert_file* are deprecated in favor of *context*. +diff --git a/Lib/http/client.py b/Lib/http/client.py +index 1a6bd8a..f0d2642 100644 +--- a/Lib/http/client.py ++++ b/Lib/http/client.py +@@ -1390,6 +1390,9 @@ else: + self.cert_file = cert_file + if context is None: + context = ssl._create_default_https_context() ++ # enable PHA for TLS 1.3 connections if available ++ if context.post_handshake_auth is not None: ++ context.post_handshake_auth = True + will_verify = context.verify_mode != ssl.CERT_NONE + if check_hostname is None: + check_hostname = context.check_hostname +@@ -1398,6 +1401,10 @@ else: + "either CERT_OPTIONAL or CERT_REQUIRED") + if key_file or cert_file: + context.load_cert_chain(cert_file, key_file) ++ # cert and key file means the user wants to authenticate. ++ # enable TLS 1.3 PHA implicitly even for custom contexts. ++ if context.post_handshake_auth is not None: ++ context.post_handshake_auth = True + self._context = context + self._check_hostname = check_hostname + +diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py +index 714d521..5795b7a 100644 +--- a/Lib/test/test_httplib.py ++++ b/Lib/test/test_httplib.py +@@ -1709,6 +1709,24 @@ class HTTPSTest(TestCase): + self.assertEqual(h, c.host) + self.assertEqual(p, c.port) + ++ def test_tls13_pha(self): ++ import ssl ++ if not ssl.HAS_TLSv1_3: ++ self.skipTest('TLS 1.3 support required') ++ # just check status of PHA flag ++ h = client.HTTPSConnection('localhost', 443) ++ self.assertTrue(h._context.post_handshake_auth) ++ ++ context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ++ self.assertFalse(context.post_handshake_auth) ++ h = client.HTTPSConnection('localhost', 443, context=context) ++ self.assertIs(h._context, context) ++ self.assertFalse(h._context.post_handshake_auth) ++ ++ h = client.HTTPSConnection('localhost', 443, context=context, ++ cert_file=CERT_localhost) ++ self.assertTrue(h._context.post_handshake_auth) ++ + + class RequestBodyTest(TestCase): + """Test cases where a request includes a message body.""" diff --git a/SOURCES/00329-fips.patch b/SOURCES/00329-fips.patch new file mode 100644 index 0000000..b1ec9f8 --- /dev/null +++ b/SOURCES/00329-fips.patch @@ -0,0 +1,4925 @@ +From f85cd78683a7f4216f85439ff12bde07c8118597 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Thu, 25 Jul 2019 16:19:52 +0200 +Subject: [PATCH 01/36] Expose OpenSSL FIPS_mode() as hashlib.get_fips_mode() + +--- + Lib/hashlib.py | 5 +++++ + Modules/_hashopenssl.c | 36 +++++++++++++++++++++++++++++++++ + Modules/clinic/_hashopenssl.c.h | 25 ++++++++++++++++++++++- + 3 files changed, 65 insertions(+), 1 deletion(-) + +diff --git a/Lib/hashlib.py b/Lib/hashlib.py +index 98d2d7981a38..ae17c5851109 100644 +--- a/Lib/hashlib.py ++++ b/Lib/hashlib.py +@@ -236,6 +236,11 @@ def prf(msg, inner=inner, outer=outer): + except ImportError: + pass + ++try: ++ from _hashlib import get_fips_mode ++except ImportError: ++ pass ++ + + for __func_name in __always_supported: + # try them all, some may not work due to the OpenSSL +diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c +index 84edd72c85a3..4876a7f7aa76 100644 +--- a/Modules/_hashopenssl.c ++++ b/Modules/_hashopenssl.c +@@ -25,6 +25,9 @@ + #include + #include "openssl/err.h" + ++/* Expose FIPS_mode */ ++#include ++ + #include "clinic/_hashopenssl.c.h" + /*[clinic input] + module _hashlib +@@ -987,6 +990,38 @@ GEN_CONSTRUCTOR(sha256) + GEN_CONSTRUCTOR(sha384) + GEN_CONSTRUCTOR(sha512) + ++/*[clinic input] ++_hashlib.get_fips_mode ++ ++Determine the OpenSSL FIPS mode of operation. ++ ++Effectively any non-zero return value indicates FIPS mode; ++values other than 1 may have additional significance. ++ ++See OpenSSL documentation for the FIPS_mode() function for details. ++[clinic start generated code]*/ ++ ++static PyObject * ++_hashlib_get_fips_mode_impl(PyObject *module) ++/*[clinic end generated code: output=ad8a7793310d3f98 input=f42a2135df2a5e11]*/ ++{ ++ int result = FIPS_mode(); ++ if (result == 0) { ++ // "If the library was built without support of the FIPS Object Module, ++ // then the function will return 0 with an error code of ++ // CRYPTO_R_FIPS_MODE_NOT_SUPPORTED (0x0f06d065)." ++ // But 0 is also a valid result value. ++ ++ unsigned long errcode = ERR_peek_last_error(); ++ if (errcode) { ++ _setException(PyExc_ValueError); ++ return NULL; ++ } ++ } ++ return PyLong_FromLong(result); ++} ++ ++ + /* List of functions exported by this module */ + + static struct PyMethodDef EVP_functions[] = { +@@ -996,6 +1031,7 @@ static struct PyMethodDef EVP_functions[] = { + pbkdf2_hmac__doc__}, + #endif + _HASHLIB_SCRYPT_METHODDEF ++ _HASHLIB_GET_FIPS_MODE_METHODDEF + CONSTRUCTOR_METH_DEF(md5), + CONSTRUCTOR_METH_DEF(sha1), + CONSTRUCTOR_METH_DEF(sha224), +diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h +index 04453526040b..8828e2776e95 100644 +--- a/Modules/clinic/_hashopenssl.c.h ++++ b/Modules/clinic/_hashopenssl.c.h +@@ -54,7 +54,30 @@ _hashlib_scrypt(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *k + + #endif /* (OPENSSL_VERSION_NUMBER > 0x10100000L && !defined(OPENSSL_NO_SCRYPT) && !defined(LIBRESSL_VERSION_NUMBER)) */ + ++PyDoc_STRVAR(_hashlib_get_fips_mode__doc__, ++"get_fips_mode($module, /)\n" ++"--\n" ++"\n" ++"Determine the OpenSSL FIPS mode of operation.\n" ++"\n" ++"Effectively any non-zero return value indicates FIPS mode;\n" ++"values other than 1 may have additional significance.\n" ++"\n" ++"See OpenSSL documentation for the FIPS_mode() function for details."); ++ ++#define _HASHLIB_GET_FIPS_MODE_METHODDEF \ ++ {"get_fips_mode", (PyCFunction)_hashlib_get_fips_mode, METH_NOARGS, _hashlib_get_fips_mode__doc__}, ++ ++static PyObject * ++_hashlib_get_fips_mode_impl(PyObject *module); ++ ++static PyObject * ++_hashlib_get_fips_mode(PyObject *module, PyObject *Py_UNUSED(ignored)) ++{ ++ return _hashlib_get_fips_mode_impl(module); ++} ++ + #ifndef _HASHLIB_SCRYPT_METHODDEF + #define _HASHLIB_SCRYPT_METHODDEF + #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ +-/*[clinic end generated code: output=118cd7036fa0fb52 input=a9049054013a1b77]*/ ++/*[clinic end generated code: output=7d683c930bbb7c36 input=a9049054013a1b77]*/ + +From b8c716a651d5884e0dfdbb2cfefb0f49a0ee0540 Mon Sep 17 00:00:00 2001 +From: Charalampos Stratakis +Date: Thu, 25 Jul 2019 17:04:06 +0200 +Subject: [PATCH 02/36] Use python's fall backs for the crypto it implements + only if we are not in FIPS mode + +--- + Lib/hashlib.py | 209 +++++++++++++++------------------------ + Lib/test/test_hashlib.py | 1 + + 2 files changed, 81 insertions(+), 129 deletions(-) + +diff --git a/Lib/hashlib.py b/Lib/hashlib.py +index ae17c5851109..7db1e02601fe 100644 +--- a/Lib/hashlib.py ++++ b/Lib/hashlib.py +@@ -67,56 +67,64 @@ + __all__ = __always_supported + ('new', 'algorithms_guaranteed', + 'algorithms_available', 'pbkdf2_hmac') + +- +-__builtin_constructor_cache = {} +- +-def __get_builtin_constructor(name): +- cache = __builtin_constructor_cache +- constructor = cache.get(name) +- if constructor is not None: +- return constructor +- try: +- if name in ('SHA1', 'sha1'): +- import _sha1 +- cache['SHA1'] = cache['sha1'] = _sha1.sha1 +- elif name in ('MD5', 'md5'): +- import _md5 +- cache['MD5'] = cache['md5'] = _md5.md5 +- elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): +- import _sha256 +- cache['SHA224'] = cache['sha224'] = _sha256.sha224 +- cache['SHA256'] = cache['sha256'] = _sha256.sha256 +- elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): +- import _sha512 +- cache['SHA384'] = cache['sha384'] = _sha512.sha384 +- cache['SHA512'] = cache['sha512'] = _sha512.sha512 +- elif name in ('blake2b', 'blake2s'): +- import _blake2 +- cache['blake2b'] = _blake2.blake2b +- cache['blake2s'] = _blake2.blake2s +- elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', +- 'shake_128', 'shake_256'}: +- import _sha3 +- cache['sha3_224'] = _sha3.sha3_224 +- cache['sha3_256'] = _sha3.sha3_256 +- cache['sha3_384'] = _sha3.sha3_384 +- cache['sha3_512'] = _sha3.sha3_512 +- cache['shake_128'] = _sha3.shake_128 +- cache['shake_256'] = _sha3.shake_256 +- except ImportError: +- pass # no extension module, this hash is unsupported. +- +- constructor = cache.get(name) +- if constructor is not None: +- return constructor +- +- raise ValueError('unsupported hash type ' + name) ++try: ++ from _hashlib import get_fips_mode ++except ImportError: ++ def get_fips_mode(): ++ return 0 ++ ++ ++if not get_fips_mode(): ++ __builtin_constructor_cache = {} ++ ++ def __get_builtin_constructor(name): ++ cache = __builtin_constructor_cache ++ constructor = cache.get(name) ++ if constructor is not None: ++ return constructor ++ try: ++ if name in ('SHA1', 'sha1'): ++ import _sha1 ++ cache['SHA1'] = cache['sha1'] = _sha1.sha1 ++ elif name in ('MD5', 'md5'): ++ import _md5 ++ cache['MD5'] = cache['md5'] = _md5.md5 ++ elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): ++ import _sha256 ++ cache['SHA224'] = cache['sha224'] = _sha256.sha224 ++ cache['SHA256'] = cache['sha256'] = _sha256.sha256 ++ elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): ++ import _sha512 ++ cache['SHA384'] = cache['sha384'] = _sha512.sha384 ++ cache['SHA512'] = cache['sha512'] = _sha512.sha512 ++ elif name in ('blake2b', 'blake2s'): ++ import _blake2 ++ cache['blake2b'] = _blake2.blake2b ++ cache['blake2s'] = _blake2.blake2s ++ elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', ++ 'shake_128', 'shake_256'}: ++ import _sha3 ++ cache['sha3_224'] = _sha3.sha3_224 ++ cache['sha3_256'] = _sha3.sha3_256 ++ cache['sha3_384'] = _sha3.sha3_384 ++ cache['sha3_512'] = _sha3.sha3_512 ++ cache['shake_128'] = _sha3.shake_128 ++ cache['shake_256'] = _sha3.shake_256 ++ except ImportError: ++ pass # no extension module, this hash is unsupported. ++ ++ constructor = cache.get(name) ++ if constructor is not None: ++ return constructor ++ ++ raise ValueError('unsupported hash type ' + name) + + + def __get_openssl_constructor(name): +- if name in {'blake2b', 'blake2s'}: +- # Prefer our blake2 implementation. +- return __get_builtin_constructor(name) ++ if not get_fips_mode(): ++ if name in {'blake2b', 'blake2s'}: ++ # Prefer our blake2 implementation. ++ return __get_builtin_constructor(name) + try: + f = getattr(_hashlib, 'openssl_' + name) + # Allow the C module to raise ValueError. The function will be +@@ -125,27 +133,30 @@ def __get_openssl_constructor(name): + # Use the C function directly (very fast) + return f + except (AttributeError, ValueError): ++ if get_fips_mode(): ++ raise + return __get_builtin_constructor(name) + +- +-def __py_new(name, data=b'', **kwargs): +- """new(name, data=b'', **kwargs) - Return a new hashing object using the +- named algorithm; optionally initialized with data (which must be +- a bytes-like object). +- """ +- return __get_builtin_constructor(name)(data, **kwargs) ++if not get_fips_mode(): ++ def __py_new(name, data=b'', **kwargs): ++ """new(name, data=b'', **kwargs) - Return a new hashing object using the ++ named algorithm; optionally initialized with data (which must be ++ a bytes-like object). ++ """ ++ return __get_builtin_constructor(name)(data, **kwargs) + + + def __hash_new(name, data=b'', **kwargs): + """new(name, data=b'') - Return a new hashing object using the named algorithm; + optionally initialized with data (which must be a bytes-like object). + """ +- if name in {'blake2b', 'blake2s'}: +- # Prefer our blake2 implementation. +- # OpenSSL 1.1.0 comes with a limited implementation of blake2b/s. +- # It does neither support keyed blake2 nor advanced features like +- # salt, personal, tree hashing or SSE. +- return __get_builtin_constructor(name)(data, **kwargs) ++ if not get_fips_mode(): ++ if name in {'blake2b', 'blake2s'}: ++ # Prefer our blake2 implementation. ++ # OpenSSL 1.1.0 comes with a limited implementation of blake2b/s. ++ # It does neither support keyed blake2 nor advanced features like ++ # salt, personal, tree hashing or SSE. ++ return __get_builtin_constructor(name)(data, **kwargs) + try: + return _hashlib.new(name, data) + except ValueError: +@@ -153,6 +164,8 @@ def __hash_new(name, data=b'', **kwargs): + # hash, try using our builtin implementations. + # This allows for SHA224/256 and SHA384/512 support even though + # the OpenSSL library prior to 0.9.8 doesn't provide them. ++ if get_fips_mode(): ++ raise + return __get_builtin_constructor(name)(data) + + +@@ -163,72 +176,14 @@ def __hash_new(name, data=b'', **kwargs): + algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names) + except ImportError: ++ if get_fips_mode(): ++ raise + new = __py_new + __get_hash = __get_builtin_constructor + +-try: +- # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA +- from _hashlib import pbkdf2_hmac +-except ImportError: +- _trans_5C = bytes((x ^ 0x5C) for x in range(256)) +- _trans_36 = bytes((x ^ 0x36) for x in range(256)) +- +- def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None): +- """Password based key derivation function 2 (PKCS #5 v2.0) + +- This Python implementations based on the hmac module about as fast +- as OpenSSL's PKCS5_PBKDF2_HMAC for short passwords and much faster +- for long passwords. +- """ +- if not isinstance(hash_name, str): +- raise TypeError(hash_name) +- +- if not isinstance(password, (bytes, bytearray)): +- password = bytes(memoryview(password)) +- if not isinstance(salt, (bytes, bytearray)): +- salt = bytes(memoryview(salt)) +- +- # Fast inline HMAC implementation +- inner = new(hash_name) +- outer = new(hash_name) +- blocksize = getattr(inner, 'block_size', 64) +- if len(password) > blocksize: +- password = new(hash_name, password).digest() +- password = password + b'\x00' * (blocksize - len(password)) +- inner.update(password.translate(_trans_36)) +- outer.update(password.translate(_trans_5C)) +- +- def prf(msg, inner=inner, outer=outer): +- # PBKDF2_HMAC uses the password as key. We can re-use the same +- # digest objects and just update copies to skip initialization. +- icpy = inner.copy() +- ocpy = outer.copy() +- icpy.update(msg) +- ocpy.update(icpy.digest()) +- return ocpy.digest() +- +- if iterations < 1: +- raise ValueError(iterations) +- if dklen is None: +- dklen = outer.digest_size +- if dklen < 1: +- raise ValueError(dklen) +- +- dkey = b'' +- loop = 1 +- from_bytes = int.from_bytes +- while len(dkey) < dklen: +- prev = prf(salt + loop.to_bytes(4, 'big')) +- # endianess doesn't matter here as long to / from use the same +- rkey = int.from_bytes(prev, 'big') +- for i in range(iterations - 1): +- prev = prf(prev) +- # rkey = rkey ^ prev +- rkey ^= from_bytes(prev, 'big') +- loop += 1 +- dkey += rkey.to_bytes(inner.digest_size, 'big') +- +- return dkey[:dklen] ++# OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA ++from _hashlib import pbkdf2_hmac + + try: + # OpenSSL's scrypt requires OpenSSL 1.1+ +@@ -236,12 +191,6 @@ def prf(msg, inner=inner, outer=outer): + except ImportError: + pass + +-try: +- from _hashlib import get_fips_mode +-except ImportError: +- pass +- +- + for __func_name in __always_supported: + # try them all, some may not work due to the OpenSSL + # version not supporting that algorithm. +@@ -254,4 +203,6 @@ def prf(msg, inner=inner, outer=outer): + + # Cleanup locals() + del __always_supported, __func_name, __get_hash +-del __py_new, __hash_new, __get_openssl_constructor ++del __hash_new, __get_openssl_constructor ++if not get_fips_mode(): ++ del __py_new +diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py +index 9711856853de..c67d2f6d00aa 100644 +--- a/Lib/test/test_hashlib.py ++++ b/Lib/test/test_hashlib.py +@@ -930,6 +930,7 @@ def _test_pbkdf2_hmac(self, pbkdf2): + iterations=1, dklen=None) + self.assertEqual(out, self.pbkdf2_results['sha1'][0][0]) + ++ @unittest.skip("The python implementation of pbkdf2_hmac has been removed") + def test_pbkdf2_hmac_py(self): + self._test_pbkdf2_hmac(py_hashlib.pbkdf2_hmac) + + +From 88185ade926f6a629acd8d24f00dfd10bfa9de0e Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Thu, 25 Jul 2019 17:19:06 +0200 +Subject: [PATCH 03/36] Disable Python's hash implementations in FIPS mode, + forcing OpenSSL + +--- + Include/_hashopenssl.h | 66 ++++++++++++++++++++++++++++++++++ + Modules/_blake2/blake2b_impl.c | 5 +++ + Modules/_blake2/blake2module.c | 3 ++ + Modules/_blake2/blake2s_impl.c | 5 +++ + Modules/_hashopenssl.c | 36 +------------------ + Modules/_sha3/sha3module.c | 5 +++ + setup.py | 31 ++++++++-------- + 7 files changed, 101 insertions(+), 50 deletions(-) + create mode 100644 Include/_hashopenssl.h + +diff --git a/Include/_hashopenssl.h b/Include/_hashopenssl.h +new file mode 100644 +index 000000000000..a726c0d3fbf3 +--- /dev/null ++++ b/Include/_hashopenssl.h +@@ -0,0 +1,66 @@ ++#ifndef Py_HASHOPENSSL_H ++#define Py_HASHOPENSSL_H ++ ++#include "Python.h" ++#include ++#include ++ ++/* LCOV_EXCL_START */ ++static PyObject * ++_setException(PyObject *exc) ++{ ++ unsigned long errcode; ++ const char *lib, *func, *reason; ++ ++ errcode = ERR_peek_last_error(); ++ if (!errcode) { ++ PyErr_SetString(exc, "unknown reasons"); ++ return NULL; ++ } ++ ERR_clear_error(); ++ ++ lib = ERR_lib_error_string(errcode); ++ func = ERR_func_error_string(errcode); ++ reason = ERR_reason_error_string(errcode); ++ ++ if (lib && func) { ++ PyErr_Format(exc, "[%s: %s] %s", lib, func, reason); ++ } ++ else if (lib) { ++ PyErr_Format(exc, "[%s] %s", lib, reason); ++ } ++ else { ++ PyErr_SetString(exc, reason); ++ } ++ return NULL; ++} ++/* LCOV_EXCL_STOP */ ++ ++ ++__attribute__((__unused__)) ++static int ++_Py_hashlib_fips_error(char *name) { ++ int result = FIPS_mode(); ++ if (result == 0) { ++ // "If the library was built without support of the FIPS Object Module, ++ // then the function will return 0 with an error code of ++ // CRYPTO_R_FIPS_MODE_NOT_SUPPORTED (0x0f06d065)." ++ // But 0 is also a valid result value. ++ ++ unsigned long errcode = ERR_peek_last_error(); ++ if (errcode) { ++ _setException(PyExc_ValueError); ++ return 1; ++ } ++ return 0; ++ } ++ PyErr_Format(PyExc_ValueError, "%s is not available in FIPS mode", ++ name); ++ return 1; ++} ++ ++#define FAIL_RETURN_IN_FIPS_MODE(name) do { \ ++ if (_Py_hashlib_fips_error(name)) return NULL; \ ++} while (0) ++ ++#endif // !Py_HASHOPENSSL_H +diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c +index 418c0184006d..341e67a8fcc4 100644 +--- a/Modules/_blake2/blake2b_impl.c ++++ b/Modules/_blake2/blake2b_impl.c +@@ -14,6 +14,7 @@ + */ + + #include "Python.h" ++#include "_hashopenssl.h" + #include "pystrhex.h" + #ifdef WITH_THREAD + #include "pythread.h" +@@ -103,6 +104,8 @@ py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + unsigned long leaf_size = 0; + unsigned long long node_offset = 0; + ++ FAIL_RETURN_IN_FIPS_MODE("_blake2"); ++ + self = new_BLAKE2bObject(type); + if (self == NULL) { + goto error; +@@ -293,6 +296,8 @@ _blake2_blake2b_update(BLAKE2bObject *self, PyObject *data) + { + Py_buffer buf; + ++ FAIL_RETURN_IN_FIPS_MODE("_blake2"); ++ + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); + + #ifdef WITH_THREAD +diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c +index e2a3d420d4eb..817b71656844 100644 +--- a/Modules/_blake2/blake2module.c ++++ b/Modules/_blake2/blake2module.c +@@ -9,6 +9,7 @@ + */ + + #include "Python.h" ++#include "_hashopenssl.h" + + #include "impl/blake2.h" + +@@ -57,6 +58,8 @@ PyInit__blake2(void) + PyObject *m; + PyObject *d; + ++ FAIL_RETURN_IN_FIPS_MODE("blake2"); ++ + m = PyModule_Create(&blake2_module); + if (m == NULL) + return NULL; +diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c +index 24e529b6596a..0dfe0586b570 100644 +--- a/Modules/_blake2/blake2s_impl.c ++++ b/Modules/_blake2/blake2s_impl.c +@@ -14,6 +14,7 @@ + */ + + #include "Python.h" ++#include "_hashopenssl.h" + #include "pystrhex.h" + #ifdef WITH_THREAD + #include "pythread.h" +@@ -103,6 +104,8 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + unsigned long leaf_size = 0; + unsigned long long node_offset = 0; + ++ FAIL_RETURN_IN_FIPS_MODE("_blake2"); ++ + self = new_BLAKE2sObject(type); + if (self == NULL) { + goto error; +@@ -293,6 +296,8 @@ _blake2_blake2s_update(BLAKE2sObject *self, PyObject *data) + { + Py_buffer buf; + ++ FAIL_RETURN_IN_FIPS_MODE("_blake2"); ++ + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); + + #ifdef WITH_THREAD +diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c +index 4876a7f7aa76..02cf6087a2ea 100644 +--- a/Modules/_hashopenssl.c ++++ b/Modules/_hashopenssl.c +@@ -17,16 +17,13 @@ + #include "structmember.h" + #include "hashlib.h" + #include "pystrhex.h" ++#include "_hashopenssl.h" + + + /* EVP is the preferred interface to hashing in OpenSSL */ + #include + /* We use the object interface to discover what hashes OpenSSL supports. */ + #include +-#include "openssl/err.h" +- +-/* Expose FIPS_mode */ +-#include + + #include "clinic/_hashopenssl.c.h" + /*[clinic input] +@@ -77,37 +74,6 @@ DEFINE_CONSTS_FOR_NEW(sha384) + DEFINE_CONSTS_FOR_NEW(sha512) + + +-/* LCOV_EXCL_START */ +-static PyObject * +-_setException(PyObject *exc) +-{ +- unsigned long errcode; +- const char *lib, *func, *reason; +- +- errcode = ERR_peek_last_error(); +- if (!errcode) { +- PyErr_SetString(exc, "unknown reasons"); +- return NULL; +- } +- ERR_clear_error(); +- +- lib = ERR_lib_error_string(errcode); +- func = ERR_func_error_string(errcode); +- reason = ERR_reason_error_string(errcode); +- +- if (lib && func) { +- PyErr_Format(exc, "[%s: %s] %s", lib, func, reason); +- } +- else if (lib) { +- PyErr_Format(exc, "[%s] %s", lib, reason); +- } +- else { +- PyErr_SetString(exc, reason); +- } +- return NULL; +-} +-/* LCOV_EXCL_STOP */ +- + static EVPobject * + newEVPobject(PyObject *name) + { +diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c +index 2c2b2dbc5c7d..624f7f247d4f 100644 +--- a/Modules/_sha3/sha3module.c ++++ b/Modules/_sha3/sha3module.c +@@ -18,6 +18,7 @@ + #include "Python.h" + #include "pystrhex.h" + #include "../hashlib.h" ++#include "_hashopenssl.h" + + /* ************************************************************************** + * SHA-3 (Keccak) and SHAKE +@@ -162,6 +163,7 @@ static PyTypeObject SHAKE256type; + static SHA3object * + newSHA3object(PyTypeObject *type) + { ++ FAIL_RETURN_IN_FIPS_MODE("_sha3"); + SHA3object *newobj; + newobj = (SHA3object *)PyObject_New(SHA3object, type); + if (newobj == NULL) { +@@ -177,6 +179,7 @@ newSHA3object(PyTypeObject *type) + static PyObject * + py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) + { ++ FAIL_RETURN_IN_FIPS_MODE("_sha3"); + SHA3object *self = NULL; + Py_buffer buf = {NULL, NULL}; + HashReturn res; +@@ -724,6 +727,8 @@ PyInit__sha3(void) + { + PyObject *m = NULL; + ++ FAIL_RETURN_IN_FIPS_MODE("_sha3"); ++ + if ((m = PyModule_Create(&_SHA3module)) == NULL) { + return NULL; + } +diff --git a/setup.py b/setup.py +index e2c18982532b..e2e659ef7950 100644 +--- a/setup.py ++++ b/setup.py +@@ -901,31 +901,30 @@ def detect_modules(self): + have_usable_openssl = (have_any_openssl and + openssl_ver >= min_openssl_ver) + ++ if not have_usable_openssl: ++ raise ValueError('Cannot build for RHEL without OpenSSL') ++ ++ ssl_args = { ++ 'include_dirs': ssl_incs, ++ 'library_dirs': ssl_libs, ++ 'libraries': ['ssl', 'crypto'], ++ } ++ + if have_any_openssl: + if have_usable_openssl: + # The _hashlib module wraps optimized implementations + # of hash functions from the OpenSSL library. + exts.append( Extension('_hashlib', ['_hashopenssl.c'], + depends = ['hashlib.h'], +- include_dirs = ssl_incs, +- library_dirs = ssl_libs, +- libraries = ['ssl', 'crypto']) ) ++ **ssl_args)) + else: + print("warning: openssl 0x%08x is too old for _hashlib" % + openssl_ver) + missing.append('_hashlib') + +- # We always compile these even when OpenSSL is available (issue #14693). +- # It's harmless and the object code is tiny (40-50 KB per module, +- # only loaded when actually used). +- exts.append( Extension('_sha256', ['sha256module.c'], +- depends=['hashlib.h']) ) +- exts.append( Extension('_sha512', ['sha512module.c'], +- depends=['hashlib.h']) ) +- exts.append( Extension('_md5', ['md5module.c'], +- depends=['hashlib.h']) ) +- exts.append( Extension('_sha1', ['sha1module.c'], +- depends=['hashlib.h']) ) ++ # RHEL: Always force OpenSSL for md5, sha1, sha256, sha512; ++ # don't build Python's implementations. ++ # sha3 and blake2 have extra functionality, so do build those: + + blake2_deps = glob(os.path.join(os.getcwd(), srcdir, + 'Modules/_blake2/impl/*')) +@@ -944,6 +943,7 @@ def detect_modules(self): + '_blake2/blake2b_impl.c', + '_blake2/blake2s_impl.c'], + define_macros=blake2_macros, ++ **ssl_args, + depends=blake2_deps) ) + + sha3_deps = glob(os.path.join(os.getcwd(), srcdir, +@@ -951,7 +951,8 @@ def detect_modules(self): + sha3_deps.append('hashlib.h') + exts.append( Extension('_sha3', + ['_sha3/sha3module.c'], +- depends=sha3_deps)) ++ **ssl_args, ++ depends=sha3_deps + ['hashlib.h'])) + + # Modules that provide persistent dictionary-like semantics. You will + # probably want to arrange for at least one of them to be available on + +From ced78ca1f37efca7a43dc467756a4ff5b5669555 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Thu, 25 Jul 2019 17:35:27 +0200 +Subject: [PATCH 04/36] Expose all hashes available to OpenSSL, using a list + +--- + Modules/_hashopenssl.c | 44 ++++++++++++++----------------------- + Modules/_hashopenssl_list.h | 21 ++++++++++++++++++ + 2 files changed, 37 insertions(+), 28 deletions(-) + create mode 100644 Modules/_hashopenssl_list.h + +diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c +index 02cf6087a2ea..7dfd70822b99 100644 +--- a/Modules/_hashopenssl.c ++++ b/Modules/_hashopenssl.c +@@ -66,12 +66,9 @@ static PyTypeObject EVPtype; + static PyObject *CONST_ ## Name ## _name_obj = NULL; \ + static EVP_MD_CTX *CONST_new_ ## Name ## _ctx_p = NULL; + +-DEFINE_CONSTS_FOR_NEW(md5) +-DEFINE_CONSTS_FOR_NEW(sha1) +-DEFINE_CONSTS_FOR_NEW(sha224) +-DEFINE_CONSTS_FOR_NEW(sha256) +-DEFINE_CONSTS_FOR_NEW(sha384) +-DEFINE_CONSTS_FOR_NEW(sha512) ++#define _HASH(py_name, openssl_name) DEFINE_CONSTS_FOR_NEW(py_name) ++#include "_hashopenssl_list.h" ++#undef _HASH + + + static EVPobject * +@@ -896,7 +893,7 @@ generate_hash_name_list(void) + * The first call will lazy-initialize, which reports an exception + * if initialization fails. + */ +-#define GEN_CONSTRUCTOR(NAME) \ ++#define GEN_CONSTRUCTOR(NAME, SSL_NAME) \ + static PyObject * \ + EVP_new_ ## NAME (PyObject *self, PyObject *args) \ + { \ +@@ -910,8 +907,8 @@ generate_hash_name_list(void) + \ + if (CONST_new_ ## NAME ## _ctx_p == NULL) { \ + EVP_MD_CTX *ctx_p = EVP_MD_CTX_new(); \ +- if (!EVP_get_digestbyname(#NAME) || \ +- !EVP_DigestInit(ctx_p, EVP_get_digestbyname(#NAME))) { \ ++ if (!EVP_get_digestbyname(SSL_NAME) || \ ++ !EVP_DigestInit(ctx_p, EVP_get_digestbyname(SSL_NAME))) { \ + _setException(PyExc_ValueError); \ + EVP_MD_CTX_free(ctx_p); \ + return NULL; \ +@@ -939,7 +936,7 @@ generate_hash_name_list(void) + {"openssl_" #NAME, (PyCFunction)EVP_new_ ## NAME, METH_VARARGS, \ + PyDoc_STR("Returns a " #NAME \ + " hash object; optionally initialized with a string") \ +- } ++ }, + + /* used in the init function to setup a constructor: initialize OpenSSL + constructor constants if they haven't been initialized already. */ +@@ -949,12 +946,9 @@ generate_hash_name_list(void) + } \ + } while (0); + +-GEN_CONSTRUCTOR(md5) +-GEN_CONSTRUCTOR(sha1) +-GEN_CONSTRUCTOR(sha224) +-GEN_CONSTRUCTOR(sha256) +-GEN_CONSTRUCTOR(sha384) +-GEN_CONSTRUCTOR(sha512) ++#define _HASH(py_name, openssl_name) GEN_CONSTRUCTOR(py_name, openssl_name) ++#include "_hashopenssl_list.h" ++#undef _HASH + + /*[clinic input] + _hashlib.get_fips_mode +@@ -998,12 +992,9 @@ static struct PyMethodDef EVP_functions[] = { + #endif + _HASHLIB_SCRYPT_METHODDEF + _HASHLIB_GET_FIPS_MODE_METHODDEF +- CONSTRUCTOR_METH_DEF(md5), +- CONSTRUCTOR_METH_DEF(sha1), +- CONSTRUCTOR_METH_DEF(sha224), +- CONSTRUCTOR_METH_DEF(sha256), +- CONSTRUCTOR_METH_DEF(sha384), +- CONSTRUCTOR_METH_DEF(sha512), ++#define _HASH(py_name, openssl_name) CONSTRUCTOR_METH_DEF(py_name) ++#include "_hashopenssl_list.h" ++#undef _HASH + {NULL, NULL} /* Sentinel */ + }; + +@@ -1061,11 +1052,8 @@ PyInit__hashlib(void) + PyModule_AddObject(m, "HASH", (PyObject *)&EVPtype); + + /* these constants are used by the convenience constructors */ +- INIT_CONSTRUCTOR_CONSTANTS(md5); +- INIT_CONSTRUCTOR_CONSTANTS(sha1); +- INIT_CONSTRUCTOR_CONSTANTS(sha224); +- INIT_CONSTRUCTOR_CONSTANTS(sha256); +- INIT_CONSTRUCTOR_CONSTANTS(sha384); +- INIT_CONSTRUCTOR_CONSTANTS(sha512); ++#define _HASH(py_name, openssl_name) INIT_CONSTRUCTOR_CONSTANTS(py_name) ++#include "_hashopenssl_list.h" ++#undef _HASH + return m; + } +diff --git a/Modules/_hashopenssl_list.h b/Modules/_hashopenssl_list.h +new file mode 100644 +index 000000000000..3c11b2eb6c62 +--- /dev/null ++++ b/Modules/_hashopenssl_list.h +@@ -0,0 +1,21 @@ ++/* Call the _HASH macro with all the hashes exported by OpenSSL, ++ * at compile time. ++ * ++ * This file is meant to be included multiple times, with different values of ++ * _HASH. ++ */ ++ ++_HASH(md5, "md5") ++_HASH(sha1, "sha1") ++_HASH(sha224, "sha224") ++_HASH(sha256, "sha256") ++_HASH(sha384, "sha384") ++_HASH(sha512, "sha512") ++_HASH(blake2b, "blake2b512") ++_HASH(blake2s, "blake2s256") ++_HASH(sha3_224, "sha3-224") ++_HASH(sha3_256, "sha3-256") ++_HASH(sha3_384, "sha3-384") ++_HASH(sha3_512, "sha3-512") ++_HASH(shake_128, "shake128") ++_HASH(shake_256, "shake256") + +From c05359891debdfd0f2b320c6c396e9baf4a9f6ab Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Thu, 25 Jul 2019 18:13:45 +0200 +Subject: [PATCH 05/36] Fix tests + +--- + Lib/hashlib.py | 5 +++- + Lib/test/test_hashlib.py | 58 +++++++++++++++++++++++++++++++--------- + 2 files changed, 49 insertions(+), 14 deletions(-) + +diff --git a/Lib/hashlib.py b/Lib/hashlib.py +index 7db1e02601fe..2def0a310c66 100644 +--- a/Lib/hashlib.py ++++ b/Lib/hashlib.py +@@ -122,7 +122,10 @@ def __get_builtin_constructor(name): + + def __get_openssl_constructor(name): + if not get_fips_mode(): +- if name in {'blake2b', 'blake2s'}: ++ if name in { ++ 'blake2b', 'blake2s', 'shake_256', 'shake_128', ++ #'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', ++ }: + # Prefer our blake2 implementation. + return __get_builtin_constructor(name) + try: +diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py +index c67d2f6d00aa..645a3d01b91d 100644 +--- a/Lib/test/test_hashlib.py ++++ b/Lib/test/test_hashlib.py +@@ -179,7 +179,9 @@ def test_hash_array(self): + a = array.array("b", range(10)) + for cons in self.hash_constructors: + c = cons(a) +- if c.name in self.shakes: ++ if (c.name in self.shakes ++ and not cons.__name__.startswith('openssl_') ++ ): + c.hexdigest(16) + else: + c.hexdigest() +@@ -226,7 +228,9 @@ def test_get_builtin_constructor(self): + def test_hexdigest(self): + for cons in self.hash_constructors: + h = cons() +- if h.name in self.shakes: ++ if (h.name in self.shakes ++ and not cons.__name__.startswith('openssl_') ++ ): + self.assertIsInstance(h.digest(16), bytes) + self.assertEqual(hexstr(h.digest(16)), h.hexdigest(16)) + else: +@@ -240,6 +244,8 @@ def test_digest_length_overflow(self): + h = cons() + if h.name not in self.shakes: + continue ++ if cons.__name__.startswith('openssl_'): ++ continue + for digest in h.digest, h.hexdigest: + with self.assertRaises((ValueError, OverflowError)): + digest(-10) +@@ -269,7 +275,9 @@ def test_large_update(self): + m1.update(bees) + m1.update(cees) + m1.update(dees) +- if m1.name in self.shakes: ++ if (m1.name in self.shakes ++ and not cons.__name__.startswith('openssl_') ++ ): + args = (16,) + else: + args = () +@@ -296,15 +304,36 @@ def check(self, name, data, hexdigest, shake=False, **kwargs): + # 2 is for hashlib.name(...) and hashlib.new(name, ...) + self.assertGreaterEqual(len(constructors), 2) + for hash_object_constructor in constructors: ++ if ( ++ kwargs ++ and hash_object_constructor.__name__.startswith('openssl_') ++ ): ++ return + m = hash_object_constructor(data, **kwargs) +- computed = m.hexdigest() if not shake else m.hexdigest(length) ++ if shake: ++ if hash_object_constructor.__name__.startswith('openssl_'): ++ if length > m.digest_size: ++ # OpenSSL doesn't give long digests ++ return ++ computed = m.hexdigest()[:length*2] ++ hexdigest = hexdigest[:length*2] ++ else: ++ computed = m.hexdigest(length) ++ else: ++ computed = m.hexdigest() + self.assertEqual( + computed, hexdigest, + "Hash algorithm %s constructed using %s returned hexdigest" + " %r for %d byte input data that should have hashed to %r." + % (name, hash_object_constructor, + computed, len(data), hexdigest)) +- computed = m.digest() if not shake else m.digest(length) ++ if shake: ++ if hash_object_constructor.__name__.startswith('openssl_'): ++ computed = m.digest()[:length] ++ else: ++ computed = m.digest(length) ++ else: ++ computed = m.digest() + digest = bytes.fromhex(hexdigest) + self.assertEqual(computed, digest) + if not shake: +@@ -344,12 +373,14 @@ def check_blocksize_name(self, name, block_size=0, digest_size=0, + for hash_object_constructor in constructors: + m = hash_object_constructor() + self.assertEqual(m.block_size, block_size) +- self.assertEqual(m.digest_size, digest_size) ++ if not hash_object_constructor.__name__.startswith('openssl_'): ++ self.assertEqual(m.digest_size, digest_size) + if digest_length: +- self.assertEqual(len(m.digest(digest_length)), +- digest_length) +- self.assertEqual(len(m.hexdigest(digest_length)), +- 2*digest_length) ++ if not hash_object_constructor.__name__.startswith('openssl_'): ++ self.assertEqual(len(m.digest(digest_length)), ++ digest_length) ++ self.assertEqual(len(m.hexdigest(digest_length)), ++ 2*digest_length) + else: + self.assertEqual(len(m.digest()), digest_size) + self.assertEqual(len(m.hexdigest()), 2*digest_size) +@@ -379,9 +410,10 @@ def check_sha3(self, name, capacity, rate, suffix): + for hash_object_constructor in constructors: + m = hash_object_constructor() + self.assertEqual(capacity + rate, 1600) +- self.assertEqual(m._capacity_bits, capacity) +- self.assertEqual(m._rate_bits, rate) +- self.assertEqual(m._suffix, suffix) ++ if not hash_object_constructor.__name__.startswith('openssl_'): ++ self.assertEqual(m._capacity_bits, capacity) ++ self.assertEqual(m._rate_bits, rate) ++ self.assertEqual(m._suffix, suffix) + + @requires_sha3 + def test_extra_sha3(self): + +From b9238e7d197c315d2c9c7aa34e5c8a4b728b25a0 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Fri, 26 Jul 2019 11:27:57 +0200 +Subject: [PATCH 06/36] Change FIPS exceptions from _blake2, _sha3 module init + to ImportError + +--- + Include/_hashopenssl.h | 11 +++++------ + Modules/_blake2/blake2b_impl.c | 4 ++-- + Modules/_blake2/blake2module.c | 2 +- + Modules/_blake2/blake2s_impl.c | 4 ++-- + Modules/_sha3/sha3module.c | 6 +++--- + 5 files changed, 13 insertions(+), 14 deletions(-) + +diff --git a/Include/_hashopenssl.h b/Include/_hashopenssl.h +index a726c0d3fbf3..47ed00304220 100644 +--- a/Include/_hashopenssl.h ++++ b/Include/_hashopenssl.h +@@ -39,7 +39,7 @@ _setException(PyObject *exc) + + __attribute__((__unused__)) + static int +-_Py_hashlib_fips_error(char *name) { ++_Py_hashlib_fips_error(PyObject *exc, char *name) { + int result = FIPS_mode(); + if (result == 0) { + // "If the library was built without support of the FIPS Object Module, +@@ -49,18 +49,17 @@ _Py_hashlib_fips_error(char *name) { + + unsigned long errcode = ERR_peek_last_error(); + if (errcode) { +- _setException(PyExc_ValueError); ++ _setException(exc); + return 1; + } + return 0; + } +- PyErr_Format(PyExc_ValueError, "%s is not available in FIPS mode", +- name); ++ PyErr_Format(exc, "%s is not available in FIPS mode", name); + return 1; + } + +-#define FAIL_RETURN_IN_FIPS_MODE(name) do { \ +- if (_Py_hashlib_fips_error(name)) return NULL; \ ++#define FAIL_RETURN_IN_FIPS_MODE(exc, name) do { \ ++ if (_Py_hashlib_fips_error(exc, name)) return NULL; \ + } while (0) + + #endif // !Py_HASHOPENSSL_H +diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c +index 341e67a8fcc4..f6bfce823b8f 100644 +--- a/Modules/_blake2/blake2b_impl.c ++++ b/Modules/_blake2/blake2b_impl.c +@@ -104,7 +104,7 @@ py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + unsigned long leaf_size = 0; + unsigned long long node_offset = 0; + +- FAIL_RETURN_IN_FIPS_MODE("_blake2"); ++ FAIL_RETURN_IN_FIPS_MODE(PyExc_ValueError, "_blake2"); + + self = new_BLAKE2bObject(type); + if (self == NULL) { +@@ -296,7 +296,7 @@ _blake2_blake2b_update(BLAKE2bObject *self, PyObject *data) + { + Py_buffer buf; + +- FAIL_RETURN_IN_FIPS_MODE("_blake2"); ++ FAIL_RETURN_IN_FIPS_MODE(PyExc_ValueError, "_blake2"); + + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); + +diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c +index 817b71656844..a9c7cbc7ebe9 100644 +--- a/Modules/_blake2/blake2module.c ++++ b/Modules/_blake2/blake2module.c +@@ -58,7 +58,7 @@ PyInit__blake2(void) + PyObject *m; + PyObject *d; + +- FAIL_RETURN_IN_FIPS_MODE("blake2"); ++ FAIL_RETURN_IN_FIPS_MODE(PyExc_ImportError, "blake2"); + + m = PyModule_Create(&blake2_module); + if (m == NULL) +diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c +index 0dfe0586b570..28ae5b651019 100644 +--- a/Modules/_blake2/blake2s_impl.c ++++ b/Modules/_blake2/blake2s_impl.c +@@ -104,7 +104,7 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + unsigned long leaf_size = 0; + unsigned long long node_offset = 0; + +- FAIL_RETURN_IN_FIPS_MODE("_blake2"); ++ FAIL_RETURN_IN_FIPS_MODE(PyExc_ValueError, "_blake2"); + + self = new_BLAKE2sObject(type); + if (self == NULL) { +@@ -296,7 +296,7 @@ _blake2_blake2s_update(BLAKE2sObject *self, PyObject *data) + { + Py_buffer buf; + +- FAIL_RETURN_IN_FIPS_MODE("_blake2"); ++ FAIL_RETURN_IN_FIPS_MODE(PyExc_ValueError, "_blake2"); + + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); + +diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c +index 624f7f247d4f..2783a75644fc 100644 +--- a/Modules/_sha3/sha3module.c ++++ b/Modules/_sha3/sha3module.c +@@ -163,7 +163,7 @@ static PyTypeObject SHAKE256type; + static SHA3object * + newSHA3object(PyTypeObject *type) + { +- FAIL_RETURN_IN_FIPS_MODE("_sha3"); ++ FAIL_RETURN_IN_FIPS_MODE(PyExc_ValueError, "_sha3"); + SHA3object *newobj; + newobj = (SHA3object *)PyObject_New(SHA3object, type); + if (newobj == NULL) { +@@ -179,7 +179,7 @@ newSHA3object(PyTypeObject *type) + static PyObject * + py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) + { +- FAIL_RETURN_IN_FIPS_MODE("_sha3"); ++ FAIL_RETURN_IN_FIPS_MODE(PyExc_ValueError, "_sha3"); + SHA3object *self = NULL; + Py_buffer buf = {NULL, NULL}; + HashReturn res; +@@ -727,7 +727,7 @@ PyInit__sha3(void) + { + PyObject *m = NULL; + +- FAIL_RETURN_IN_FIPS_MODE("_sha3"); ++ FAIL_RETURN_IN_FIPS_MODE(PyExc_ImportError, "_sha3"); + + if ((m = PyModule_Create(&_SHA3module)) == NULL) { + return NULL; + +From d9cf1f68ed7c7d10846119d146583d6fd2d52143 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Fri, 26 Jul 2019 11:24:09 +0200 +Subject: [PATCH 07/36] Make hashlib importable under FIPS mode + +--- + Lib/hashlib.py | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/Lib/hashlib.py b/Lib/hashlib.py +index 2def0a310c66..ca1dd2022515 100644 +--- a/Lib/hashlib.py ++++ b/Lib/hashlib.py +@@ -132,12 +132,14 @@ def __get_openssl_constructor(name): + f = getattr(_hashlib, 'openssl_' + name) + # Allow the C module to raise ValueError. The function will be + # defined but the hash not actually available thanks to OpenSSL. +- f() ++ if not get_fips_mode(): ++ # N.B. In "FIPS mode", there is no fallback. ++ # If this test fails, we want to export the broken hash ++ # constructor anyway. ++ f() + # Use the C function directly (very fast) + return f + except (AttributeError, ValueError): +- if get_fips_mode(): +- raise + return __get_builtin_constructor(name) + + if not get_fips_mode(): + +From 5b21c029a9b69b06f9ef245264db2ddd227011f9 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Fri, 26 Jul 2019 15:41:10 +0200 +Subject: [PATCH 08/36] Implement hmac.new using new built-in module, + _hmacopenssl + +--- + Lib/hmac.py | 33 ++- + Modules/_hmacopenssl.c | 396 ++++++++++++++++++++++++++++++++ + Modules/clinic/_hmacopenssl.c.h | 133 +++++++++++ + setup.py | 4 + + 4 files changed, 565 insertions(+), 1 deletion(-) + create mode 100644 Modules/_hmacopenssl.c + create mode 100644 Modules/clinic/_hmacopenssl.c.h + +diff --git a/Lib/hmac.py b/Lib/hmac.py +index 121029aa670b..ed98406bd2e1 100644 +--- a/Lib/hmac.py ++++ b/Lib/hmac.py +@@ -6,6 +6,8 @@ + import warnings as _warnings + from _operator import _compare_digest as compare_digest + import hashlib as _hashlib ++import _hashlib as _hashlibopenssl ++import _hmacopenssl + + trans_5C = bytes((x ^ 0x5C) for x in range(256)) + trans_36 = bytes((x ^ 0x36) for x in range(256)) +@@ -37,6 +39,11 @@ def __init__(self, key, msg = None, digestmod = None): + + Note: key and msg must be a bytes or bytearray objects. + """ ++ if _hashlib.get_fips_mode(): ++ raise ValueError( ++ 'hmac.HMAC is not available in FIPS mode. ' ++ + 'Use hmac.new().' ++ ) + + if not isinstance(key, (bytes, bytearray)): + raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) +@@ -90,6 +97,8 @@ def name(self): + def update(self, msg): + """Update this hashing object with the string msg. + """ ++ if _hashlib.get_fips_mode(): ++ raise ValueError('hmac.HMAC is not available in FIPS mode') + self.inner.update(msg) + + def copy(self): +@@ -130,6 +139,19 @@ def hexdigest(self): + h = self._current() + return h.hexdigest() + ++ ++def _get_openssl_name(digestmod): ++ if isinstance(digestmod, str): ++ return digestmod.lower() ++ elif callable(digestmod): ++ digestmod = digestmod(b'') ++ ++ if not isinstance(digestmod, _hashlibopenssl.HASH): ++ raise TypeError( ++ 'Only OpenSSL hashlib hashes are accepted in FIPS mode.') ++ ++ return digestmod.name.lower().replace('_', '-') ++ + def new(key, msg = None, digestmod = None): + """Create a new hashing object and return it. + +@@ -141,4 +163,13 @@ def new(key, msg = None, digestmod = None): + method, and can ask for the hash value at any time by calling its digest() + method. + """ +- return HMAC(key, msg, digestmod) ++ if _hashlib.get_fips_mode(): ++ if digestmod is None: ++ digestmod = 'md5' ++ name = _get_openssl_name(digestmod) ++ result = _hmacopenssl.new(key, digestmod=name) ++ if msg: ++ result.update(msg) ++ return result ++ else: ++ return HMAC(key, msg, digestmod) +diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c +new file mode 100644 +index 000000000000..ca95d725f019 +--- /dev/null ++++ b/Modules/_hmacopenssl.c +@@ -0,0 +1,396 @@ ++/* Module that wraps all OpenSSL MHAC algorithm */ ++ ++/* Copyright (C) 2019 Red Hat, Inc. Red Hat, Inc. and/or its affiliates ++ * ++ * Based on _hashopenssl.c, which is: ++ * Copyright (C) 2005-2010 Gregory P. Smith (greg@krypto.org) ++ * Licensed to PSF under a Contributor Agreement. ++ * ++ * Derived from a skeleton of shamodule.c containing work performed by: ++ * ++ * Andrew Kuchling (amk@amk.ca) ++ * Greg Stein (gstein@lyra.org) ++ * ++ */ ++ ++#define PY_SSIZE_T_CLEAN ++ ++#include "Python.h" ++#include "structmember.h" ++#include "hashlib.h" ++#include "pystrhex.h" ++#include "_hashopenssl.h" ++ ++ ++#include ++ ++static PyTypeObject HmacType; ++ ++typedef struct { ++ PyObject_HEAD ++ PyObject *name; /* name of the hash algorithm */ ++ HMAC_CTX *ctx; /* OpenSSL hmac context */ ++ PyThread_type_lock lock; /* HMAC context lock */ ++} HmacObject; ++ ++#include "clinic/_hmacopenssl.c.h" ++/*[clinic input] ++module _hmacopenssl ++class _hmacopenssl.HMAC "HmacObject *" "&HmacType" ++[clinic start generated code]*/ ++/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c98d3f2af591c085]*/ ++ ++ ++/*[clinic input] ++_hmacopenssl.new ++ ++ key: Py_buffer ++ * ++ digestmod: str ++ ++Return a new hmac object. ++[clinic start generated code]*/ ++ ++static PyObject * ++_hmacopenssl_new_impl(PyObject *module, Py_buffer *key, ++ const char *digestmod) ++/*[clinic end generated code: output=46f1cb4e02921922 input=be8c0c2e4fad508c]*/ ++{ ++ if (digestmod == NULL) { ++ PyErr_SetString(PyExc_ValueError, "digestmod must be specified"); ++ return NULL; ++ } ++ ++ /* name mut be lowercase */ ++ for (int i=0; digestmod[i]; i++) { ++ if ( ++ ((digestmod[i] < 'a') || (digestmod[i] > 'z')) ++ && ((digestmod[i] < '0') || (digestmod[i] > '9')) ++ && digestmod[i] != '-' ++ ) { ++ PyErr_SetString(PyExc_ValueError, "digestmod must be lowercase"); ++ return NULL; ++ } ++ } ++ ++ const EVP_MD *digest = EVP_get_digestbyname(digestmod); ++ if (!digest) { ++ PyErr_SetString(PyExc_ValueError, "unknown hash function"); ++ return NULL; ++ } ++ ++ PyObject *name = NULL; ++ HMAC_CTX *ctx = NULL; ++ HmacObject *retval = NULL; ++ ++ name = PyUnicode_FromFormat("hmac-%s", digestmod); ++ if (name == NULL) { ++ goto error; ++ } ++ ++ ctx = HMAC_CTX_new(); ++ if (ctx == NULL) { ++ _setException(PyExc_ValueError); ++ goto error; ++ } ++ ++ int r = HMAC_Init_ex( ++ ctx, ++ (const char*)key->buf, ++ key->len, ++ digest, ++ NULL /*impl*/); ++ if (r == 0) { ++ _setException(PyExc_ValueError); ++ goto error; ++ } ++ ++ retval = (HmacObject *)PyObject_New(HmacObject, &HmacType); ++ if (retval == NULL) { ++ goto error; ++ } ++ ++ retval->name = name; ++ retval->ctx = ctx; ++ retval->lock = NULL; ++ ++ return (PyObject*)retval; ++ ++error: ++ if (ctx) HMAC_CTX_free(ctx); ++ if (name) Py_DECREF(name); ++ if (retval) PyObject_Del(name); ++ return NULL; ++} ++ ++/*[clinic input] ++_hmacopenssl.HMAC.copy ++ ++Return a copy (“clone”) of the HMAC object. ++[clinic start generated code]*/ ++ ++static PyObject * ++_hmacopenssl_HMAC_copy_impl(HmacObject *self) ++/*[clinic end generated code: output=fe5ee41faf30dcf0 input=f5ed20feec42d8d0]*/ ++{ ++ HmacObject *retval = (HmacObject *)PyObject_New(HmacObject, &HmacType); ++ if (retval == NULL) { ++ return NULL; ++ } ++ ++ Py_INCREF(self->name); ++ retval->name = self->name; ++ ++ int r = HMAC_CTX_copy(retval->ctx, self->ctx); ++ if (r == 0) { ++ PyObject_Del(retval); ++ return _setException(PyExc_ValueError); ++ } ++ ++ return (PyObject*)retval; ++} ++ ++static void ++_hmac_dealloc(HmacObject *self) ++{ ++ if (self->lock != NULL) { ++ PyThread_free_lock(self->lock); ++ } ++ HMAC_CTX_free(self->ctx); ++ Py_XDECREF(self->name); ++ PyObject_Del(self); ++} ++ ++static PyObject * ++_hmac_repr(HmacObject *self) ++{ ++ return PyUnicode_FromFormat("<%U HMAC object @ %p>", self->name, self); ++} ++ ++/*[clinic input] ++_hmacopenssl.HMAC.update ++ ++ msg: Py_buffer ++ ++Update the HMAC object with msg. ++[clinic start generated code]*/ ++ ++static PyObject * ++_hmacopenssl_HMAC_update_impl(HmacObject *self, Py_buffer *msg) ++/*[clinic end generated code: output=0efeee663a98cee5 input=0683d64f35808cb9]*/ ++{ ++ if (self->lock == NULL && msg->len >= HASHLIB_GIL_MINSIZE) { ++ self->lock = PyThread_allocate_lock(); ++ /* fail? lock = NULL and we fail over to non-threaded code. */ ++ } ++ ++ int r; ++ ++ if (self->lock != NULL) { ++ Py_BEGIN_ALLOW_THREADS ++ PyThread_acquire_lock(self->lock, 1); ++ r = HMAC_Update(self->ctx, (const unsigned char*)msg->buf, msg->len); ++ PyThread_release_lock(self->lock); ++ Py_END_ALLOW_THREADS ++ } else { ++ r = HMAC_Update(self->ctx, (const unsigned char*)msg->buf, msg->len); ++ } ++ ++ if (r == 0) { ++ _setException(PyExc_ValueError); ++ return NULL; ++ } ++ Py_RETURN_NONE; ++} ++ ++static unsigned int ++_digest_size(HmacObject *self) ++{ ++ const EVP_MD *md = HMAC_CTX_get_md(self->ctx); ++ if (md == NULL) { ++ _setException(PyExc_ValueError); ++ return 0; ++ } ++ return EVP_MD_size(md); ++} ++ ++static int ++_digest(HmacObject *self, unsigned char *buf, unsigned int len) ++{ ++ HMAC_CTX *temp_ctx = HMAC_CTX_new(); ++ if (temp_ctx == NULL) { ++ PyErr_NoMemory(); ++ return 0; ++ } ++ int r = HMAC_CTX_copy(temp_ctx, self->ctx); ++ if (r == 0) { ++ _setException(PyExc_ValueError); ++ return 0; ++ } ++ r = HMAC_Final(temp_ctx, buf, &len); ++ HMAC_CTX_free(temp_ctx); ++ if (r == 0) { ++ _setException(PyExc_ValueError); ++ return 0; ++ } ++ return 1; ++} ++ ++/*[clinic input] ++_hmacopenssl.HMAC.digest ++ ++Return the digest of the bytes passed to the update() method so far. ++[clinic start generated code]*/ ++ ++static PyObject * ++_hmacopenssl_HMAC_digest_impl(HmacObject *self) ++/*[clinic end generated code: output=3aa6dbfc46ec4957 input=bf769a10b1d9edd9]*/ ++{ ++ unsigned int digest_size = _digest_size(self); ++ if (digest_size == 0) { ++ return _setException(PyExc_ValueError); ++ } ++ unsigned char buf[digest_size]; /* FIXME: C99 feature */ ++ int r = _digest(self, buf, digest_size); ++ if (r == 0) { ++ return NULL; ++ } ++ return PyBytes_FromStringAndSize((const char *)buf, digest_size); ++} ++ ++/*[clinic input] ++_hmacopenssl.HMAC.hexdigest ++ ++Return hexadecimal digest of the bytes passed to the update() method so far. ++ ++This may be used to exchange the value safely in email or other non-binary ++environments. ++[clinic start generated code]*/ ++ ++static PyObject * ++_hmacopenssl_HMAC_hexdigest_impl(HmacObject *self) ++/*[clinic end generated code: output=630f6fa89f9f1e48 input=b8e60ec8b811c4cd]*/ ++{ ++ unsigned int digest_size = _digest_size(self); ++ if (digest_size == 0) { ++ return _setException(PyExc_ValueError); ++ } ++ unsigned char buf[digest_size]; /* FIXME: C99 feature */ ++ int r = _digest(self, buf, digest_size); ++ if (r == 0) { ++ return NULL; ++ } ++ return _Py_strhex((const char *)buf, digest_size); ++} ++ ++ ++ ++static PyObject * ++_hmacopenssl_get_digest_size(HmacObject *self, void *closure) ++{ ++ unsigned int digest_size = _digest_size(self); ++ if (digest_size == 0) { ++ return _setException(PyExc_ValueError); ++ } ++ return PyLong_FromLong(digest_size); ++} ++ ++static PyObject * ++_hmacopenssl_get_block_size(HmacObject *self, void *closure) ++{ ++ const EVP_MD *md = HMAC_CTX_get_md(self->ctx); ++ if (md == NULL) { ++ return _setException(PyExc_ValueError); ++ } ++ return PyLong_FromLong(EVP_MD_size(md)); ++} ++ ++static PyMethodDef Hmac_methods[] = { ++ _HMACOPENSSL_HMAC_UPDATE_METHODDEF ++ _HMACOPENSSL_HMAC_DIGEST_METHODDEF ++ _HMACOPENSSL_HMAC_HEXDIGEST_METHODDEF ++ _HMACOPENSSL_HMAC_COPY_METHODDEF ++ {NULL, NULL} /* sentinel */ ++}; ++ ++static PyGetSetDef Hmac_getset[] = { ++ {"digest_size", (getter)_hmacopenssl_get_digest_size, NULL, NULL, NULL}, ++ {"block_size", (getter)_hmacopenssl_get_block_size, NULL, NULL, NULL}, ++ {NULL} /* Sentinel */ ++}; ++ ++static PyMemberDef Hmac_members[] = { ++ {"name", T_OBJECT, offsetof(HmacObject, name), READONLY, PyDoc_STR("HMAC name")}, ++}; ++ ++PyDoc_STRVAR(hmactype_doc, ++"The object used to calculate HMAC of a message.\n\ ++\n\ ++Methods:\n\ ++\n\ ++update() -- updates the current digest with an additional string\n\ ++digest() -- return the current digest value\n\ ++hexdigest() -- return the current digest as a string of hexadecimal digits\n\ ++copy() -- return a copy of the current hash object\n\ ++\n\ ++Attributes:\n\ ++\n\ ++name -- the name, including the hash algorithm used by this object\n\ ++digest_size -- number of bytes in digest() output\n"); ++ ++static PyTypeObject HmacType = { ++ PyVarObject_HEAD_INIT(NULL, 0) ++ "_hmacopenssl.HMAC", /*tp_name*/ ++ sizeof(HmacObject), /*tp_basicsize*/ ++ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, ++ .tp_doc = hmactype_doc, ++ .tp_repr = (reprfunc)_hmac_repr, ++ .tp_dealloc = (destructor)_hmac_dealloc, ++ .tp_methods = Hmac_methods, ++ .tp_getset = Hmac_getset, ++ .tp_members = Hmac_members, ++}; ++ ++static struct PyMethodDef hmacopenssl_functions[] = { ++ _HMACOPENSSL_NEW_METHODDEF ++ {NULL, NULL} /* Sentinel */ ++}; ++ ++ ++ ++/* Initialize this module. */ ++ ++ ++static struct PyModuleDef _hmacopenssl_module = { ++ PyModuleDef_HEAD_INIT, ++ "_hmacopenssl", ++ NULL, ++ -1, ++ hmacopenssl_functions, ++ NULL, ++ NULL, ++ NULL, ++ NULL ++}; ++ ++PyMODINIT_FUNC ++PyInit__hmacopenssl(void) ++{ ++ /* TODO build EVP_functions openssl_* entries dynamically based ++ * on what hashes are supported rather than listing many ++ * but having some be unsupported. Only init appropriate ++ * constants. */ ++ ++ Py_TYPE(&HmacType) = &PyType_Type; ++ if (PyType_Ready(&HmacType) < 0) ++ return NULL; ++ ++ PyObject *m = PyModule_Create(&_hmacopenssl_module); ++ if (m == NULL) ++ return NULL; ++ ++ Py_INCREF((PyObject *)&HmacType); ++ PyModule_AddObject(m, "HMAC", (PyObject *)&HmacType); ++ ++ return m; ++} +diff --git a/Modules/clinic/_hmacopenssl.c.h b/Modules/clinic/_hmacopenssl.c.h +new file mode 100644 +index 000000000000..b472a6eddd34 +--- /dev/null ++++ b/Modules/clinic/_hmacopenssl.c.h +@@ -0,0 +1,133 @@ ++/*[clinic input] ++preserve ++[clinic start generated code]*/ ++ ++PyDoc_STRVAR(_hmacopenssl_new__doc__, ++"new($module, /, key, *, digestmod)\n" ++"--\n" ++"\n" ++"Return a new hmac object."); ++ ++#define _HMACOPENSSL_NEW_METHODDEF \ ++ {"new", (PyCFunction)_hmacopenssl_new, METH_FASTCALL, _hmacopenssl_new__doc__}, ++ ++static PyObject * ++_hmacopenssl_new_impl(PyObject *module, Py_buffer *key, ++ const char *digestmod); ++ ++static PyObject * ++_hmacopenssl_new(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) ++{ ++ PyObject *return_value = NULL; ++ static const char * const _keywords[] = {"key", "digestmod", NULL}; ++ static _PyArg_Parser _parser = {"y*$s:new", _keywords, 0}; ++ Py_buffer key = {NULL, NULL}; ++ const char *digestmod; ++ ++ if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, ++ &key, &digestmod)) { ++ goto exit; ++ } ++ return_value = _hmacopenssl_new_impl(module, &key, digestmod); ++ ++exit: ++ /* Cleanup for key */ ++ if (key.obj) { ++ PyBuffer_Release(&key); ++ } ++ ++ return return_value; ++} ++ ++PyDoc_STRVAR(_hmacopenssl_HMAC_copy__doc__, ++"copy($self, /)\n" ++"--\n" ++"\n" ++"Return a copy (“clone”) of the HMAC object."); ++ ++#define _HMACOPENSSL_HMAC_COPY_METHODDEF \ ++ {"copy", (PyCFunction)_hmacopenssl_HMAC_copy, METH_NOARGS, _hmacopenssl_HMAC_copy__doc__}, ++ ++static PyObject * ++_hmacopenssl_HMAC_copy_impl(HmacObject *self); ++ ++static PyObject * ++_hmacopenssl_HMAC_copy(HmacObject *self, PyObject *Py_UNUSED(ignored)) ++{ ++ return _hmacopenssl_HMAC_copy_impl(self); ++} ++ ++PyDoc_STRVAR(_hmacopenssl_HMAC_update__doc__, ++"update($self, /, msg)\n" ++"--\n" ++"\n" ++"Update the HMAC object with msg."); ++ ++#define _HMACOPENSSL_HMAC_UPDATE_METHODDEF \ ++ {"update", (PyCFunction)_hmacopenssl_HMAC_update, METH_FASTCALL, _hmacopenssl_HMAC_update__doc__}, ++ ++static PyObject * ++_hmacopenssl_HMAC_update_impl(HmacObject *self, Py_buffer *msg); ++ ++static PyObject * ++_hmacopenssl_HMAC_update(HmacObject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) ++{ ++ PyObject *return_value = NULL; ++ static const char * const _keywords[] = {"msg", NULL}; ++ static _PyArg_Parser _parser = {"y*:update", _keywords, 0}; ++ Py_buffer msg = {NULL, NULL}; ++ ++ if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, ++ &msg)) { ++ goto exit; ++ } ++ return_value = _hmacopenssl_HMAC_update_impl(self, &msg); ++ ++exit: ++ /* Cleanup for msg */ ++ if (msg.obj) { ++ PyBuffer_Release(&msg); ++ } ++ ++ return return_value; ++} ++ ++PyDoc_STRVAR(_hmacopenssl_HMAC_digest__doc__, ++"digest($self, /)\n" ++"--\n" ++"\n" ++"Return the digest of the bytes passed to the update() method so far."); ++ ++#define _HMACOPENSSL_HMAC_DIGEST_METHODDEF \ ++ {"digest", (PyCFunction)_hmacopenssl_HMAC_digest, METH_NOARGS, _hmacopenssl_HMAC_digest__doc__}, ++ ++static PyObject * ++_hmacopenssl_HMAC_digest_impl(HmacObject *self); ++ ++static PyObject * ++_hmacopenssl_HMAC_digest(HmacObject *self, PyObject *Py_UNUSED(ignored)) ++{ ++ return _hmacopenssl_HMAC_digest_impl(self); ++} ++ ++PyDoc_STRVAR(_hmacopenssl_HMAC_hexdigest__doc__, ++"hexdigest($self, /)\n" ++"--\n" ++"\n" ++"Return hexadecimal digest of the bytes passed to the update() method so far.\n" ++"\n" ++"This may be used to exchange the value safely in email or other non-binary\n" ++"environments."); ++ ++#define _HMACOPENSSL_HMAC_HEXDIGEST_METHODDEF \ ++ {"hexdigest", (PyCFunction)_hmacopenssl_HMAC_hexdigest, METH_NOARGS, _hmacopenssl_HMAC_hexdigest__doc__}, ++ ++static PyObject * ++_hmacopenssl_HMAC_hexdigest_impl(HmacObject *self); ++ ++static PyObject * ++_hmacopenssl_HMAC_hexdigest(HmacObject *self, PyObject *Py_UNUSED(ignored)) ++{ ++ return _hmacopenssl_HMAC_hexdigest_impl(self); ++} ++/*[clinic end generated code: output=10b6e8cac6d7a2c9 input=a9049054013a1b77]*/ +diff --git a/setup.py b/setup.py +index e2e659ef7950..282aa4178ed3 100644 +--- a/setup.py ++++ b/setup.py +@@ -922,6 +922,10 @@ def detect_modules(self): + openssl_ver) + missing.append('_hashlib') + ++ exts.append( Extension('_hmacopenssl', ['_hmacopenssl.c'], ++ depends = ['hashlib.h'], ++ **ssl_args)) ++ + # RHEL: Always force OpenSSL for md5, sha1, sha256, sha512; + # don't build Python's implementations. + # sha3 and blake2 have extra functionality, so do build those: + +From 7eb3d47afdad9052f505ecbbbfab065dbbc240e0 Mon Sep 17 00:00:00 2001 +From: Marcel Plch +Date: Mon, 29 Jul 2019 12:45:11 +0200 +Subject: [PATCH 09/36] FIPS review + +* Port _hmacopenssl to multiphase init. +* Make _hmacopenssl.HMAC.copy create same type as self. +* hmac.py cosmetic nitpick +--- + Lib/hmac.py | 2 +- + Modules/_hmacopenssl.c | 112 +++++++++++++++++++++++++---------------- + 2 files changed, 70 insertions(+), 44 deletions(-) + +diff --git a/Lib/hmac.py b/Lib/hmac.py +index ed98406bd2e1..b9bf16b84ca0 100644 +--- a/Lib/hmac.py ++++ b/Lib/hmac.py +@@ -42,7 +42,7 @@ def __init__(self, key, msg = None, digestmod = None): + if _hashlib.get_fips_mode(): + raise ValueError( + 'hmac.HMAC is not available in FIPS mode. ' +- + 'Use hmac.new().' ++ 'Use hmac.new().' + ) + + if not isinstance(key, (bytes, bytearray)): +diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c +index ca95d725f019..216ed04f2360 100644 +--- a/Modules/_hmacopenssl.c ++++ b/Modules/_hmacopenssl.c +@@ -24,7 +24,10 @@ + + #include + +-static PyTypeObject HmacType; ++typedef struct hmacopenssl_state { ++ PyTypeObject *HmacType; ++} hmacopenssl_state; ++ + + typedef struct { + PyObject_HEAD +@@ -36,9 +39,9 @@ typedef struct { + #include "clinic/_hmacopenssl.c.h" + /*[clinic input] + module _hmacopenssl +-class _hmacopenssl.HMAC "HmacObject *" "&HmacType" ++class _hmacopenssl.HMAC "HmacObject *" "PyModule_GetState(module)->HmacType" + [clinic start generated code]*/ +-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c98d3f2af591c085]*/ ++/*[clinic end generated code: output=da39a3ee5e6b4b0d input=204b7f45847f57b4]*/ + + + /*[clinic input] +@@ -56,11 +59,18 @@ _hmacopenssl_new_impl(PyObject *module, Py_buffer *key, + const char *digestmod) + /*[clinic end generated code: output=46f1cb4e02921922 input=be8c0c2e4fad508c]*/ + { ++ hmacopenssl_state *state; ++ + if (digestmod == NULL) { + PyErr_SetString(PyExc_ValueError, "digestmod must be specified"); + return NULL; + } + ++ state = PyModule_GetState(module); ++ if (state == NULL) { ++ return NULL; ++ } ++ + /* name mut be lowercase */ + for (int i=0; digestmod[i]; i++) { + if ( +@@ -105,7 +115,7 @@ _hmacopenssl_new_impl(PyObject *module, Py_buffer *key, + goto error; + } + +- retval = (HmacObject *)PyObject_New(HmacObject, &HmacType); ++ retval = (HmacObject *)PyObject_New(HmacObject, state->HmacType); + if (retval == NULL) { + goto error; + } +@@ -133,7 +143,9 @@ static PyObject * + _hmacopenssl_HMAC_copy_impl(HmacObject *self) + /*[clinic end generated code: output=fe5ee41faf30dcf0 input=f5ed20feec42d8d0]*/ + { +- HmacObject *retval = (HmacObject *)PyObject_New(HmacObject, &HmacType); ++ HmacObject *retval; ++ ++ retval = (HmacObject *)PyObject_New(HmacObject, (PyTypeObject *)PyObject_Type((PyObject *)self)); + if (retval == NULL) { + return NULL; + } +@@ -147,7 +159,7 @@ _hmacopenssl_HMAC_copy_impl(HmacObject *self) + return _setException(PyExc_ValueError); + } + +- return (PyObject*)retval; ++ return (PyObject *)retval; + } + + static void +@@ -338,19 +350,24 @@ Attributes:\n\ + name -- the name, including the hash algorithm used by this object\n\ + digest_size -- number of bytes in digest() output\n"); + +-static PyTypeObject HmacType = { +- PyVarObject_HEAD_INIT(NULL, 0) +- "_hmacopenssl.HMAC", /*tp_name*/ +- sizeof(HmacObject), /*tp_basicsize*/ +- .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +- .tp_doc = hmactype_doc, +- .tp_repr = (reprfunc)_hmac_repr, +- .tp_dealloc = (destructor)_hmac_dealloc, +- .tp_methods = Hmac_methods, +- .tp_getset = Hmac_getset, +- .tp_members = Hmac_members, ++static PyType_Slot HmacType_slots[] = { ++ {Py_tp_doc, hmactype_doc}, ++ {Py_tp_repr, (reprfunc)_hmac_repr}, ++ {Py_tp_dealloc,(destructor)_hmac_dealloc}, ++ {Py_tp_methods, Hmac_methods}, ++ {Py_tp_getset, Hmac_getset}, ++ {Py_tp_members, Hmac_members}, ++ {0, NULL} ++}; ++ ++PyType_Spec HmacType_spec = { ++ "_hmacopenssl.HMAC", /* name */ ++ sizeof(HmacObject), /* basicsize */ ++ .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, ++ .slots = HmacType_slots, + }; + ++ + static struct PyMethodDef hmacopenssl_functions[] = { + _HMACOPENSSL_NEW_METHODDEF + {NULL, NULL} /* Sentinel */ +@@ -360,37 +377,46 @@ static struct PyMethodDef hmacopenssl_functions[] = { + + /* Initialize this module. */ + +- +-static struct PyModuleDef _hmacopenssl_module = { +- PyModuleDef_HEAD_INIT, +- "_hmacopenssl", +- NULL, +- -1, +- hmacopenssl_functions, +- NULL, +- NULL, +- NULL, +- NULL +-}; +- +-PyMODINIT_FUNC +-PyInit__hmacopenssl(void) +-{ ++static int ++hmacopenssl_exec(PyObject *m) { + /* TODO build EVP_functions openssl_* entries dynamically based + * on what hashes are supported rather than listing many +- * but having some be unsupported. Only init appropriate ++ * and having some unsupported. Only init appropriate + * constants. */ ++ PyObject *temp; + +- Py_TYPE(&HmacType) = &PyType_Type; +- if (PyType_Ready(&HmacType) < 0) +- return NULL; ++ temp = PyType_FromSpec(&HmacType_spec); ++ if (temp == NULL) { ++ goto fail; ++ } + +- PyObject *m = PyModule_Create(&_hmacopenssl_module); +- if (m == NULL) +- return NULL; ++ if (PyModule_AddObject(m, "HMAC", temp) == -1) { ++ goto fail; ++ } ++ ++ return 0; + +- Py_INCREF((PyObject *)&HmacType); +- PyModule_AddObject(m, "HMAC", (PyObject *)&HmacType); ++fail: ++ Py_XDECREF(temp); ++ return -1; ++} + +- return m; ++static PyModuleDef_Slot hmacopenssl_slots[] = { ++ {Py_mod_exec, hmacopenssl_exec}, ++ {0, NULL}, ++}; ++ ++static struct PyModuleDef _hmacopenssl_def = { ++ PyModuleDef_HEAD_INIT, /* m_base */ ++ .m_name = "_hmacopenssl", ++ .m_methods = hmacopenssl_functions, ++ .m_slots = hmacopenssl_slots, ++ .m_size = sizeof(hmacopenssl_state) ++}; ++ ++ ++PyMODINIT_FUNC ++PyInit__hmacopenssl(void) ++{ ++ return PyModuleDef_Init(&_hmacopenssl_def); + } + +From 641e76725f2660da75490ea0c10da52688093778 Mon Sep 17 00:00:00 2001 +From: Marcel Plch +Date: Mon, 29 Jul 2019 13:05:04 +0200 +Subject: [PATCH 10/36] revert cosmetic nitpick and remove trailing whitespace + +--- + Lib/hmac.py | 2 +- + Modules/_hmacopenssl.c | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/Lib/hmac.py b/Lib/hmac.py +index b9bf16b84ca0..ed98406bd2e1 100644 +--- a/Lib/hmac.py ++++ b/Lib/hmac.py +@@ -42,7 +42,7 @@ def __init__(self, key, msg = None, digestmod = None): + if _hashlib.get_fips_mode(): + raise ValueError( + 'hmac.HMAC is not available in FIPS mode. ' +- 'Use hmac.new().' ++ + 'Use hmac.new().' + ) + + if not isinstance(key, (bytes, bytearray)): +diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c +index 216ed04f2360..221714ca4349 100644 +--- a/Modules/_hmacopenssl.c ++++ b/Modules/_hmacopenssl.c +@@ -363,7 +363,7 @@ static PyType_Slot HmacType_slots[] = { + PyType_Spec HmacType_spec = { + "_hmacopenssl.HMAC", /* name */ + sizeof(HmacObject), /* basicsize */ +- .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, ++ .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = HmacType_slots, + }; + +@@ -407,7 +407,7 @@ static PyModuleDef_Slot hmacopenssl_slots[] = { + }; + + static struct PyModuleDef _hmacopenssl_def = { +- PyModuleDef_HEAD_INIT, /* m_base */ ++ PyModuleDef_HEAD_INIT, /* m_base */ + .m_name = "_hmacopenssl", + .m_methods = hmacopenssl_functions, + .m_slots = hmacopenssl_slots, + +From 0a1522aacb0b070632f7feaad6f92bfe8fc9c89b Mon Sep 17 00:00:00 2001 +From: Charalampos Stratakis +Date: Wed, 31 Jul 2019 15:43:43 +0200 +Subject: [PATCH 11/36] Add initial tests for various hashes under FIPS mode + +--- + Lib/test/test_fips.py | 64 +++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 64 insertions(+) + create mode 100644 Lib/test/test_fips.py + +diff --git a/Lib/test/test_fips.py b/Lib/test/test_fips.py +new file mode 100644 +index 000000000000..bee911ef405a +--- /dev/null ++++ b/Lib/test/test_fips.py +@@ -0,0 +1,64 @@ ++import unittest ++import hmac, _hmacopenssl ++import hashlib, _hashlib ++ ++ ++ ++class HashlibFipsTests(unittest.TestCase): ++ ++ @unittest.skipUnless(hashlib.get_fips_mode(), "Test only when FIPS is enabled") ++ def test_fips_imports(self): ++ """blake2s and blake2b should fail to import in FIPS mode ++ """ ++ with self.assertRaises(ValueError, msg='blake2s not available in FIPS'): ++ m = hashlib.blake2s() ++ with self.assertRaises(ValueError, msg='blake2b not available in FIPS'): ++ m = hashlib.blake2b() ++ ++ def compare_hashes(self, python_hash, openssl_hash): ++ """ ++ Compare between the python implementation and the openssl one that the digests ++ are the same ++ """ ++ if python_hash.name.startswith('shake_128'): ++ m = python_hash.hexdigest(16) ++ elif python_hash.name.startswith('shake_256'): ++ m = python_hash.hexdigest(32) ++ else: ++ m = python_hash.hexdigest() ++ h = openssl_hash.hexdigest() ++ ++ self.assertEqual(m, h) ++ ++ @unittest.skipIf(hashlib.get_fips_mode(), "blake2 hashes are not available under FIPS") ++ def test_blake2_hashes(self): ++ self.compare_hashes(hashlib.blake2b(b'abc'), _hashlib.openssl_blake2b(b'abc')) ++ self.compare_hashes(hashlib.blake2s(b'abc'), _hashlib.openssl_blake2s(b'abc')) ++ ++ def test_sha3_hashes(self): ++ self.compare_hashes(hashlib.sha3_224(b'abc'), _hashlib.openssl_sha3_224(b'abc')) ++ self.compare_hashes(hashlib.sha3_256(b'abc'), _hashlib.openssl_sha3_256(b'abc')) ++ self.compare_hashes(hashlib.sha3_384(b'abc'), _hashlib.openssl_sha3_384(b'abc')) ++ self.compare_hashes(hashlib.sha3_512(b'abc'), _hashlib.openssl_sha3_512(b'abc')) ++ ++ @unittest.skipIf(hashlib.get_fips_mode(), "shake hashes are not available under FIPS") ++ def test_shake_hashes(self): ++ self.compare_hashes(hashlib.shake_128(b'abc'), _hashlib.openssl_shake_128(b'abc')) ++ self.compare_hashes(hashlib.shake_256(b'abc'), _hashlib.openssl_shake_256(b'abc')) ++ ++ def test_sha(self): ++ self.compare_hashes(hashlib.sha1(b'abc'), _hashlib.openssl_sha1(b'abc')) ++ self.compare_hashes(hashlib.sha224(b'abc'), _hashlib.openssl_sha224(b'abc')) ++ self.compare_hashes(hashlib.sha256(b'abc'), _hashlib.openssl_sha256(b'abc')) ++ self.compare_hashes(hashlib.sha384(b'abc'), _hashlib.openssl_sha384(b'abc')) ++ self.compare_hashes(hashlib.sha512(b'abc'), _hashlib.openssl_sha512(b'abc')) ++ ++ def test_hmac_digests(self): ++ self.compare_hashes(_hmacopenssl.new(b'My hovercraft is full of eels', digestmod='sha384'), ++ hmac.new(b'My hovercraft is full of eels', digestmod='sha384')) ++ ++ ++ ++ ++if __name__ == "__main__": ++ unittest.main() + +From f20b01bb18523587cb5beedc67a8ee08a502788d Mon Sep 17 00:00:00 2001 +From: Marcel Plch +Date: Thu, 1 Aug 2019 16:39:37 +0200 +Subject: [PATCH 12/36] Initialize HMAC type. + +--- + Modules/_hmacopenssl.c | 12 ++++++++---- + 1 file changed, 8 insertions(+), 4 deletions(-) + +diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c +index 221714ca4349..239445a0831b 100644 +--- a/Modules/_hmacopenssl.c ++++ b/Modules/_hmacopenssl.c +@@ -22,12 +22,12 @@ + #include "_hashopenssl.h" + + +-#include + + typedef struct hmacopenssl_state { + PyTypeObject *HmacType; + } hmacopenssl_state; + ++#include + + typedef struct { + PyObject_HEAD +@@ -39,7 +39,7 @@ typedef struct { + #include "clinic/_hmacopenssl.c.h" + /*[clinic input] + module _hmacopenssl +-class _hmacopenssl.HMAC "HmacObject *" "PyModule_GetState(module)->HmacType" ++class _hmacopenssl.HMAC "HmacObject *" "((hmacopenssl_state *)PyModule_GetState(module))->HmacType" + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=204b7f45847f57b4]*/ + +@@ -71,7 +71,7 @@ _hmacopenssl_new_impl(PyObject *module, Py_buffer *key, + return NULL; + } + +- /* name mut be lowercase */ ++ /* name must be lowercase */ + for (int i=0; digestmod[i]; i++) { + if ( + ((digestmod[i] < 'a') || (digestmod[i] > 'z')) +@@ -383,7 +383,8 @@ hmacopenssl_exec(PyObject *m) { + * on what hashes are supported rather than listing many + * and having some unsupported. Only init appropriate + * constants. */ +- PyObject *temp; ++ PyObject *temp = NULL; ++ hmacopenssl_state *state; + + temp = PyType_FromSpec(&HmacType_spec); + if (temp == NULL) { +@@ -394,6 +395,9 @@ hmacopenssl_exec(PyObject *m) { + goto fail; + } + ++ state = PyModule_GetState(m); ++ state->HmacType = (PyTypeObject *)temp; ++ + return 0; + + fail: + +From b819ca654873ea36144ce1cb6f8c422a6d2f5d7d Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Thu, 1 Aug 2019 17:57:05 +0200 +Subject: [PATCH 13/36] Use a stronger hash in multiprocessing handshake + +Adapted from patch by David Malcolm, +https://bugs.python.org/issue17258 +--- + Lib/multiprocessing/connection.py | 8 ++++++-- + 1 file changed, 6 insertions(+), 2 deletions(-) + +diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py +index d3797503a755..a0b1538f88b3 100644 +--- a/Lib/multiprocessing/connection.py ++++ b/Lib/multiprocessing/connection.py +@@ -42,6 +42,10 @@ + # A very generous timeout when it comes to local connections... + CONNECTION_TIMEOUT = 20. + ++# The hmac module implicitly defaults to using MD5. ++# Support using a stronger algorithm for the challenge/response code: ++HMAC_DIGEST_NAME='sha256' ++ + _mmap_counter = itertools.count() + + default_family = 'AF_INET' +@@ -718,7 +722,7 @@ def deliver_challenge(connection, authkey): + assert isinstance(authkey, bytes) + message = os.urandom(MESSAGE_LENGTH) + connection.send_bytes(CHALLENGE + message) +- digest = hmac.new(authkey, message, 'md5').digest() ++ digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest() + response = connection.recv_bytes(256) # reject large message + if response == digest: + connection.send_bytes(WELCOME) +@@ -732,7 +736,7 @@ def answer_challenge(connection, authkey): + message = connection.recv_bytes(256) # reject large message + assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message + message = message[len(CHALLENGE):] +- digest = hmac.new(authkey, message, 'md5').digest() ++ digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest() + connection.send_bytes(digest) + response = connection.recv_bytes(256) # reject large message + if response != WELCOME: + +From 01950bb63c0a52faf5304e3978a6308641305a03 Mon Sep 17 00:00:00 2001 +From: Marcel Plch +Date: Fri, 2 Aug 2019 17:36:01 +0200 +Subject: [PATCH 14/36] Fix refcounting + +--- + Modules/_hmacopenssl.c | 35 ++++++++++++++++++++++++++++++++++- + 1 file changed, 34 insertions(+), 1 deletion(-) + +diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c +index 239445a0831b..9c2882833d1c 100644 +--- a/Modules/_hmacopenssl.c ++++ b/Modules/_hmacopenssl.c +@@ -373,6 +373,34 @@ static struct PyMethodDef hmacopenssl_functions[] = { + {NULL, NULL} /* Sentinel */ + }; + ++static int ++hmacopenssl_traverse(PyObject *self, visitproc visit, void *arg) ++{ ++ hmacopenssl_state *state; ++ ++ state = PyModule_GetState(self); ++ ++ if (state) { ++ Py_VISIT(state->HmacType); ++ } ++ ++ return 0; ++} ++ ++static int ++hmacopenssl_clear(PyObject *self) ++{ ++ hmacopenssl_state *state; ++ ++ state = PyModule_GetState(self); ++ ++ if (state) { ++ Py_CLEAR(state->HmacType); ++ } ++ ++ return 0; ++} ++ + + + /* Initialize this module. */ +@@ -396,7 +424,10 @@ hmacopenssl_exec(PyObject *m) { + } + + state = PyModule_GetState(m); ++ + state->HmacType = (PyTypeObject *)temp; ++ Py_INCREF(temp); ++ + + return 0; + +@@ -415,7 +446,9 @@ static struct PyModuleDef _hmacopenssl_def = { + .m_name = "_hmacopenssl", + .m_methods = hmacopenssl_functions, + .m_slots = hmacopenssl_slots, +- .m_size = sizeof(hmacopenssl_state) ++ .m_size = sizeof(hmacopenssl_state), ++ .m_traverse = hmacopenssl_traverse, ++ .m_clear = hmacopenssl_clear + }; + + + +From 823eae4610e3235f60a89715ba37a970c18a7c0b Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 13:37:05 +0200 +Subject: [PATCH 15/36] hmac: Don't default to md5 in FIPS mode + +--- + Lib/hmac.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Lib/hmac.py b/Lib/hmac.py +index ed98406bd2e1..7b8821edd582 100644 +--- a/Lib/hmac.py ++++ b/Lib/hmac.py +@@ -165,7 +165,7 @@ def new(key, msg = None, digestmod = None): + """ + if _hashlib.get_fips_mode(): + if digestmod is None: +- digestmod = 'md5' ++ raise ValueError("'digestmod' argument is mandatory in FIPS mode") + name = _get_openssl_name(digestmod) + result = _hmacopenssl.new(key, digestmod=name) + if msg: + +From fd74d1c00d95542eef3c238f990da5bc7fc05619 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 14:20:58 +0200 +Subject: [PATCH 16/36] Make _hmacopenssl.HMAC subclassable; subclass it as + hmac.HMAC under FIPS + +This removes the _hmacopenssl.new function. +--- + Lib/hmac.py | 27 +++++++----- + Lib/test/test_fips.py | 2 +- + Modules/_hmacopenssl.c | 75 ++++++++++++++++----------------- + Modules/clinic/_hmacopenssl.c.h | 39 +---------------- + 4 files changed, 56 insertions(+), 87 deletions(-) + +diff --git a/Lib/hmac.py b/Lib/hmac.py +index 7b8821edd582..d479c5a4492f 100644 +--- a/Lib/hmac.py ++++ b/Lib/hmac.py +@@ -141,6 +141,8 @@ def hexdigest(self): + + + def _get_openssl_name(digestmod): ++ if digestmod is None: ++ raise ValueError("'digestmod' argument is mandatory in FIPS mode") + if isinstance(digestmod, str): + return digestmod.lower() + elif callable(digestmod): +@@ -152,6 +154,20 @@ def _get_openssl_name(digestmod): + + return digestmod.name.lower().replace('_', '-') + ++ ++class HMAC_openssl(_hmacopenssl.HMAC): ++ def __new__(cls, key, msg = None, digestmod = None): ++ name = _get_openssl_name(digestmod) ++ result = _hmacopenssl.HMAC.__new__(cls, key, digestmod=name) ++ if msg: ++ result.update(msg) ++ return result ++ ++ ++if _hashlib.get_fips_mode(): ++ HMAC = HMAC_openssl ++ ++ + def new(key, msg = None, digestmod = None): + """Create a new hashing object and return it. + +@@ -163,13 +179,4 @@ def new(key, msg = None, digestmod = None): + method, and can ask for the hash value at any time by calling its digest() + method. + """ +- if _hashlib.get_fips_mode(): +- if digestmod is None: +- raise ValueError("'digestmod' argument is mandatory in FIPS mode") +- name = _get_openssl_name(digestmod) +- result = _hmacopenssl.new(key, digestmod=name) +- if msg: +- result.update(msg) +- return result +- else: +- return HMAC(key, msg, digestmod) ++ return HMAC(key, msg, digestmod) +diff --git a/Lib/test/test_fips.py b/Lib/test/test_fips.py +index bee911ef405a..34812e6098ae 100644 +--- a/Lib/test/test_fips.py ++++ b/Lib/test/test_fips.py +@@ -54,7 +54,7 @@ def test_sha(self): + self.compare_hashes(hashlib.sha512(b'abc'), _hashlib.openssl_sha512(b'abc')) + + def test_hmac_digests(self): +- self.compare_hashes(_hmacopenssl.new(b'My hovercraft is full of eels', digestmod='sha384'), ++ self.compare_hashes(_hmacopenssl.HMAC(b'My hovercraft is full of eels', digestmod='sha384'), + hmac.new(b'My hovercraft is full of eels', digestmod='sha384')) + + +diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c +index 9c2882833d1c..7d3d9739f3ab 100644 +--- a/Modules/_hmacopenssl.c ++++ b/Modules/_hmacopenssl.c +@@ -41,33 +41,25 @@ typedef struct { + module _hmacopenssl + class _hmacopenssl.HMAC "HmacObject *" "((hmacopenssl_state *)PyModule_GetState(module))->HmacType" + [clinic start generated code]*/ +-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=204b7f45847f57b4]*/ ++/*[clinic end generated code: output=da39a3ee5e6b4b0d input=9fe07a087adc2cf9]*/ + + +-/*[clinic input] +-_hmacopenssl.new +- +- key: Py_buffer +- * +- digestmod: str +- +-Return a new hmac object. +-[clinic start generated code]*/ +- + static PyObject * +-_hmacopenssl_new_impl(PyObject *module, Py_buffer *key, +- const char *digestmod) +-/*[clinic end generated code: output=46f1cb4e02921922 input=be8c0c2e4fad508c]*/ ++Hmac_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) + { +- hmacopenssl_state *state; +- +- if (digestmod == NULL) { +- PyErr_SetString(PyExc_ValueError, "digestmod must be specified"); ++ static char *kwarg_names[] = {"key", "digestmod", NULL}; ++ Py_buffer key = {NULL, NULL}; ++ char *digestmod = NULL; ++ ++ int ret = PyArg_ParseTupleAndKeywords( ++ args, kwds, "y*|$s:_hmacopenssl.HMAC", kwarg_names, ++ &key, &digestmod); ++ if (ret == 0) { + return NULL; + } + +- state = PyModule_GetState(module); +- if (state == NULL) { ++ if (digestmod == NULL) { ++ PyErr_SetString(PyExc_ValueError, "digestmod must be specified"); + return NULL; + } + +@@ -106,8 +98,8 @@ _hmacopenssl_new_impl(PyObject *module, Py_buffer *key, + + int r = HMAC_Init_ex( + ctx, +- (const char*)key->buf, +- key->len, ++ (const char*)key.buf, ++ key.len, + digest, + NULL /*impl*/); + if (r == 0) { +@@ -115,7 +107,10 @@ _hmacopenssl_new_impl(PyObject *module, Py_buffer *key, + goto error; + } + +- retval = (HmacObject *)PyObject_New(HmacObject, state->HmacType); ++ PyBuffer_Release(&key); ++ key.buf = NULL; ++ ++ retval = (HmacObject *)subtype->tp_alloc(subtype, 0); + if (retval == NULL) { + goto error; + } +@@ -130,6 +125,7 @@ _hmacopenssl_new_impl(PyObject *module, Py_buffer *key, + if (ctx) HMAC_CTX_free(ctx); + if (name) Py_DECREF(name); + if (retval) PyObject_Del(name); ++ if (key.buf) PyBuffer_Release(&key); + return NULL; + } + +@@ -145,19 +141,27 @@ _hmacopenssl_HMAC_copy_impl(HmacObject *self) + { + HmacObject *retval; + +- retval = (HmacObject *)PyObject_New(HmacObject, (PyTypeObject *)PyObject_Type((PyObject *)self)); ++ HMAC_CTX *ctx = HMAC_CTX_new(); ++ if (ctx == NULL) { ++ return _setException(PyExc_ValueError); ++ } ++ ++ int r = HMAC_CTX_copy(ctx, self->ctx); ++ if (r == 0) { ++ HMAC_CTX_free(ctx); ++ return _setException(PyExc_ValueError); ++ } ++ ++ retval = (HmacObject *)Py_TYPE(self)->tp_alloc(Py_TYPE(self), 0); + if (retval == NULL) { ++ HMAC_CTX_free(ctx); + return NULL; + } +- ++ retval->ctx = ctx; + Py_INCREF(self->name); + retval->name = self->name; + +- int r = HMAC_CTX_copy(retval->ctx, self->ctx); +- if (r == 0) { +- PyObject_Del(retval); +- return _setException(PyExc_ValueError); +- } ++ retval->lock = NULL; + + return (PyObject *)retval; + } +@@ -169,8 +173,8 @@ _hmac_dealloc(HmacObject *self) + PyThread_free_lock(self->lock); + } + HMAC_CTX_free(self->ctx); +- Py_XDECREF(self->name); +- PyObject_Del(self); ++ Py_CLEAR(self->name); ++ Py_TYPE(self)->tp_free(self); + } + + static PyObject * +@@ -357,6 +361,7 @@ static PyType_Slot HmacType_slots[] = { + {Py_tp_methods, Hmac_methods}, + {Py_tp_getset, Hmac_getset}, + {Py_tp_members, Hmac_members}, ++ {Py_tp_new, Hmac_new}, + {0, NULL} + }; + +@@ -368,11 +373,6 @@ PyType_Spec HmacType_spec = { + }; + + +-static struct PyMethodDef hmacopenssl_functions[] = { +- _HMACOPENSSL_NEW_METHODDEF +- {NULL, NULL} /* Sentinel */ +-}; +- + static int + hmacopenssl_traverse(PyObject *self, visitproc visit, void *arg) + { +@@ -444,7 +444,6 @@ static PyModuleDef_Slot hmacopenssl_slots[] = { + static struct PyModuleDef _hmacopenssl_def = { + PyModuleDef_HEAD_INIT, /* m_base */ + .m_name = "_hmacopenssl", +- .m_methods = hmacopenssl_functions, + .m_slots = hmacopenssl_slots, + .m_size = sizeof(hmacopenssl_state), + .m_traverse = hmacopenssl_traverse, +diff --git a/Modules/clinic/_hmacopenssl.c.h b/Modules/clinic/_hmacopenssl.c.h +index b472a6eddd34..861acc11bfd9 100644 +--- a/Modules/clinic/_hmacopenssl.c.h ++++ b/Modules/clinic/_hmacopenssl.c.h +@@ -2,43 +2,6 @@ + preserve + [clinic start generated code]*/ + +-PyDoc_STRVAR(_hmacopenssl_new__doc__, +-"new($module, /, key, *, digestmod)\n" +-"--\n" +-"\n" +-"Return a new hmac object."); +- +-#define _HMACOPENSSL_NEW_METHODDEF \ +- {"new", (PyCFunction)_hmacopenssl_new, METH_FASTCALL, _hmacopenssl_new__doc__}, +- +-static PyObject * +-_hmacopenssl_new_impl(PyObject *module, Py_buffer *key, +- const char *digestmod); +- +-static PyObject * +-_hmacopenssl_new(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +-{ +- PyObject *return_value = NULL; +- static const char * const _keywords[] = {"key", "digestmod", NULL}; +- static _PyArg_Parser _parser = {"y*$s:new", _keywords, 0}; +- Py_buffer key = {NULL, NULL}; +- const char *digestmod; +- +- if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, +- &key, &digestmod)) { +- goto exit; +- } +- return_value = _hmacopenssl_new_impl(module, &key, digestmod); +- +-exit: +- /* Cleanup for key */ +- if (key.obj) { +- PyBuffer_Release(&key); +- } +- +- return return_value; +-} +- + PyDoc_STRVAR(_hmacopenssl_HMAC_copy__doc__, + "copy($self, /)\n" + "--\n" +@@ -130,4 +93,4 @@ _hmacopenssl_HMAC_hexdigest(HmacObject *self, PyObject *Py_UNUSED(ignored)) + { + return _hmacopenssl_HMAC_hexdigest_impl(self); + } +-/*[clinic end generated code: output=10b6e8cac6d7a2c9 input=a9049054013a1b77]*/ ++/*[clinic end generated code: output=d93ad460795d49b5 input=a9049054013a1b77]*/ + +From d3b322a12c10890fee2bc4837a3d37f0a81455a6 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 16:10:36 +0200 +Subject: [PATCH 17/36] Fix _hmacopenssl.HMAC.block_size + +--- + Modules/_hmacopenssl.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c +index 7d3d9739f3ab..a24c8ba0229d 100644 +--- a/Modules/_hmacopenssl.c ++++ b/Modules/_hmacopenssl.c +@@ -318,7 +318,7 @@ _hmacopenssl_get_block_size(HmacObject *self, void *closure) + if (md == NULL) { + return _setException(PyExc_ValueError); + } +- return PyLong_FromLong(EVP_MD_size(md)); ++ return PyLong_FromLong(EVP_MD_block_size(md)); + } + + static PyMethodDef Hmac_methods[] = { + +From e643e9c61bce62bcc3caed8c022f1afee979feb8 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 14:45:52 +0200 +Subject: [PATCH 18/36] Skip hanging test + +--- + Lib/test/test_logging.py | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py +index 763a5d1df0e3..0b229465983c 100644 +--- a/Lib/test/test_logging.py ++++ b/Lib/test/test_logging.py +@@ -46,6 +46,8 @@ + import unittest + import warnings + import weakref ++import hashlib ++ + try: + import threading + # The following imports are needed only for tests which +@@ -1815,6 +1817,7 @@ def handle_request(self, request): + request.end_headers() + self.handled.set() + ++ @unittest.skipIf(hashlib.get_fips_mode(), 'Hangs in FIPS mode.') + def test_output(self): + # The log message sent to the HTTPHandler is properly received. + logger = logging.getLogger("http") + +From 30e11ab9d19beea8dafe96da7f47bf1513849f3d Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 15:02:08 +0200 +Subject: [PATCH 19/36] distutils upload: Skip md5 checksum in FIPS mode + +--- + Lib/distutils/command/upload.py | 11 ++++++++++- + Lib/distutils/tests/test_upload.py | 13 +++++++++++-- + 2 files changed, 21 insertions(+), 3 deletions(-) + +diff --git a/Lib/distutils/command/upload.py b/Lib/distutils/command/upload.py +index 32dda359badb..0edb39efd4cb 100644 +--- a/Lib/distutils/command/upload.py ++++ b/Lib/distutils/command/upload.py +@@ -102,7 +102,6 @@ def upload_file(self, command, pyversion, filename): + 'content': (os.path.basename(filename),content), + 'filetype': command, + 'pyversion': pyversion, +- 'md5_digest': hashlib.md5(content).hexdigest(), + + # additional meta-data + 'metadata_version': '1.0', +@@ -121,6 +120,16 @@ def upload_file(self, command, pyversion, filename): + 'requires': meta.get_requires(), + 'obsoletes': meta.get_obsoletes(), + } ++ try: ++ digest = hashlib.md5(content).hexdigest() ++ except ValueError as e: ++ msg = 'calculating md5 checksum failed: %s' % e ++ self.announce(msg, log.ERROR) ++ if not hashlib.get_fips_mode(): ++ # this really shouldn't fail ++ raise ++ else: ++ data['md5_digest'] = digest + comment = '' + if command == 'bdist_rpm': + dist, version, id = platform.dist() +diff --git a/Lib/distutils/tests/test_upload.py b/Lib/distutils/tests/test_upload.py +index c17d8e7d54e9..b4b64e97737d 100644 +--- a/Lib/distutils/tests/test_upload.py ++++ b/Lib/distutils/tests/test_upload.py +@@ -3,6 +3,7 @@ + import unittest + import unittest.mock as mock + from urllib.request import HTTPError ++import hashlib + + from test.support import run_unittest + +@@ -130,7 +131,11 @@ def test_upload(self): + + # what did we send ? + headers = dict(self.last_open.req.headers) +- self.assertEqual(headers['Content-length'], '2162') ++ if hashlib.get_fips_mode(): ++ # md5 hash is omitted ++ self.assertEqual(headers['Content-length'], '2020') ++ else: ++ self.assertEqual(headers['Content-length'], '2162') + content_type = headers['Content-type'] + self.assertTrue(content_type.startswith('multipart/form-data')) + self.assertEqual(self.last_open.req.get_method(), 'POST') +@@ -166,7 +171,11 @@ def test_upload_correct_cr(self): + cmd.run() + + headers = dict(self.last_open.req.headers) +- self.assertEqual(headers['Content-length'], '2172') ++ if hashlib.get_fips_mode(): ++ # md5 hash is omitted ++ self.assertEqual(headers['Content-length'], '2030') ++ else: ++ self.assertEqual(headers['Content-length'], '2172') + self.assertIn(b'long description\r', self.last_open.req.data) + + def test_upload_fails(self): + +From 8eb2ce39572904704ada072e25d8e6bf5727f492 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 15:32:25 +0200 +Subject: [PATCH 20/36] Fix HMAC tests on FIPS mode + +--- + Lib/hmac.py | 3 +++ + Lib/test/test_hmac.py | 50 ++++++++++++++++++++++++++++++++++--------- + 2 files changed, 43 insertions(+), 10 deletions(-) + +diff --git a/Lib/hmac.py b/Lib/hmac.py +index d479c5a4492f..7af94c39ed63 100644 +--- a/Lib/hmac.py ++++ b/Lib/hmac.py +@@ -157,6 +157,9 @@ def _get_openssl_name(digestmod): + + class HMAC_openssl(_hmacopenssl.HMAC): + def __new__(cls, key, msg = None, digestmod = None): ++ if not isinstance(key, (bytes, bytearray)): ++ raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) ++ + name = _get_openssl_name(digestmod) + result = _hmacopenssl.HMAC.__new__(cls, key, digestmod=name) + if msg: +diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py +index 067e13f1079a..6f53a6d1fd5b 100644 +--- a/Lib/test/test_hmac.py ++++ b/Lib/test/test_hmac.py +@@ -17,6 +17,7 @@ def wrapper(*args, **kwargs): + + class TestVectorsTestCase(unittest.TestCase): + ++ @unittest.skipIf(hashlib.get_fips_mode(), 'md5 unacceptable in FIPS mode.') + def test_md5_vectors(self): + # Test the HMAC module against test vectors from the RFC. + +@@ -242,6 +243,7 @@ def test_sha384_rfc4231(self): + def test_sha512_rfc4231(self): + self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128) + ++ @unittest.skipIf(hashlib.get_fips_mode(), 'MockCrazyHash unacceptable in FIPS mode.') + def test_legacy_block_size_warnings(self): + class MockCrazyHash(object): + """Ain't no block_size attribute here.""" +@@ -264,6 +266,7 @@ def digest(self): + hmac.HMAC(b'a', b'b', digestmod=MockCrazyHash) + self.fail('Expected warning about small block_size') + ++ @unittest.skipIf(hashlib.get_fips_mode(), 'md5 is not default in FIPS mode.') + def test_with_digestmod_warning(self): + with self.assertWarns(PendingDeprecationWarning): + key = b"\x0b" * 16 +@@ -275,6 +278,7 @@ def test_with_digestmod_warning(self): + + class ConstructorTestCase(unittest.TestCase): + ++ @unittest.skipIf(hashlib.get_fips_mode(), 'md5 is not default in FIPS mode.') + @ignore_warning + def test_normal(self): + # Standard constructor call. +@@ -284,6 +288,14 @@ def test_normal(self): + except Exception: + self.fail("Standard constructor call raised exception.") + ++ def test_normal_digestmod(self): ++ # Standard constructor call. ++ failed = 0 ++ try: ++ h = hmac.HMAC(b"key", digestmod='sha1') ++ except Exception: ++ self.fail("Standard constructor call raised exception.") ++ + @ignore_warning + def test_with_str_key(self): + # Pass a key of type str, which is an error, because it expects a key +@@ -302,25 +314,25 @@ def test_dot_new_with_str_key(self): + def test_withtext(self): + # Constructor call with text. + try: +- h = hmac.HMAC(b"key", b"hash this!") ++ h = hmac.HMAC(b"key", b"hash this!", digestmod='sha1') + except Exception: + self.fail("Constructor call with text argument raised exception.") +- self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864') ++ self.assertEqual(h.hexdigest(), '3f2e20f3e2f006270db98760b9725a008c5bd114') + + def test_with_bytearray(self): + try: + h = hmac.HMAC(bytearray(b"key"), bytearray(b"hash this!"), +- digestmod="md5") ++ digestmod="sha1") + except Exception: + self.fail("Constructor call with bytearray arguments raised exception.") +- self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864') ++ self.assertEqual(h.hexdigest(), '3f2e20f3e2f006270db98760b9725a008c5bd114') + + def test_with_memoryview_msg(self): + try: +- h = hmac.HMAC(b"key", memoryview(b"hash this!"), digestmod="md5") ++ h = hmac.HMAC(b"key", memoryview(b"hash this!"), digestmod="sha1") + except Exception: + self.fail("Constructor call with memoryview msg raised exception.") +- self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864') ++ self.assertEqual(h.hexdigest(), '3f2e20f3e2f006270db98760b9725a008c5bd114') + + def test_withmodule(self): + # Constructor call with text and digest module. +@@ -331,6 +343,7 @@ def test_withmodule(self): + + class SanityTestCase(unittest.TestCase): + ++ @unittest.skipIf(hashlib.get_fips_mode(), "md5 is not default in FIPS mode") + @ignore_warning + def test_default_is_md5(self): + # Testing if HMAC defaults to MD5 algorithm. +@@ -342,7 +355,7 @@ def test_exercise_all_methods(self): + # Exercising all methods once. + # This must not raise any exceptions + try: +- h = hmac.HMAC(b"my secret key", digestmod="md5") ++ h = hmac.HMAC(b"my secret key", digestmod="sha1") + h.update(b"compute the hash of this text!") + dig = h.digest() + dig = h.hexdigest() +@@ -352,9 +365,10 @@ def test_exercise_all_methods(self): + + class CopyTestCase(unittest.TestCase): + ++ @unittest.skipIf(hashlib.get_fips_mode(), "Internal attributes unavailable in FIPS mode") + def test_attributes(self): + # Testing if attributes are of same type. +- h1 = hmac.HMAC(b"key", digestmod="md5") ++ h1 = hmac.HMAC(b"key", digestmod="sha1") + h2 = h1.copy() + self.assertTrue(h1.digest_cons == h2.digest_cons, + "digest constructors don't match.") +@@ -363,9 +377,10 @@ def test_attributes(self): + self.assertEqual(type(h1.outer), type(h2.outer), + "Types of outer don't match.") + ++ @unittest.skipIf(hashlib.get_fips_mode(), "Internal attributes unavailable in FIPS mode") + def test_realcopy(self): + # Testing if the copy method created a real copy. +- h1 = hmac.HMAC(b"key", digestmod="md5") ++ h1 = hmac.HMAC(b"key", digestmod="sha1") + h2 = h1.copy() + # Using id() in case somebody has overridden __eq__/__ne__. + self.assertTrue(id(h1) != id(h2), "No real copy of the HMAC instance.") +@@ -374,9 +389,24 @@ def test_realcopy(self): + self.assertTrue(id(h1.outer) != id(h2.outer), + "No real copy of the attribute 'outer'.") + ++ def test_realcopy(self): ++ # Testing if the copy method created a real copy. ++ h1 = hmac.HMAC(b"key", digestmod="sha1") ++ h2 = h1.copy() ++ # Using id() in case somebody has overridden __eq__/__ne__. ++ self.assertTrue(id(h1) != id(h2), "No real copy of the HMAC instance.") ++ old_digest = h1.digest() ++ assert h1.digest() == h2.digest() ++ h1.update(b'hi') ++ assert h1.digest() != h2.digest() ++ assert h2.digest() == old_digest ++ new_digest = h1.digest() ++ h2.update(b'hi') ++ assert h1.digest() == h2.digest() == new_digest ++ + def test_equality(self): + # Testing if the copy has the same digests. +- h1 = hmac.HMAC(b"key", digestmod="md5") ++ h1 = hmac.HMAC(b"key", digestmod="sha1") + h1.update(b"some random text") + h2 = h1.copy() + self.assertEqual(h1.digest(), h2.digest(), + +From d7ac1f3fe2f7024d582ba9d0f10008499f832b76 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 16:24:40 +0200 +Subject: [PATCH 21/36] test_smtplib: Skip tests of CRAM-MD5 auth + +--- + Lib/test/test_smtplib.py | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py +index 87047514e7aa..150e9fbcf053 100644 +--- a/Lib/test/test_smtplib.py ++++ b/Lib/test/test_smtplib.py +@@ -15,6 +15,7 @@ + import select + import errno + import textwrap ++import hashlib + + import unittest + from test import support, mock_socket +@@ -968,6 +969,7 @@ def testAUTH_LOGIN(self): + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + ++ @unittest.skipIf(hashlib.get_fips_mode(), "md5 auth unacceptable in FIPS mode") + def testAUTH_CRAM_MD5(self): + self.serv.add_feature("AUTH CRAM-MD5") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) +@@ -975,6 +977,7 @@ def testAUTH_CRAM_MD5(self): + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + ++ @unittest.skipIf(hashlib.get_fips_mode(), "md5 auth unacceptable in FIPS mode") + def testAUTH_multiple(self): + # Test that multiple authentication methods are tried. + self.serv.add_feature("AUTH BOGUS PLAIN LOGIN CRAM-MD5") +@@ -983,6 +986,7 @@ def testAUTH_multiple(self): + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + ++ @unittest.skipIf(hashlib.get_fips_mode(), "md5 auth unacceptable in FIPS mode") + def test_auth_function(self): + supported = {'CRAM-MD5', 'PLAIN', 'LOGIN'} + for mechanism in supported: + +From d1e2300f593d8c32bd3d23cd43eae02192e3eb03 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 16:34:49 +0200 +Subject: [PATCH 22/36] test_tarfile: Replace md5 checksums with sha1 + +--- + Lib/test/test_tarfile.py | 50 ++++++++++++++++++++-------------------- + 1 file changed, 25 insertions(+), 25 deletions(-) + +diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py +index 4cd7d5370f58..c8c8eebd3194 100644 +--- a/Lib/test/test_tarfile.py ++++ b/Lib/test/test_tarfile.py +@@ -1,7 +1,7 @@ + import sys + import os + import io +-from hashlib import md5 ++from hashlib import sha1 + from contextlib import contextmanager + from random import Random + import pathlib +@@ -27,8 +27,8 @@ + except ImportError: + lzma = None + +-def md5sum(data): +- return md5(data).hexdigest() ++def sha1sum(data): ++ return sha1(data).hexdigest() + + TEMPDIR = os.path.abspath(support.TESTFN) + "-tardir" + tarextdir = TEMPDIR + '-extract-test' +@@ -39,8 +39,8 @@ def md5sum(data): + tmpname = os.path.join(TEMPDIR, "tmp.tar") + dotlessname = os.path.join(TEMPDIR, "testtar") + +-md5_regtype = "65f477c818ad9e15f7feab0c6d37742f" +-md5_sparse = "a54fbc4ca4f4399a90e1b27164012fc6" ++sha1_regtype = "bc5bb0da4d26c3e37a76d2fbf89a2a9972aeceaa" ++sha1_sparse = "693f6a770ef62ebdffc5666bd426b9506e6d8285" + + + class TarTest: +@@ -95,7 +95,7 @@ def test_fileobj_regular_file(self): + data = fobj.read() + self.assertEqual(len(data), tarinfo.size, + "regular file extraction failed") +- self.assertEqual(md5sum(data), md5_regtype, ++ self.assertEqual(sha1sum(data), sha1_regtype, + "regular file extraction failed") + + def test_fileobj_readlines(self): +@@ -180,7 +180,7 @@ def test_fileobj_text(self): + with self.tar.extractfile("ustar/regtype") as fobj: + fobj = io.TextIOWrapper(fobj) + data = fobj.read().encode("iso8859-1") +- self.assertEqual(md5sum(data), md5_regtype) ++ self.assertEqual(sha1sum(data), sha1_regtype) + try: + fobj.seek(100) + except AttributeError: +@@ -546,13 +546,13 @@ def test_extract_hardlink(self): + self.addCleanup(support.unlink, os.path.join(TEMPDIR, "ustar/lnktype")) + with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f: + data = f.read() +- self.assertEqual(md5sum(data), md5_regtype) ++ self.assertEqual(sha1sum(data), sha1_regtype) + + tar.extract("ustar/symtype", TEMPDIR) + self.addCleanup(support.unlink, os.path.join(TEMPDIR, "ustar/symtype")) + with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f: + data = f.read() +- self.assertEqual(md5sum(data), md5_regtype) ++ self.assertEqual(sha1sum(data), sha1_regtype) + + def test_extractall(self): + # Test if extractall() correctly restores directory permissions +@@ -687,7 +687,7 @@ def test_fileobj_regular_file(self): + data = fobj.read() + self.assertEqual(len(data), tarinfo.size, + "regular file extraction failed") +- self.assertEqual(md5sum(data), md5_regtype, ++ self.assertEqual(sha1sum(data), sha1_regtype, + "regular file extraction failed") + + def test_provoke_stream_error(self): +@@ -799,8 +799,8 @@ class MemberReadTest(ReadTest, unittest.TestCase): + def _test_member(self, tarinfo, chksum=None, **kwargs): + if chksum is not None: + with self.tar.extractfile(tarinfo) as f: +- self.assertEqual(md5sum(f.read()), chksum, +- "wrong md5sum for %s" % tarinfo.name) ++ self.assertEqual(sha1sum(f.read()), chksum, ++ "wrong sha1sum for %s" % tarinfo.name) + + kwargs["mtime"] = 0o7606136617 + kwargs["uid"] = 1000 +@@ -815,11 +815,11 @@ def _test_member(self, tarinfo, chksum=None, **kwargs): + + def test_find_regtype(self): + tarinfo = self.tar.getmember("ustar/regtype") +- self._test_member(tarinfo, size=7011, chksum=md5_regtype) ++ self._test_member(tarinfo, size=7011, chksum=sha1_regtype) + + def test_find_conttype(self): + tarinfo = self.tar.getmember("ustar/conttype") +- self._test_member(tarinfo, size=7011, chksum=md5_regtype) ++ self._test_member(tarinfo, size=7011, chksum=sha1_regtype) + + def test_find_dirtype(self): + tarinfo = self.tar.getmember("ustar/dirtype") +@@ -851,28 +851,28 @@ def test_find_fifotype(self): + + def test_find_sparse(self): + tarinfo = self.tar.getmember("ustar/sparse") +- self._test_member(tarinfo, size=86016, chksum=md5_sparse) ++ self._test_member(tarinfo, size=86016, chksum=sha1_sparse) + + def test_find_gnusparse(self): + tarinfo = self.tar.getmember("gnu/sparse") +- self._test_member(tarinfo, size=86016, chksum=md5_sparse) ++ self._test_member(tarinfo, size=86016, chksum=sha1_sparse) + + def test_find_gnusparse_00(self): + tarinfo = self.tar.getmember("gnu/sparse-0.0") +- self._test_member(tarinfo, size=86016, chksum=md5_sparse) ++ self._test_member(tarinfo, size=86016, chksum=sha1_sparse) + + def test_find_gnusparse_01(self): + tarinfo = self.tar.getmember("gnu/sparse-0.1") +- self._test_member(tarinfo, size=86016, chksum=md5_sparse) ++ self._test_member(tarinfo, size=86016, chksum=sha1_sparse) + + def test_find_gnusparse_10(self): + tarinfo = self.tar.getmember("gnu/sparse-1.0") +- self._test_member(tarinfo, size=86016, chksum=md5_sparse) ++ self._test_member(tarinfo, size=86016, chksum=sha1_sparse) + + def test_find_umlauts(self): + tarinfo = self.tar.getmember("ustar/umlauts-" + "\xc4\xd6\xdc\xe4\xf6\xfc\xdf") +- self._test_member(tarinfo, size=7011, chksum=md5_regtype) ++ self._test_member(tarinfo, size=7011, chksum=sha1_regtype) + + def test_find_ustar_longname(self): + name = "ustar/" + "12345/" * 39 + "1234567/longname" +@@ -880,7 +880,7 @@ def test_find_ustar_longname(self): + + def test_find_regtype_oldv7(self): + tarinfo = self.tar.getmember("misc/regtype-old-v7") +- self._test_member(tarinfo, size=7011, chksum=md5_regtype) ++ self._test_member(tarinfo, size=7011, chksum=sha1_regtype) + + def test_find_pax_umlauts(self): + self.tar.close() +@@ -888,7 +888,7 @@ def test_find_pax_umlauts(self): + encoding="iso8859-1") + tarinfo = self.tar.getmember("pax/umlauts-" + "\xc4\xd6\xdc\xe4\xf6\xfc\xdf") +- self._test_member(tarinfo, size=7011, chksum=md5_regtype) ++ self._test_member(tarinfo, size=7011, chksum=sha1_regtype) + + + class LongnameTest: +@@ -950,8 +950,8 @@ def _test_sparse_file(self, name): + filename = os.path.join(TEMPDIR, name) + with open(filename, "rb") as fobj: + data = fobj.read() +- self.assertEqual(md5sum(data), md5_sparse, +- "wrong md5sum for %s" % name) ++ self.assertEqual(sha1sum(data), sha1_sparse, ++ "wrong sha1sum for %s" % name) + + if self._fs_supports_holes(): + s = os.stat(filename) +@@ -2431,7 +2431,7 @@ def _test_link_extraction(self, name): + self.tar.extract(name, TEMPDIR) + with open(os.path.join(TEMPDIR, name), "rb") as f: + data = f.read() +- self.assertEqual(md5sum(data), md5_regtype) ++ self.assertEqual(sha1sum(data), sha1_regtype) + + # See issues #1578269, #8879, and #17689 for some history on these skips + @unittest.skipIf(hasattr(os.path, "islink"), + +From 4e35a5240d0bf99b60a76d3d25a82f9b8c901d59 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 16:37:12 +0200 +Subject: [PATCH 23/36] test_tools: Skip md5sum tests in FIPS mode + +--- + Lib/test/test_tools/test_md5sum.py | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/Lib/test/test_tools/test_md5sum.py b/Lib/test/test_tools/test_md5sum.py +index fb565b73778f..7028a4dc2143 100644 +--- a/Lib/test/test_tools/test_md5sum.py ++++ b/Lib/test/test_tools/test_md5sum.py +@@ -4,11 +4,15 @@ + import unittest + from test import support + from test.support.script_helper import assert_python_ok, assert_python_failure ++import hashlib + + from test.test_tools import scriptsdir, skip_if_missing + + skip_if_missing() + ++if hashlib.get_fips_mode(): ++ raise unittest.SkipTest("md5sum won't work at all in FIPS mode") ++ + class MD5SumTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + +From 82d94582bf36170c924b26f33b56c6177a1e30f7 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 16:43:13 +0200 +Subject: [PATCH 24/36] test_urllib2_localnet: Skip tests of md5 auth + +--- + Lib/test/test_urllib2_localnet.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py +index ef0091c49300..2d0d62f610ef 100644 +--- a/Lib/test/test_urllib2_localnet.py ++++ b/Lib/test/test_urllib2_localnet.py +@@ -317,6 +317,7 @@ def test_basic_auth_httperror(self): + self.assertRaises(urllib.error.HTTPError, urllib.request.urlopen, self.server_url) + + ++@unittest.skipIf(hashlib.get_fips_mode(), "md5 digest auth unacceptable in FIPS mode") + @unittest.skipUnless(threading, "Threading required for this test.") + class ProxyAuthTests(unittest.TestCase): + URL = "http://localhost" + +From 78d82fdbe050e6fb27f5d736f3030357046cb60e Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 16:45:21 +0200 +Subject: [PATCH 25/36] test_uuid: Skip uuid3 test + +--- + Lib/test/test_uuid.py | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py +index aa3de74cef0a..a4ec69058889 100644 +--- a/Lib/test/test_uuid.py ++++ b/Lib/test/test_uuid.py +@@ -6,6 +6,7 @@ + import shutil + import subprocess + import uuid ++import hashlib + + def importable(name): + try: +@@ -366,6 +367,7 @@ def test_uuid1(self): + equal(((u.clock_seq_hi_variant & 0x3f) << 8) | + u.clock_seq_low, 0x3fff) + ++ @unittest.skipIf(hashlib.get_fips_mode(), "uuid3 (md5-based) unacceptable in FIPS mode") + def test_uuid3(self): + equal = self.assertEqual + + +From c7bfe61f072475b900bb68c824aa0b1fa696a975 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 17:21:16 +0200 +Subject: [PATCH 26/36] _hashopenssl: Include hash name in the error message + +--- + Modules/_hashopenssl.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c +index 7dfd70822b99..0563473c627e 100644 +--- a/Modules/_hashopenssl.c ++++ b/Modules/_hashopenssl.c +@@ -431,7 +431,7 @@ EVPnew(PyObject *name_obj, + EVPobject *self; + + if (!digest && !initial_ctx) { +- PyErr_SetString(PyExc_ValueError, "unsupported hash type"); ++ PyErr_Format(PyExc_ValueError, "unsupported hash type %U", name_obj); + return NULL; + } + +@@ -622,7 +622,7 @@ pbkdf2_hmac(PyObject *self, PyObject *args, PyObject *kwdict) + + digest = EVP_get_digestbyname(name); + if (digest == NULL) { +- PyErr_SetString(PyExc_ValueError, "unsupported hash type"); ++ PyErr_Format(PyExc_ValueError, "unsupported hash type %s", name); + goto end; + } + + +From 5c732e6288270c247cc443fe2c2fd4fb4d95442b Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 18:23:57 +0200 +Subject: [PATCH 27/36] Make hashlib tests pass in FIPS mode (with some hashlib + changes) + +--- + Lib/hashlib.py | 18 +++++++++-- + Lib/test/test_hashlib.py | 67 ++++++++++++++++++++++++++++------------ + Modules/_hashopenssl.c | 14 +++++++++ + 3 files changed, 78 insertions(+), 21 deletions(-) + +diff --git a/Lib/hashlib.py b/Lib/hashlib.py +index ca1dd2022515..d3344f60b2ea 100644 +--- a/Lib/hashlib.py ++++ b/Lib/hashlib.py +@@ -155,7 +155,18 @@ def __hash_new(name, data=b'', **kwargs): + """new(name, data=b'') - Return a new hashing object using the named algorithm; + optionally initialized with data (which must be a bytes-like object). + """ +- if not get_fips_mode(): ++ if get_fips_mode(): ++ # Use OpenSSL names for Python built-in hashes ++ orig_name = name ++ name = { ++ 'sha3_224': "sha3-224", ++ 'sha3_256': "sha3-256", ++ 'sha3_384': "sha3-384", ++ 'sha3_512': "sha3-512", ++ 'shake_128': "shake128", ++ 'shake_256': "shake256", ++ }.get(name, name) ++ else: + if name in {'blake2b', 'blake2s'}: + # Prefer our blake2 implementation. + # OpenSSL 1.1.0 comes with a limited implementation of blake2b/s. +@@ -163,7 +174,10 @@ def __hash_new(name, data=b'', **kwargs): + # salt, personal, tree hashing or SSE. + return __get_builtin_constructor(name)(data, **kwargs) + try: +- return _hashlib.new(name, data) ++ retval = _hashlib.new(name, data) ++ if get_fips_mode() and name != orig_name: ++ retval._set_name(orig_name) ++ return retval + except ValueError: + # If the _hashlib module (OpenSSL) doesn't support the named + # hash, try using our builtin implementations. +diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py +index 645a3d01b91d..30b6abcbef90 100644 +--- a/Lib/test/test_hashlib.py ++++ b/Lib/test/test_hashlib.py +@@ -29,6 +29,11 @@ + c_hashlib = import_fresh_module('hashlib', fresh=['_hashlib']) + py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) + ++if hashlib.get_fips_mode(): ++ FIPS_DISABLED = {'md5', 'MD5', 'blake2b', 'blake2s'} ++else: ++ FIPS_DISABLED = set() ++ + try: + import _blake2 + except ImportError: +@@ -84,6 +89,11 @@ class HashLibTestCase(unittest.TestCase): + # Issue #14693: fallback modules are always compiled under POSIX + _warn_on_extension_import = os.name == 'posix' or COMPILED_WITH_PYDEBUG + ++ if hashlib.get_fips_mode(): ++ shakes = set() ++ supported_hash_names = tuple( ++ n for n in supported_hash_names if n not in FIPS_DISABLED) ++ + def _conditional_import_module(self, module_name): + """Import a module and return a reference to it or None on failure.""" + try: +@@ -91,8 +101,20 @@ def _conditional_import_module(self, module_name): + except ModuleNotFoundError as error: + if self._warn_on_extension_import: + warnings.warn('Did a C extension fail to compile? %s' % error) ++ except ImportError as error: ++ if not hashlib.get_fips_mode(): ++ raise + return None + ++ def _has_shake_extras(self, hasher): ++ """Return true if the hasher should have "shake" API (digest length)""" ++ if hasher.name not in self.shakes: ++ return False ++ _hashlib = self._conditional_import_module('_hashlib') ++ if _hashlib and isinstance(hasher, _hashlib.HASH): ++ return False ++ return True ++ + def __init__(self, *args, **kwargs): + algorithms = set() + for algorithm in self.supported_hash_names: +@@ -179,15 +201,13 @@ def test_hash_array(self): + a = array.array("b", range(10)) + for cons in self.hash_constructors: + c = cons(a) +- if (c.name in self.shakes +- and not cons.__name__.startswith('openssl_') +- ): ++ if self._has_shake_extras(c): + c.hexdigest(16) + else: + c.hexdigest() + + def test_algorithms_guaranteed(self): +- self.assertEqual(hashlib.algorithms_guaranteed, ++ self.assertEqual(hashlib.algorithms_guaranteed - FIPS_DISABLED, + set(_algo for _algo in self.supported_hash_names + if _algo.islower())) + +@@ -199,6 +219,12 @@ def test_unknown_hash(self): + self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam') + self.assertRaises(TypeError, hashlib.new, 1) + ++ @unittest.skipUnless(hashlib.get_fips_mode(), "Builtin constructor only unavailable in FIPS mode") ++ def test_get_builtin_constructor_fips(self): ++ with self.assertRaises(AttributeError): ++ hashlib.__get_builtin_constructor ++ ++ @unittest.skipIf(hashlib.get_fips_mode(), "No builtin constructors in FIPS mode") + def test_get_builtin_constructor(self): + get_builtin_constructor = getattr(hashlib, + '__get_builtin_constructor') +@@ -228,9 +254,7 @@ def test_get_builtin_constructor(self): + def test_hexdigest(self): + for cons in self.hash_constructors: + h = cons() +- if (h.name in self.shakes +- and not cons.__name__.startswith('openssl_') +- ): ++ if self._has_shake_extras(h): + self.assertIsInstance(h.digest(16), bytes) + self.assertEqual(hexstr(h.digest(16)), h.hexdigest(16)) + else: +@@ -242,9 +266,7 @@ def test_digest_length_overflow(self): + large_sizes = (2**29, 2**32-10, 2**32+10, 2**61, 2**64-10, 2**64+10) + for cons in self.hash_constructors: + h = cons() +- if h.name not in self.shakes: +- continue +- if cons.__name__.startswith('openssl_'): ++ if not self._has_shake_extras(h): + continue + for digest in h.digest, h.hexdigest: + with self.assertRaises((ValueError, OverflowError)): +@@ -275,9 +297,7 @@ def test_large_update(self): + m1.update(bees) + m1.update(cees) + m1.update(dees) +- if (m1.name in self.shakes +- and not cons.__name__.startswith('openssl_') +- ): ++ if self._has_shake_extras(m1): + args = (16,) + else: + args = () +@@ -346,7 +366,8 @@ def check_no_unicode(self, algorithm_name): + self.assertRaises(TypeError, hash_object_constructor, 'spam') + + def test_no_unicode(self): +- self.check_no_unicode('md5') ++ if not hashlib.get_fips_mode(): ++ self.check_no_unicode('md5') + self.check_no_unicode('sha1') + self.check_no_unicode('sha224') + self.check_no_unicode('sha256') +@@ -389,7 +410,8 @@ def check_blocksize_name(self, name, block_size=0, digest_size=0, + self.assertIn(name.split("_")[0], repr(m)) + + def test_blocksize_name(self): +- self.check_blocksize_name('md5', 64, 16) ++ if not hashlib.get_fips_mode(): ++ self.check_blocksize_name('md5', 64, 16) + self.check_blocksize_name('sha1', 64, 20) + self.check_blocksize_name('sha224', 64, 28) + self.check_blocksize_name('sha256', 64, 32) +@@ -429,22 +451,27 @@ def test_blocksize_name_blake2(self): + self.check_blocksize_name('blake2b', 128, 64) + self.check_blocksize_name('blake2s', 64, 32) + ++ @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") + def test_case_md5_0(self): + self.check('md5', b'', 'd41d8cd98f00b204e9800998ecf8427e') + ++ @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") + def test_case_md5_1(self): + self.check('md5', b'abc', '900150983cd24fb0d6963f7d28e17f72') + ++ @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") + def test_case_md5_2(self): + self.check('md5', + b'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', + 'd174ab98d277d9f5a5611c2c9f419d9f') + ++ @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") + @unittest.skipIf(sys.maxsize < _4G + 5, 'test cannot run on 32-bit systems') + @bigmemtest(size=_4G + 5, memuse=1, dry_run=False) + def test_case_md5_huge(self, size): + self.check('md5', b'A'*size, 'c9af2dff37468ce5dfee8f2cfc0a9c6d') + ++ @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") + @unittest.skipIf(sys.maxsize < _4G - 1, 'test cannot run on 32-bit systems') + @bigmemtest(size=_4G - 1, memuse=1, dry_run=False) + def test_case_md5_uintmax(self, size): +@@ -826,14 +853,16 @@ def test_gil(self): + m = cons(b'x' * gil_minsize) + m.update(b'1') + +- m = hashlib.md5() ++ m = hashlib.sha1() + m.update(b'1') + m.update(b'#' * gil_minsize) + m.update(b'1') +- self.assertEqual(m.hexdigest(), 'cb1e1a2cbc80be75e19935d621fb9b21') ++ self.assertEqual(m.hexdigest(), ++ 'c45f7445ca0ea087d7a1758fbea07935f267c46a') + +- m = hashlib.md5(b'x' * gil_minsize) +- self.assertEqual(m.hexdigest(), 'cfb767f225d58469c5de3632a8803958') ++ m = hashlib.sha1(b'x' * gil_minsize) ++ self.assertEqual(m.hexdigest(), ++ '63fda1efde982ba1ffe9d53035bff5c9ce4758fb') + + @unittest.skipUnless(threading, 'Threading required for this test.') + @support.reap_threads +diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c +index 0563473c627e..e330423e2662 100644 +--- a/Modules/_hashopenssl.c ++++ b/Modules/_hashopenssl.c +@@ -256,11 +256,25 @@ EVP_update(EVPobject *self, PyObject *args) + Py_RETURN_NONE; + } + ++static PyObject * ++EVP_set_name(EVPobject *self, PyObject *new_name) ++{ ++ if (!PyUnicode_CheckExact(new_name)) { ++ PyErr_SetString(PyExc_TypeError, "expected string"); ++ return NULL; ++ } ++ Py_DECREF(self->name); ++ Py_INCREF(new_name); ++ self->name = new_name; ++ Py_RETURN_NONE; ++} ++ + static PyMethodDef EVP_methods[] = { + {"update", (PyCFunction)EVP_update, METH_VARARGS, EVP_update__doc__}, + {"digest", (PyCFunction)EVP_digest, METH_NOARGS, EVP_digest__doc__}, + {"hexdigest", (PyCFunction)EVP_hexdigest, METH_NOARGS, EVP_hexdigest__doc__}, + {"copy", (PyCFunction)EVP_copy, METH_NOARGS, EVP_copy__doc__}, ++ {"_set_name", (PyCFunction)EVP_set_name, METH_O, EVP_copy__doc__}, + {NULL, NULL} /* sentinel */ + }; + + +From 9f8dde4113dc93248ed376f17ceef349bcd2f855 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 5 Aug 2019 19:12:38 +0200 +Subject: [PATCH 28/36] Fixups + +- Adjust error message of the original hmac.HMAC class +- Don't duplicate a atest name +--- + Lib/hmac.py | 2 +- + Lib/test/test_hmac.py | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/Lib/hmac.py b/Lib/hmac.py +index 7af94c39ed63..33b5be613d96 100644 +--- a/Lib/hmac.py ++++ b/Lib/hmac.py +@@ -41,7 +41,7 @@ def __init__(self, key, msg = None, digestmod = None): + """ + if _hashlib.get_fips_mode(): + raise ValueError( +- 'hmac.HMAC is not available in FIPS mode. ' ++ 'This class is not available in FIPS mode. ' + + 'Use hmac.new().' + ) + +diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py +index 6f53a6d1fd5b..17ffcf37eb14 100644 +--- a/Lib/test/test_hmac.py ++++ b/Lib/test/test_hmac.py +@@ -389,7 +389,7 @@ def test_realcopy(self): + self.assertTrue(id(h1.outer) != id(h2.outer), + "No real copy of the attribute 'outer'.") + +- def test_realcopy(self): ++ def test_realcopy_by_digest(self): + # Testing if the copy method created a real copy. + h1 = hmac.HMAC(b"key", digestmod="sha1") + h2 = h1.copy() + +From 0056a10cb639a97b6deac312752be9ed0f19f2cf Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 19 Aug 2019 13:59:40 +0200 +Subject: [PATCH 29/36] Make uuid.uuid3 work (using libuuid via ctypes) + +--- + Lib/test/test_uuid.py | 1 - + Lib/uuid.py | 9 +++++++++ + 2 files changed, 9 insertions(+), 1 deletion(-) + +diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py +index a4ec69058889..17e03388ce7b 100644 +--- a/Lib/test/test_uuid.py ++++ b/Lib/test/test_uuid.py +@@ -367,7 +367,6 @@ def test_uuid1(self): + equal(((u.clock_seq_hi_variant & 0x3f) << 8) | + u.clock_seq_low, 0x3fff) + +- @unittest.skipIf(hashlib.get_fips_mode(), "uuid3 (md5-based) unacceptable in FIPS mode") + def test_uuid3(self): + equal = self.assertEqual + +diff --git a/Lib/uuid.py b/Lib/uuid.py +index db8b2ef94ed4..42291ff5e0d6 100644 +--- a/Lib/uuid.py ++++ b/Lib/uuid.py +@@ -476,6 +476,7 @@ def _netbios_getnode(): + # If ctypes is available, use it to find system routines for UUID generation. + # XXX This makes the module non-thread-safe! + _uuid_generate_time = _UuidCreate = None ++_uuid_generate_md5 = None + try: + import ctypes, ctypes.util + import sys +@@ -492,6 +493,8 @@ def _netbios_getnode(): + continue + if hasattr(lib, 'uuid_generate_time'): + _uuid_generate_time = lib.uuid_generate_time ++ # The library that has uuid_generate_time should have md5 too. ++ _uuid_generate_md5 = getattr(lib, 'uuid_generate_md5') + break + del _libnames + +@@ -614,6 +617,12 @@ def uuid1(node=None, clock_seq=None): + + def uuid3(namespace, name): + """Generate a UUID from the MD5 hash of a namespace UUID and a name.""" ++ if _uuid_generate_md5: ++ name_b = bytes(name, "utf-8") ++ _buffer = ctypes.create_string_buffer(16) ++ _uuid_generate_md5(_buffer, namespace.bytes, name_b, len(name_b)) ++ return UUID(bytes=bytes_(_buffer.raw)) ++ + from hashlib import md5 + hash = md5(namespace.bytes + bytes(name, "utf-8")).digest() + return UUID(bytes=hash[:16], version=3) + +From 27805c1110b734c5aa039443f4cf217f0d2da44d Mon Sep 17 00:00:00 2001 +From: Lumir Balhar +Date: Wed, 14 Aug 2019 14:43:07 +0200 +Subject: [PATCH 30/36] distutils upload: only add md5 if available, but + *always* use sha256 + +--- + Lib/distutils/command/upload.py | 3 ++- + Lib/distutils/tests/test_upload.py | 14 ++++++++------ + 2 files changed, 10 insertions(+), 7 deletions(-) + +diff --git a/Lib/distutils/command/upload.py b/Lib/distutils/command/upload.py +index 0edb39efd4cb..170acb3d6b65 100644 +--- a/Lib/distutils/command/upload.py ++++ b/Lib/distutils/command/upload.py +@@ -102,6 +102,7 @@ def upload_file(self, command, pyversion, filename): + 'content': (os.path.basename(filename),content), + 'filetype': command, + 'pyversion': pyversion, ++ 'sha256_digest': hashlib.sha256(content).hexdigest(), + + # additional meta-data + 'metadata_version': '1.0', +@@ -124,7 +125,7 @@ def upload_file(self, command, pyversion, filename): + digest = hashlib.md5(content).hexdigest() + except ValueError as e: + msg = 'calculating md5 checksum failed: %s' % e +- self.announce(msg, log.ERROR) ++ self.announce(msg, log.INFO) + if not hashlib.get_fips_mode(): + # this really shouldn't fail + raise +diff --git a/Lib/distutils/tests/test_upload.py b/Lib/distutils/tests/test_upload.py +index b4b64e97737d..f720a7905dd8 100644 +--- a/Lib/distutils/tests/test_upload.py ++++ b/Lib/distutils/tests/test_upload.py +@@ -132,10 +132,11 @@ def test_upload(self): + # what did we send ? + headers = dict(self.last_open.req.headers) + if hashlib.get_fips_mode(): +- # md5 hash is omitted +- self.assertEqual(headers['Content-length'], '2020') ++ # only sha256 hash is used ++ self.assertEqual(headers['Content-length'], '2197') + else: +- self.assertEqual(headers['Content-length'], '2162') ++ # both sha256 and md5 hashes are used ++ self.assertEqual(headers['Content-length'], '2339') + content_type = headers['Content-type'] + self.assertTrue(content_type.startswith('multipart/form-data')) + self.assertEqual(self.last_open.req.get_method(), 'POST') +@@ -172,10 +173,11 @@ def test_upload_correct_cr(self): + + headers = dict(self.last_open.req.headers) + if hashlib.get_fips_mode(): +- # md5 hash is omitted +- self.assertEqual(headers['Content-length'], '2030') ++ # only sha256 hash is used ++ self.assertEqual(headers['Content-length'], '2207') + else: +- self.assertEqual(headers['Content-length'], '2172') ++ # both sha256 and md5 hashes are used ++ self.assertEqual(headers['Content-length'], '2349') + self.assertIn(b'long description\r', self.last_open.req.data) + + def test_upload_fails(self): + +From 584d432c6a3e987fdd661e994626befaed523a40 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 26 Aug 2019 15:55:48 +0200 +Subject: [PATCH 31/36] Add the usedforsecurity argument back + +--- + Lib/hashlib.py | 4 ++- + Modules/_hashopenssl.c | 82 +++++++++++++++++++++++++++++++----------- + 2 files changed, 65 insertions(+), 21 deletions(-) + +diff --git a/Lib/hashlib.py b/Lib/hashlib.py +index d3344f60b2ea..cd3b035b1d76 100644 +--- a/Lib/hashlib.py ++++ b/Lib/hashlib.py +@@ -174,7 +174,9 @@ def __hash_new(name, data=b'', **kwargs): + # salt, personal, tree hashing or SSE. + return __get_builtin_constructor(name)(data, **kwargs) + try: +- retval = _hashlib.new(name, data) ++ usedforsecurity = kwargs.pop('usedforsecurity', True) ++ retval = _hashlib.new( ++ name, data, usedforsecurity=usedforsecurity) + if get_fips_mode() and name != orig_name: + retval._set_name(orig_name) + return retval +diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c +index e330423e2662..b621c330c32d 100644 +--- a/Modules/_hashopenssl.c ++++ b/Modules/_hashopenssl.c +@@ -318,6 +318,25 @@ EVP_repr(EVPobject *self) + return PyUnicode_FromFormat("<%U HASH object @ %p>", self->name, self); + } + ++ ++static void ++mc_ctx_init(EVP_MD_CTX *ctx, int usedforsecurity) ++{ ++ EVP_MD_CTX_init(ctx); ++ /* ++ If the user has declared that this digest is being used in a ++ non-security role (e.g. indexing into a data structure), set ++ the exception flag for openssl to allow it ++ */ ++ if (!usedforsecurity) { ++#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW ++ EVP_MD_CTX_set_flags(ctx, ++ EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); ++#endif ++ } ++} ++ ++ + #if HASH_OBJ_CONSTRUCTOR + static int + EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds) +@@ -328,9 +347,10 @@ EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds) + Py_buffer view; + char *nameStr; + const EVP_MD *digest; ++ int usedforsecurity=1; + +- if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:HASH", kwlist, +- &name_obj, &data_obj)) { ++ if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O$d:HASH", kwlist, ++ &name_obj, &data_obj, &usedforsecurity)) { + return -1; + } + +@@ -351,7 +371,8 @@ EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds) + PyBuffer_Release(&view); + return -1; + } +- if (!EVP_DigestInit(self->ctx, digest)) { ++ mc_ctx_init(&self->ctx, usedforsecurity); ++ if (!EVP_DigestInit_ex(self->ctx, digest, NULL)) { + _setException(PyExc_ValueError); + if (data_obj) + PyBuffer_Release(&view); +@@ -440,7 +461,7 @@ static PyTypeObject EVPtype = { + static PyObject * + EVPnew(PyObject *name_obj, + const EVP_MD *digest, const EVP_MD_CTX *initial_ctx, +- const unsigned char *cp, Py_ssize_t len) ++ const unsigned char *cp, Py_ssize_t len, int usedforsecurity) + { + EVPobject *self; + +@@ -455,7 +476,8 @@ EVPnew(PyObject *name_obj, + if (initial_ctx) { + EVP_MD_CTX_copy(self->ctx, initial_ctx); + } else { +- if (!EVP_DigestInit(self->ctx, digest)) { ++ mc_ctx_init(self->ctx, usedforsecurity); ++ if (!EVP_DigestInit_ex(self->ctx, digest, NULL)) { + _setException(PyExc_ValueError); + Py_DECREF(self); + return NULL; +@@ -484,26 +506,35 @@ An optional string argument may be provided and will be\n\ + automatically hashed.\n\ + \n\ + The MD5 and SHA1 algorithms are always supported.\n"); ++static PyObject *_EVP_new_impl(PyObject *name_obj, char *name, PyObject *data_obj, int usedforsecurity); + + static PyObject * + EVP_new(PyObject *self, PyObject *args, PyObject *kwdict) + { +- static char *kwlist[] = {"name", "string", NULL}; ++ static char *kwlist[] = {"name", "string", "usedforsecurity", NULL}; + PyObject *name_obj = NULL; + PyObject *data_obj = NULL; +- Py_buffer view = { 0 }; +- PyObject *ret_obj; +- char *name; +- const EVP_MD *digest; ++ int usedforsecurity = 1; + +- if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|O:new", kwlist, +- &name_obj, &data_obj)) { ++ if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|O$p:new", kwlist, ++ &name_obj, &data_obj, &usedforsecurity)) { + return NULL; + } ++ return _EVP_new_impl(name_obj, NULL, data_obj, usedforsecurity); ++} + +- if (!PyArg_Parse(name_obj, "s", &name)) { +- PyErr_SetString(PyExc_TypeError, "name must be a string"); +- return NULL; ++static PyObject * ++_EVP_new_impl(PyObject *name_obj, char *name, PyObject *data_obj, int usedforsecurity) ++{ ++ Py_buffer view = { 0 }; ++ PyObject *ret_obj; ++ const EVP_MD *digest; ++ ++ if (!name) { ++ if (!PyArg_Parse(name_obj, "s", &name)) { ++ PyErr_SetString(PyExc_TypeError, "name must be a string"); ++ return NULL; ++ } + } + + if (data_obj) +@@ -511,7 +542,7 @@ EVP_new(PyObject *self, PyObject *args, PyObject *kwdict) + + digest = EVP_get_digestbyname(name); + +- ret_obj = EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf, view.len); ++ ret_obj = EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf, view.len, usedforsecurity); + + if (data_obj) + PyBuffer_Release(&view); +@@ -906,18 +937,27 @@ generate_hash_name_list(void) + * code that wants to make hashes of a bunch of small strings. + * The first call will lazy-initialize, which reports an exception + * if initialization fails. ++ * ++ * Note that for usedforsecurity=0, we fall back to new(). + */ + #define GEN_CONSTRUCTOR(NAME, SSL_NAME) \ + static PyObject * \ +- EVP_new_ ## NAME (PyObject *self, PyObject *args) \ ++ EVP_new_ ## NAME (PyObject *self, PyObject *args, PyObject *kw) \ + { \ + PyObject *data_obj = NULL; \ + Py_buffer view = { 0 }; \ + PyObject *ret_obj; \ ++ int usedforsecurity = 1; \ ++ char *kwnames[] = {"", "usedforsecurity", NULL}; \ + \ +- if (!PyArg_ParseTuple(args, "|O:" #NAME , &data_obj)) { \ ++ if (!PyArg_ParseTupleAndKeywords(args, kw, "|O$p:" #NAME, kwnames, &data_obj, &usedforsecurity)) { \ + return NULL; \ + } \ ++ if (!usedforsecurity) { \ ++ return _EVP_new_impl( \ ++ CONST_ ## NAME ## _name_obj, SSL_NAME, \ ++ data_obj, usedforsecurity); \ ++ } \ + \ + if (CONST_new_ ## NAME ## _ctx_p == NULL) { \ + EVP_MD_CTX *ctx_p = EVP_MD_CTX_new(); \ +@@ -938,7 +978,8 @@ generate_hash_name_list(void) + NULL, \ + CONST_new_ ## NAME ## _ctx_p, \ + (unsigned char*)view.buf, \ +- view.len); \ ++ view.len, \ ++ usedforsecurity); \ + \ + if (data_obj) \ + PyBuffer_Release(&view); \ +@@ -947,7 +988,8 @@ generate_hash_name_list(void) + + /* a PyMethodDef structure for the constructor */ + #define CONSTRUCTOR_METH_DEF(NAME) \ +- {"openssl_" #NAME, (PyCFunction)EVP_new_ ## NAME, METH_VARARGS, \ ++ {"openssl_" #NAME, (PyCFunction)EVP_new_ ## NAME, \ ++ METH_VARARGS|METH_KEYWORDS, \ + PyDoc_STR("Returns a " #NAME \ + " hash object; optionally initialized with a string") \ + }, + +From 07c8dd0441a76fe841461a1b81d4c7f9c697d643 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 26 Aug 2019 18:59:15 +0200 +Subject: [PATCH 32/36] Add no-op usedforsecurity argument to internal hash + implementations + +--- + Modules/_blake2/blake2b_impl.c | 6 ++++-- + Modules/_blake2/blake2s_impl.c | 6 ++++-- + Modules/_blake2/clinic/blake2b_impl.c.h | 16 +++++++++------- + Modules/_blake2/clinic/blake2s_impl.c.h | 16 +++++++++------- + Modules/_sha3/sha3module.c | 5 +++++ + 5 files changed, 31 insertions(+), 18 deletions(-) + +diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c +index f6bfce823b8f..ae9d244200ab 100644 +--- a/Modules/_blake2/blake2b_impl.c ++++ b/Modules/_blake2/blake2b_impl.c +@@ -87,6 +87,8 @@ _blake2.blake2b.__new__ as py_blake2b_new + inner_size: int = 0 + last_node: bool = False + ++ usedforsecurity: bool = True ++ + Return a new BLAKE2b hash object. + [clinic start generated code]*/ + +@@ -95,8 +97,8 @@ py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, PyObject *leaf_size_obj, + PyObject *node_offset_obj, int node_depth, +- int inner_size, int last_node) +-/*[clinic end generated code: output=7506d8d890e5f13b input=aca35b33c5612b4b]*/ ++ int inner_size, int last_node, int usedforsecurity) ++/*[clinic end generated code: output=02dcc52ee784622b input=c5dfcb847f9065ac]*/ + { + BLAKE2bObject *self = NULL; + Py_buffer buf; +diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c +index 28ae5b651019..bf80d6b7428c 100644 +--- a/Modules/_blake2/blake2s_impl.c ++++ b/Modules/_blake2/blake2s_impl.c +@@ -87,6 +87,8 @@ _blake2.blake2s.__new__ as py_blake2s_new + inner_size: int = 0 + last_node: bool = False + ++ usedforsecurity: bool = True ++ + Return a new BLAKE2s hash object. + [clinic start generated code]*/ + +@@ -95,8 +97,8 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, PyObject *leaf_size_obj, + PyObject *node_offset_obj, int node_depth, +- int inner_size, int last_node) +-/*[clinic end generated code: output=fe060b258a8cbfc6 input=3abfaabe7f5f62cc]*/ ++ int inner_size, int last_node, int usedforsecurity) ++/*[clinic end generated code: output=e32ea5e22d54db91 input=af5344c57efd870d]*/ + { + BLAKE2sObject *self = NULL; + Py_buffer buf; +diff --git a/Modules/_blake2/clinic/blake2b_impl.c.h b/Modules/_blake2/clinic/blake2b_impl.c.h +index 9b2965eb6b31..9688c04dda80 100644 +--- a/Modules/_blake2/clinic/blake2b_impl.c.h ++++ b/Modules/_blake2/clinic/blake2b_impl.c.h +@@ -5,7 +5,8 @@ preserve + PyDoc_STRVAR(py_blake2b_new__doc__, + "blake2b(data=b\'\', /, *, digest_size=_blake2.blake2b.MAX_DIGEST_SIZE,\n" + " key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" +-" node_offset=0, node_depth=0, inner_size=0, last_node=False)\n" ++" node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" ++" usedforsecurity=True)\n" + "--\n" + "\n" + "Return a new BLAKE2b hash object."); +@@ -15,14 +16,14 @@ py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, PyObject *leaf_size_obj, + PyObject *node_offset_obj, int node_depth, +- int inner_size, int last_node); ++ int inner_size, int last_node, int usedforsecurity); + + static PyObject * + py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) + { + PyObject *return_value = NULL; +- static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", NULL}; +- static _PyArg_Parser _parser = {"|O$iy*y*y*iiOOiip:blake2b", _keywords, 0}; ++ static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; ++ static _PyArg_Parser _parser = {"|O$iy*y*y*iiOOiipp:blake2b", _keywords, 0}; + PyObject *data = NULL; + int digest_size = BLAKE2B_OUTBYTES; + Py_buffer key = {NULL, NULL}; +@@ -35,12 +36,13 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) + int node_depth = 0; + int inner_size = 0; + int last_node = 0; ++ int usedforsecurity = 1; + + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, +- &data, &digest_size, &key, &salt, &person, &fanout, &depth, &leaf_size_obj, &node_offset_obj, &node_depth, &inner_size, &last_node)) { ++ &data, &digest_size, &key, &salt, &person, &fanout, &depth, &leaf_size_obj, &node_offset_obj, &node_depth, &inner_size, &last_node, &usedforsecurity)) { + goto exit; + } +- return_value = py_blake2b_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size_obj, node_offset_obj, node_depth, inner_size, last_node); ++ return_value = py_blake2b_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size_obj, node_offset_obj, node_depth, inner_size, last_node, usedforsecurity); + + exit: + /* Cleanup for key */ +@@ -121,4 +123,4 @@ _blake2_blake2b_hexdigest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) + { + return _blake2_blake2b_hexdigest_impl(self); + } +-/*[clinic end generated code: output=0eb559f418fc0a21 input=a9049054013a1b77]*/ ++/*[clinic end generated code: output=d5f214b052c75135 input=a9049054013a1b77]*/ +diff --git a/Modules/_blake2/clinic/blake2s_impl.c.h b/Modules/_blake2/clinic/blake2s_impl.c.h +index 42b87b7099df..5653e93044d1 100644 +--- a/Modules/_blake2/clinic/blake2s_impl.c.h ++++ b/Modules/_blake2/clinic/blake2s_impl.c.h +@@ -5,7 +5,8 @@ preserve + PyDoc_STRVAR(py_blake2s_new__doc__, + "blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" + " key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" +-" node_offset=0, node_depth=0, inner_size=0, last_node=False)\n" ++" node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" ++" usedforsecurity=True)\n" + "--\n" + "\n" + "Return a new BLAKE2s hash object."); +@@ -15,14 +16,14 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, PyObject *leaf_size_obj, + PyObject *node_offset_obj, int node_depth, +- int inner_size, int last_node); ++ int inner_size, int last_node, int usedforsecurity); + + static PyObject * + py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) + { + PyObject *return_value = NULL; +- static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", NULL}; +- static _PyArg_Parser _parser = {"|O$iy*y*y*iiOOiip:blake2s", _keywords, 0}; ++ static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; ++ static _PyArg_Parser _parser = {"|O$iy*y*y*iiOOiipp:blake2s", _keywords, 0}; + PyObject *data = NULL; + int digest_size = BLAKE2S_OUTBYTES; + Py_buffer key = {NULL, NULL}; +@@ -35,12 +36,13 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) + int node_depth = 0; + int inner_size = 0; + int last_node = 0; ++ int usedforsecurity = 1; + + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, +- &data, &digest_size, &key, &salt, &person, &fanout, &depth, &leaf_size_obj, &node_offset_obj, &node_depth, &inner_size, &last_node)) { ++ &data, &digest_size, &key, &salt, &person, &fanout, &depth, &leaf_size_obj, &node_offset_obj, &node_depth, &inner_size, &last_node, &usedforsecurity)) { + goto exit; + } +- return_value = py_blake2s_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size_obj, node_offset_obj, node_depth, inner_size, last_node); ++ return_value = py_blake2s_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size_obj, node_offset_obj, node_depth, inner_size, last_node, usedforsecurity); + + exit: + /* Cleanup for key */ +@@ -121,4 +123,4 @@ _blake2_blake2s_hexdigest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) + { + return _blake2_blake2s_hexdigest_impl(self); + } +-/*[clinic end generated code: output=13d4b08ea9ee2d62 input=a9049054013a1b77]*/ ++/*[clinic end generated code: output=2373a3b3fa542e89 input=a9049054013a1b77]*/ +diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c +index 2783a75644fc..62db9cb5616f 100644 +--- a/Modules/_sha3/sha3module.c ++++ b/Modules/_sha3/sha3module.c +@@ -185,6 +185,11 @@ py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) + HashReturn res; + PyObject *data = NULL; + ++ // Ignore "usedforsecurity" ++ if (PyMapping_DelItemString(kwargs, "usedforsecurity")) { ++ PyErr_Clear(); ++ } ++ + if (!_PyArg_NoKeywords(type->tp_name, kwargs)) { + return NULL; + } + +From 595066f4993f94838f53f06b2729739f30a60eb8 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 26 Aug 2019 19:09:39 +0200 +Subject: [PATCH 33/36] Test the usedforsecurity flag + +--- + Lib/test/test_hashlib.py | 82 ++++++++++++++++++++++++++-------------- + 1 file changed, 54 insertions(+), 28 deletions(-) + +diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py +index 30b6abcbef90..b359c4fb14a9 100644 +--- a/Lib/test/test_hashlib.py ++++ b/Lib/test/test_hashlib.py +@@ -22,6 +22,7 @@ + from test import support + from test.support import _4G, bigmemtest, import_fresh_module + from http.client import HTTPException ++from functools import partial + + # Were we compiled --with-pydebug or with #define Py_DEBUG? + COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount') +@@ -30,8 +31,10 @@ + py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) + + if hashlib.get_fips_mode(): +- FIPS_DISABLED = {'md5', 'MD5', 'blake2b', 'blake2s'} ++ FIPS_UNAVAILABLE = {'blake2b', 'blake2s'} ++ FIPS_DISABLED = {'md5', 'MD5', *FIPS_UNAVAILABLE} + else: ++ FIPS_UNAVAILABLE = set() + FIPS_DISABLED = set() + + try: +@@ -76,6 +79,15 @@ def read_vectors(hash_name): + yield parts + + ++def _is_openssl_constructor(constructor): ++ if getattr(constructor, '__name__', '').startswith('openssl_'): ++ return True ++ if isinstance(constructor, partial): ++ if constructor.func.__name__.startswith('openssl_'): ++ return True ++ return False ++ ++ + class HashLibTestCase(unittest.TestCase): + supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1', + 'sha224', 'SHA224', 'sha256', 'SHA256', +@@ -91,8 +103,6 @@ class HashLibTestCase(unittest.TestCase): + + if hashlib.get_fips_mode(): + shakes = set() +- supported_hash_names = tuple( +- n for n in supported_hash_names if n not in FIPS_DISABLED) + + def _conditional_import_module(self, module_name): + """Import a module and return a reference to it or None on failure.""" +@@ -118,7 +128,8 @@ def _has_shake_extras(self, hasher): + def __init__(self, *args, **kwargs): + algorithms = set() + for algorithm in self.supported_hash_names: +- algorithms.add(algorithm.lower()) ++ if algorithm not in FIPS_UNAVAILABLE: ++ algorithms.add(algorithm.lower()) + + _blake2 = self._conditional_import_module('_blake2') + if _blake2: +@@ -128,15 +139,21 @@ def __init__(self, *args, **kwargs): + for algorithm in algorithms: + self.constructors_to_test[algorithm] = set() + ++ def _add_constructor(algorithm, constructor): ++ constructors.add(partial(constructor, usedforsecurity=False)) ++ if algorithm not in FIPS_DISABLED: ++ constructors.add(constructor) ++ constructors.add(partial(constructor, usedforsecurity=True)) ++ + # For each algorithm, test the direct constructor and the use + # of hashlib.new given the algorithm name. + for algorithm, constructors in self.constructors_to_test.items(): +- constructors.add(getattr(hashlib, algorithm)) ++ _add_constructor(algorithm, getattr(hashlib, algorithm)) + def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs): + if data is None: + return hashlib.new(_alg, **kwargs) + return hashlib.new(_alg, data, **kwargs) +- constructors.add(_test_algorithm_via_hashlib_new) ++ _add_constructor(algorithm, _test_algorithm_via_hashlib_new) + + _hashlib = self._conditional_import_module('_hashlib') + if _hashlib: +@@ -147,7 +164,7 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs): + for algorithm, constructors in self.constructors_to_test.items(): + constructor = getattr(_hashlib, 'openssl_'+algorithm, None) + if constructor: +- constructors.add(constructor) ++ _add_constructor(algorithm, constructor) + + def add_builtin_constructor(name): + constructor = getattr(hashlib, "__get_builtin_constructor")(name) +@@ -207,7 +224,7 @@ def test_hash_array(self): + c.hexdigest() + + def test_algorithms_guaranteed(self): +- self.assertEqual(hashlib.algorithms_guaranteed - FIPS_DISABLED, ++ self.assertEqual(hashlib.algorithms_guaranteed, + set(_algo for _algo in self.supported_hash_names + if _algo.islower())) + +@@ -283,7 +300,9 @@ def test_name_attribute(self): + self.assertIn(h.name, self.supported_hash_names) + else: + self.assertNotIn(h.name, self.supported_hash_names) +- self.assertEqual(h.name, hashlib.new(h.name).name) ++ if h.name not in FIPS_DISABLED: ++ self.assertEqual(h.name, hashlib.new(h.name).name) ++ self.assertEqual(h.name, hashlib.new(h.name, usedforsecurity=False).name) + + def test_large_update(self): + aas = b'a' * 128 +@@ -325,13 +344,15 @@ def check(self, name, data, hexdigest, shake=False, **kwargs): + self.assertGreaterEqual(len(constructors), 2) + for hash_object_constructor in constructors: + if ( +- kwargs +- and hash_object_constructor.__name__.startswith('openssl_') ++ (kwargs.keys() - {'usedforsecurity'}) ++ and _is_openssl_constructor(hash_object_constructor) + ): ++ # Don't check openssl constructors with ++ # any extra keys (except usedforsecurity) + return + m = hash_object_constructor(data, **kwargs) + if shake: +- if hash_object_constructor.__name__.startswith('openssl_'): ++ if _is_openssl_constructor(hash_object_constructor): + if length > m.digest_size: + # OpenSSL doesn't give long digests + return +@@ -348,7 +369,7 @@ def check(self, name, data, hexdigest, shake=False, **kwargs): + % (name, hash_object_constructor, + computed, len(data), hexdigest)) + if shake: +- if hash_object_constructor.__name__.startswith('openssl_'): ++ if _is_openssl_constructor(hash_object_constructor): + computed = m.digest()[:length] + else: + computed = m.digest(length) +@@ -366,8 +387,7 @@ def check_no_unicode(self, algorithm_name): + self.assertRaises(TypeError, hash_object_constructor, 'spam') + + def test_no_unicode(self): +- if not hashlib.get_fips_mode(): +- self.check_no_unicode('md5') ++ self.check_no_unicode('md5') + self.check_no_unicode('sha1') + self.check_no_unicode('sha224') + self.check_no_unicode('sha256') +@@ -394,10 +414,10 @@ def check_blocksize_name(self, name, block_size=0, digest_size=0, + for hash_object_constructor in constructors: + m = hash_object_constructor() + self.assertEqual(m.block_size, block_size) +- if not hash_object_constructor.__name__.startswith('openssl_'): ++ if not _is_openssl_constructor(hash_object_constructor): + self.assertEqual(m.digest_size, digest_size) + if digest_length: +- if not hash_object_constructor.__name__.startswith('openssl_'): ++ if not _is_openssl_constructor(hash_object_constructor): + self.assertEqual(len(m.digest(digest_length)), + digest_length) + self.assertEqual(len(m.hexdigest(digest_length)), +@@ -432,7 +452,7 @@ def check_sha3(self, name, capacity, rate, suffix): + for hash_object_constructor in constructors: + m = hash_object_constructor() + self.assertEqual(capacity + rate, 1600) +- if not hash_object_constructor.__name__.startswith('openssl_'): ++ if not _is_openssl_constructor(hash_object_constructor): + self.assertEqual(m._capacity_bits, capacity) + self.assertEqual(m._rate_bits, rate) + self.assertEqual(m._suffix, suffix) +@@ -451,31 +471,27 @@ def test_blocksize_name_blake2(self): + self.check_blocksize_name('blake2b', 128, 64) + self.check_blocksize_name('blake2s', 64, 32) + +- @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") + def test_case_md5_0(self): +- self.check('md5', b'', 'd41d8cd98f00b204e9800998ecf8427e') ++ self.check('md5', b'', 'd41d8cd98f00b204e9800998ecf8427e', usedforsecurity=False) + +- @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") + def test_case_md5_1(self): +- self.check('md5', b'abc', '900150983cd24fb0d6963f7d28e17f72') ++ self.check('md5', b'abc', '900150983cd24fb0d6963f7d28e17f72', usedforsecurity=False) + +- @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") + def test_case_md5_2(self): + self.check('md5', + b'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', +- 'd174ab98d277d9f5a5611c2c9f419d9f') ++ 'd174ab98d277d9f5a5611c2c9f419d9f', ++ usedforsecurity=False) + +- @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") + @unittest.skipIf(sys.maxsize < _4G + 5, 'test cannot run on 32-bit systems') + @bigmemtest(size=_4G + 5, memuse=1, dry_run=False) + def test_case_md5_huge(self, size): +- self.check('md5', b'A'*size, 'c9af2dff37468ce5dfee8f2cfc0a9c6d') ++ self.check('md5', b'A'*size, 'c9af2dff37468ce5dfee8f2cfc0a9c6d', usedforsecurity=False) + +- @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") + @unittest.skipIf(sys.maxsize < _4G - 1, 'test cannot run on 32-bit systems') + @bigmemtest(size=_4G - 1, memuse=1, dry_run=False) + def test_case_md5_uintmax(self, size): +- self.check('md5', b'A'*size, '28138d306ff1b8281f1a9067e1a1a2b3') ++ self.check('md5', b'A'*size, '28138d306ff1b8281f1a9067e1a1a2b3', usedforsecurity=False) + + # use the three examples from Federal Information Processing Standards + # Publication 180-1, Secure Hash Standard, 1995 April 17 +@@ -901,6 +917,16 @@ def hash_in_chunks(chunk_size): + + self.assertEqual(expected_hash, hasher.hexdigest()) + ++ @unittest.skipUnless(hashlib.get_fips_mode(), 'Needs FIPS mode.') ++ def test_usedforsecurity_repeat(self): ++ """Make sure usedforsecurity flag isn't copied to other contexts""" ++ for i in range(3): ++ for cons in hashlib.md5, partial(hashlib.new, 'md5'): ++ self.assertRaises(ValueError, cons) ++ self.assertRaises(ValueError, partial(cons, usedforsecurity=True)) ++ self.assertEqual(cons(usedforsecurity=False).hexdigest(), ++ 'd41d8cd98f00b204e9800998ecf8427e') ++ + + class KDFTests(unittest.TestCase): + + +From 723bfe45d5a7666a75a12633b21a9f153805eccf Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Mon, 26 Aug 2019 19:39:48 +0200 +Subject: [PATCH 34/36] Don't re-export get_fips_mode from hashlib + +Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1745685 +--- + Lib/distutils/command/upload.py | 3 ++- + Lib/distutils/tests/test_upload.py | 5 +++-- + Lib/hashlib.py | 23 ++++++++++++----------- + Lib/hmac.py | 6 +++--- + Lib/test/test_fips.py | 6 +++--- + Lib/test/test_hashlib.py | 16 +++++++++------- + Lib/test/test_hmac.py | 15 ++++++++------- + Lib/test/test_logging.py | 3 ++- + Lib/test/test_smtplib.py | 7 ++++--- + Lib/test/test_tools/test_md5sum.py | 4 ++-- + Lib/test/test_urllib2_localnet.py | 3 ++- + 11 files changed, 50 insertions(+), 41 deletions(-) + +diff --git a/Lib/distutils/command/upload.py b/Lib/distutils/command/upload.py +index 170acb3d6b65..d0a4aee001ea 100644 +--- a/Lib/distutils/command/upload.py ++++ b/Lib/distutils/command/upload.py +@@ -126,7 +126,8 @@ def upload_file(self, command, pyversion, filename): + except ValueError as e: + msg = 'calculating md5 checksum failed: %s' % e + self.announce(msg, log.INFO) +- if not hashlib.get_fips_mode(): ++ from _hashlib import get_fips_mode ++ if not get_fips_mode(): + # this really shouldn't fail + raise + else: +diff --git a/Lib/distutils/tests/test_upload.py b/Lib/distutils/tests/test_upload.py +index f720a7905dd8..a198b213577b 100644 +--- a/Lib/distutils/tests/test_upload.py ++++ b/Lib/distutils/tests/test_upload.py +@@ -4,6 +4,7 @@ + import unittest.mock as mock + from urllib.request import HTTPError + import hashlib ++from _hashlib import get_fips_mode + + from test.support import run_unittest + +@@ -131,7 +132,7 @@ def test_upload(self): + + # what did we send ? + headers = dict(self.last_open.req.headers) +- if hashlib.get_fips_mode(): ++ if get_fips_mode(): + # only sha256 hash is used + self.assertEqual(headers['Content-length'], '2197') + else: +@@ -172,7 +173,7 @@ def test_upload_correct_cr(self): + cmd.run() + + headers = dict(self.last_open.req.headers) +- if hashlib.get_fips_mode(): ++ if get_fips_mode(): + # only sha256 hash is used + self.assertEqual(headers['Content-length'], '2207') + else: +diff --git a/Lib/hashlib.py b/Lib/hashlib.py +index cd3b035b1d76..3e9a4aa27a79 100644 +--- a/Lib/hashlib.py ++++ b/Lib/hashlib.py +@@ -68,13 +68,13 @@ + 'algorithms_available', 'pbkdf2_hmac') + + try: +- from _hashlib import get_fips_mode ++ from _hashlib import get_fips_mode as _hashlib_get_fips_mode + except ImportError: +- def get_fips_mode(): ++ def _hashlib_get_fips_mode(): + return 0 + + +-if not get_fips_mode(): ++if not _hashlib_get_fips_mode(): + __builtin_constructor_cache = {} + + def __get_builtin_constructor(name): +@@ -121,7 +121,7 @@ def __get_builtin_constructor(name): + + + def __get_openssl_constructor(name): +- if not get_fips_mode(): ++ if not _hashlib.get_fips_mode(): + if name in { + 'blake2b', 'blake2s', 'shake_256', 'shake_128', + #'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', +@@ -132,7 +132,7 @@ def __get_openssl_constructor(name): + f = getattr(_hashlib, 'openssl_' + name) + # Allow the C module to raise ValueError. The function will be + # defined but the hash not actually available thanks to OpenSSL. +- if not get_fips_mode(): ++ if not _hashlib.get_fips_mode(): + # N.B. In "FIPS mode", there is no fallback. + # If this test fails, we want to export the broken hash + # constructor anyway. +@@ -142,7 +142,7 @@ def __get_openssl_constructor(name): + except (AttributeError, ValueError): + return __get_builtin_constructor(name) + +-if not get_fips_mode(): ++if not _hashlib_get_fips_mode(): + def __py_new(name, data=b'', **kwargs): + """new(name, data=b'', **kwargs) - Return a new hashing object using the + named algorithm; optionally initialized with data (which must be +@@ -155,7 +155,7 @@ def __hash_new(name, data=b'', **kwargs): + """new(name, data=b'') - Return a new hashing object using the named algorithm; + optionally initialized with data (which must be a bytes-like object). + """ +- if get_fips_mode(): ++ if _hashlib.get_fips_mode(): + # Use OpenSSL names for Python built-in hashes + orig_name = name + name = { +@@ -177,7 +177,7 @@ def __hash_new(name, data=b'', **kwargs): + usedforsecurity = kwargs.pop('usedforsecurity', True) + retval = _hashlib.new( + name, data, usedforsecurity=usedforsecurity) +- if get_fips_mode() and name != orig_name: ++ if _hashlib.get_fips_mode() and name != orig_name: + retval._set_name(orig_name) + return retval + except ValueError: +@@ -185,7 +185,7 @@ def __hash_new(name, data=b'', **kwargs): + # hash, try using our builtin implementations. + # This allows for SHA224/256 and SHA384/512 support even though + # the OpenSSL library prior to 0.9.8 doesn't provide them. +- if get_fips_mode(): ++ if _hashlib.get_fips_mode(): + raise + return __get_builtin_constructor(name)(data) + +@@ -197,7 +197,7 @@ def __hash_new(name, data=b'', **kwargs): + algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names) + except ImportError: +- if get_fips_mode(): ++ if _hashlib_get_fips_mode(): + raise + new = __py_new + __get_hash = __get_builtin_constructor +@@ -225,5 +225,6 @@ def __hash_new(name, data=b'', **kwargs): + # Cleanup locals() + del __always_supported, __func_name, __get_hash + del __hash_new, __get_openssl_constructor +-if not get_fips_mode(): ++if not _hashlib.get_fips_mode(): + del __py_new ++del _hashlib_get_fips_mode +diff --git a/Lib/hmac.py b/Lib/hmac.py +index 33b5be613d96..ca83d9dc0d35 100644 +--- a/Lib/hmac.py ++++ b/Lib/hmac.py +@@ -39,7 +39,7 @@ def __init__(self, key, msg = None, digestmod = None): + + Note: key and msg must be a bytes or bytearray objects. + """ +- if _hashlib.get_fips_mode(): ++ if _hashlibopenssl.get_fips_mode(): + raise ValueError( + 'This class is not available in FIPS mode. ' + + 'Use hmac.new().' +@@ -97,7 +97,7 @@ def name(self): + def update(self, msg): + """Update this hashing object with the string msg. + """ +- if _hashlib.get_fips_mode(): ++ if _hashlibopenssl.get_fips_mode(): + raise ValueError('hmac.HMAC is not available in FIPS mode') + self.inner.update(msg) + +@@ -167,7 +167,7 @@ def __new__(cls, key, msg = None, digestmod = None): + return result + + +-if _hashlib.get_fips_mode(): ++if _hashlibopenssl.get_fips_mode(): + HMAC = HMAC_openssl + + +diff --git a/Lib/test/test_fips.py b/Lib/test/test_fips.py +index 34812e6098ae..86e61e29c0b4 100644 +--- a/Lib/test/test_fips.py ++++ b/Lib/test/test_fips.py +@@ -6,7 +6,7 @@ + + class HashlibFipsTests(unittest.TestCase): + +- @unittest.skipUnless(hashlib.get_fips_mode(), "Test only when FIPS is enabled") ++ @unittest.skipUnless(_hashlib.get_fips_mode(), "Test only when FIPS is enabled") + def test_fips_imports(self): + """blake2s and blake2b should fail to import in FIPS mode + """ +@@ -30,7 +30,7 @@ def compare_hashes(self, python_hash, openssl_hash): + + self.assertEqual(m, h) + +- @unittest.skipIf(hashlib.get_fips_mode(), "blake2 hashes are not available under FIPS") ++ @unittest.skipIf(_hashlib.get_fips_mode(), "blake2 hashes are not available under FIPS") + def test_blake2_hashes(self): + self.compare_hashes(hashlib.blake2b(b'abc'), _hashlib.openssl_blake2b(b'abc')) + self.compare_hashes(hashlib.blake2s(b'abc'), _hashlib.openssl_blake2s(b'abc')) +@@ -41,7 +41,7 @@ def test_sha3_hashes(self): + self.compare_hashes(hashlib.sha3_384(b'abc'), _hashlib.openssl_sha3_384(b'abc')) + self.compare_hashes(hashlib.sha3_512(b'abc'), _hashlib.openssl_sha3_512(b'abc')) + +- @unittest.skipIf(hashlib.get_fips_mode(), "shake hashes are not available under FIPS") ++ @unittest.skipIf(_hashlib.get_fips_mode(), "shake hashes are not available under FIPS") + def test_shake_hashes(self): + self.compare_hashes(hashlib.shake_128(b'abc'), _hashlib.openssl_shake_128(b'abc')) + self.compare_hashes(hashlib.shake_256(b'abc'), _hashlib.openssl_shake_256(b'abc')) +diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py +index b359c4fb14a9..eadfb7968be8 100644 +--- a/Lib/test/test_hashlib.py ++++ b/Lib/test/test_hashlib.py +@@ -30,7 +30,9 @@ + c_hashlib = import_fresh_module('hashlib', fresh=['_hashlib']) + py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) + +-if hashlib.get_fips_mode(): ++from _hashlib import get_fips_mode as _get_fips_mode ++ ++if _get_fips_mode(): + FIPS_UNAVAILABLE = {'blake2b', 'blake2s'} + FIPS_DISABLED = {'md5', 'MD5', *FIPS_UNAVAILABLE} + else: +@@ -101,7 +103,7 @@ class HashLibTestCase(unittest.TestCase): + # Issue #14693: fallback modules are always compiled under POSIX + _warn_on_extension_import = os.name == 'posix' or COMPILED_WITH_PYDEBUG + +- if hashlib.get_fips_mode(): ++ if _get_fips_mode(): + shakes = set() + + def _conditional_import_module(self, module_name): +@@ -112,7 +114,7 @@ def _conditional_import_module(self, module_name): + if self._warn_on_extension_import: + warnings.warn('Did a C extension fail to compile? %s' % error) + except ImportError as error: +- if not hashlib.get_fips_mode(): ++ if not _get_fips_mode(): + raise + return None + +@@ -236,12 +238,12 @@ def test_unknown_hash(self): + self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam') + self.assertRaises(TypeError, hashlib.new, 1) + +- @unittest.skipUnless(hashlib.get_fips_mode(), "Builtin constructor only unavailable in FIPS mode") ++ @unittest.skipUnless(_get_fips_mode(), "Builtin constructor only unavailable in FIPS mode") + def test_get_builtin_constructor_fips(self): + with self.assertRaises(AttributeError): + hashlib.__get_builtin_constructor + +- @unittest.skipIf(hashlib.get_fips_mode(), "No builtin constructors in FIPS mode") ++ @unittest.skipIf(_get_fips_mode(), "No builtin constructors in FIPS mode") + def test_get_builtin_constructor(self): + get_builtin_constructor = getattr(hashlib, + '__get_builtin_constructor') +@@ -430,7 +432,7 @@ def check_blocksize_name(self, name, block_size=0, digest_size=0, + self.assertIn(name.split("_")[0], repr(m)) + + def test_blocksize_name(self): +- if not hashlib.get_fips_mode(): ++ if not _get_fips_mode(): + self.check_blocksize_name('md5', 64, 16) + self.check_blocksize_name('sha1', 64, 20) + self.check_blocksize_name('sha224', 64, 28) +@@ -917,7 +919,7 @@ def hash_in_chunks(chunk_size): + + self.assertEqual(expected_hash, hasher.hexdigest()) + +- @unittest.skipUnless(hashlib.get_fips_mode(), 'Needs FIPS mode.') ++ @unittest.skipUnless(_get_fips_mode(), 'Needs FIPS mode.') + def test_usedforsecurity_repeat(self): + """Make sure usedforsecurity flag isn't copied to other contexts""" + for i in range(3): +diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py +index 17ffcf37eb14..7fbbe057d736 100644 +--- a/Lib/test/test_hmac.py ++++ b/Lib/test/test_hmac.py +@@ -3,6 +3,7 @@ + import hashlib + import unittest + import warnings ++from _hashlib import get_fips_mode + + + def ignore_warning(func): +@@ -17,7 +18,7 @@ def wrapper(*args, **kwargs): + + class TestVectorsTestCase(unittest.TestCase): + +- @unittest.skipIf(hashlib.get_fips_mode(), 'md5 unacceptable in FIPS mode.') ++ @unittest.skipIf(get_fips_mode(), 'md5 unacceptable in FIPS mode.') + def test_md5_vectors(self): + # Test the HMAC module against test vectors from the RFC. + +@@ -243,7 +244,7 @@ def test_sha384_rfc4231(self): + def test_sha512_rfc4231(self): + self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128) + +- @unittest.skipIf(hashlib.get_fips_mode(), 'MockCrazyHash unacceptable in FIPS mode.') ++ @unittest.skipIf(get_fips_mode(), 'MockCrazyHash unacceptable in FIPS mode.') + def test_legacy_block_size_warnings(self): + class MockCrazyHash(object): + """Ain't no block_size attribute here.""" +@@ -266,7 +267,7 @@ def digest(self): + hmac.HMAC(b'a', b'b', digestmod=MockCrazyHash) + self.fail('Expected warning about small block_size') + +- @unittest.skipIf(hashlib.get_fips_mode(), 'md5 is not default in FIPS mode.') ++ @unittest.skipIf(get_fips_mode(), 'md5 is not default in FIPS mode.') + def test_with_digestmod_warning(self): + with self.assertWarns(PendingDeprecationWarning): + key = b"\x0b" * 16 +@@ -278,7 +279,7 @@ def test_with_digestmod_warning(self): + + class ConstructorTestCase(unittest.TestCase): + +- @unittest.skipIf(hashlib.get_fips_mode(), 'md5 is not default in FIPS mode.') ++ @unittest.skipIf(get_fips_mode(), 'md5 is not default in FIPS mode.') + @ignore_warning + def test_normal(self): + # Standard constructor call. +@@ -343,7 +344,7 @@ def test_withmodule(self): + + class SanityTestCase(unittest.TestCase): + +- @unittest.skipIf(hashlib.get_fips_mode(), "md5 is not default in FIPS mode") ++ @unittest.skipIf(get_fips_mode(), "md5 is not default in FIPS mode") + @ignore_warning + def test_default_is_md5(self): + # Testing if HMAC defaults to MD5 algorithm. +@@ -365,7 +366,7 @@ def test_exercise_all_methods(self): + + class CopyTestCase(unittest.TestCase): + +- @unittest.skipIf(hashlib.get_fips_mode(), "Internal attributes unavailable in FIPS mode") ++ @unittest.skipIf(get_fips_mode(), "Internal attributes unavailable in FIPS mode") + def test_attributes(self): + # Testing if attributes are of same type. + h1 = hmac.HMAC(b"key", digestmod="sha1") +@@ -377,7 +378,7 @@ def test_attributes(self): + self.assertEqual(type(h1.outer), type(h2.outer), + "Types of outer don't match.") + +- @unittest.skipIf(hashlib.get_fips_mode(), "Internal attributes unavailable in FIPS mode") ++ @unittest.skipIf(get_fips_mode(), "Internal attributes unavailable in FIPS mode") + def test_realcopy(self): + # Testing if the copy method created a real copy. + h1 = hmac.HMAC(b"key", digestmod="sha1") +diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py +index 0b229465983c..0028ae05a1b5 100644 +--- a/Lib/test/test_logging.py ++++ b/Lib/test/test_logging.py +@@ -47,6 +47,7 @@ + import warnings + import weakref + import hashlib ++from _hashlib import get_fips_mode + + try: + import threading +@@ -1817,7 +1818,7 @@ def handle_request(self, request): + request.end_headers() + self.handled.set() + +- @unittest.skipIf(hashlib.get_fips_mode(), 'Hangs in FIPS mode.') ++ @unittest.skipIf(get_fips_mode(), 'Hangs in FIPS mode.') + def test_output(self): + # The log message sent to the HTTPHandler is properly received. + logger = logging.getLogger("http") +diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py +index 150e9fbcf053..987b114f99ef 100644 +--- a/Lib/test/test_smtplib.py ++++ b/Lib/test/test_smtplib.py +@@ -16,6 +16,7 @@ + import errno + import textwrap + import hashlib ++from _hashlib import get_fips_mode + + import unittest + from test import support, mock_socket +@@ -969,7 +970,7 @@ def testAUTH_LOGIN(self): + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + +- @unittest.skipIf(hashlib.get_fips_mode(), "md5 auth unacceptable in FIPS mode") ++ @unittest.skipIf(get_fips_mode(), "md5 auth unacceptable in FIPS mode") + def testAUTH_CRAM_MD5(self): + self.serv.add_feature("AUTH CRAM-MD5") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) +@@ -977,7 +978,7 @@ def testAUTH_CRAM_MD5(self): + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + +- @unittest.skipIf(hashlib.get_fips_mode(), "md5 auth unacceptable in FIPS mode") ++ @unittest.skipIf(get_fips_mode(), "md5 auth unacceptable in FIPS mode") + def testAUTH_multiple(self): + # Test that multiple authentication methods are tried. + self.serv.add_feature("AUTH BOGUS PLAIN LOGIN CRAM-MD5") +@@ -986,7 +987,7 @@ def testAUTH_multiple(self): + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + +- @unittest.skipIf(hashlib.get_fips_mode(), "md5 auth unacceptable in FIPS mode") ++ @unittest.skipIf(get_fips_mode(), "md5 auth unacceptable in FIPS mode") + def test_auth_function(self): + supported = {'CRAM-MD5', 'PLAIN', 'LOGIN'} + for mechanism in supported: +diff --git a/Lib/test/test_tools/test_md5sum.py b/Lib/test/test_tools/test_md5sum.py +index 7028a4dc2143..3ba1ca0f146c 100644 +--- a/Lib/test/test_tools/test_md5sum.py ++++ b/Lib/test/test_tools/test_md5sum.py +@@ -4,13 +4,13 @@ + import unittest + from test import support + from test.support.script_helper import assert_python_ok, assert_python_failure +-import hashlib ++import _hashlib + + from test.test_tools import scriptsdir, skip_if_missing + + skip_if_missing() + +-if hashlib.get_fips_mode(): ++if _hashlib.get_fips_mode(): + raise unittest.SkipTest("md5sum won't work at all in FIPS mode") + + class MD5SumTests(unittest.TestCase): +diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py +index 2d0d62f610ef..84831ab2ac0e 100644 +--- a/Lib/test/test_urllib2_localnet.py ++++ b/Lib/test/test_urllib2_localnet.py +@@ -6,6 +6,7 @@ + import http.server + import unittest + import hashlib ++from _hashlib import get_fips_mode + + from test import support + +@@ -317,7 +318,7 @@ def test_basic_auth_httperror(self): + self.assertRaises(urllib.error.HTTPError, urllib.request.urlopen, self.server_url) + + +-@unittest.skipIf(hashlib.get_fips_mode(), "md5 digest auth unacceptable in FIPS mode") ++@unittest.skipIf(get_fips_mode(), "md5 digest auth unacceptable in FIPS mode") + @unittest.skipUnless(threading, "Threading required for this test.") + class ProxyAuthTests(unittest.TestCase): + URL = "http://localhost" + +From 60afcf719206757577b6fc8c6ea119e02130a0f4 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Thu, 29 Aug 2019 10:25:28 +0200 +Subject: [PATCH 35/36] Skip error checking in _hashlib.get_fips_mode + +Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1745499 +--- + Modules/_hashopenssl.c | 30 ++++++++++++++++-------------- + 1 file changed, 16 insertions(+), 14 deletions(-) + +diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c +index b621c330c32d..6bfb12c5618b 100644 +--- a/Modules/_hashopenssl.c ++++ b/Modules/_hashopenssl.c +@@ -1021,20 +1021,22 @@ static PyObject * + _hashlib_get_fips_mode_impl(PyObject *module) + /*[clinic end generated code: output=ad8a7793310d3f98 input=f42a2135df2a5e11]*/ + { +- int result = FIPS_mode(); +- if (result == 0) { +- // "If the library was built without support of the FIPS Object Module, +- // then the function will return 0 with an error code of +- // CRYPTO_R_FIPS_MODE_NOT_SUPPORTED (0x0f06d065)." +- // But 0 is also a valid result value. +- +- unsigned long errcode = ERR_peek_last_error(); +- if (errcode) { +- _setException(PyExc_ValueError); +- return NULL; +- } +- } +- return PyLong_FromLong(result); ++ // XXX: This function skips error checking. ++ // This is only appropriate for RHEL. ++ ++ // From the OpenSSL docs: ++ // "If the library was built without support of the FIPS Object Module, ++ // then the function will return 0 with an error code of ++ // CRYPTO_R_FIPS_MODE_NOT_SUPPORTED (0x0f06d065)." ++ // In RHEL: ++ // * we do build with FIPS, so the function always succeeds ++ // * even if it didn't, people seem used to errors being left on the ++ // OpenSSL error stack. ++ ++ // For more info, see: ++ // https://bugzilla.redhat.com/show_bug.cgi?id=1745499 ++ ++ return PyLong_FromLong(FIPS_mode()); + } + + +From 68b3c51d9c7d6bb635174e8d02db5c73ab521ea0 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Thu, 10 Oct 2019 13:04:50 +0200 +Subject: [PATCH] Skip error checking in _Py_hashlib_fips_error + +https://bugzilla.redhat.com/show_bug.cgi?id=1760106 +--- + Include/_hashopenssl.h | 12 +++--------- + 1 file changed, 3 insertions(+), 9 deletions(-) + +diff --git a/Include/_hashopenssl.h b/Include/_hashopenssl.h +index 47ed003042209..d4cbdef984d83 100644 +--- a/Include/_hashopenssl.h ++++ b/Include/_hashopenssl.h +@@ -42,16 +42,10 @@ static int + _Py_hashlib_fips_error(PyObject *exc, char *name) { + int result = FIPS_mode(); + if (result == 0) { +- // "If the library was built without support of the FIPS Object Module, +- // then the function will return 0 with an error code of +- // CRYPTO_R_FIPS_MODE_NOT_SUPPORTED (0x0f06d065)." +- // But 0 is also a valid result value. ++ // XXX: This function skips error checking. ++ // This is only appropriate for RHEL. ++ // See _hashlib.get_fips_mode for details. + +- unsigned long errcode = ERR_peek_last_error(); +- if (errcode) { +- _setException(exc); +- return 1; +- } + return 0; + } + PyErr_Format(exc, "%s is not available in FIPS mode", name); + diff --git a/SOURCES/9900-centos-fix-ssl-connection-resets.patch b/SOURCES/9900-centos-fix-ssl-connection-resets.patch deleted file mode 100644 index 952ca69..0000000 --- a/SOURCES/9900-centos-fix-ssl-connection-resets.patch +++ /dev/null @@ -1,21 +0,0 @@ -diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py -index 0aeabc10f2..72b7a94a21 100644 ---- a/Lib/test/test_ssl.py -+++ b/Lib/test/test_ssl.py -@@ -2021,6 +2021,16 @@ if _have_threads: - sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" - % (msg, ctype, msg.lower(), ctype)) - self.write(msg.lower()) -+ except ConnectionResetError: -+ # XXX: OpenSSL 1.1.1 sometimes raises ConnectionResetError -+ # when connection is not shut down gracefully. -+ if self.server.chatty and support.verbose: -+ sys.stdout.write( -+ " Connection reset by peer: {}\n".format( -+ self.addr) -+ ) -+ self.close() -+ self.running = False - except OSError: - if self.server.chatty: - handle_error("Test server failure:\n") diff --git a/SPECS/python3.spec b/SPECS/python3.spec index 5338e24..f61528d 100644 --- a/SPECS/python3.spec +++ b/SPECS/python3.spec @@ -14,7 +14,7 @@ URL: https://www.python.org/ # WARNING When rebasing to a new Python version, # remember to update the python3-docs package as well Version: %{pybasever}.8 -Release: 4%{?dist} +Release: 15.1%{?dist} License: Python @@ -25,6 +25,10 @@ License: Python # Note that the bcond macros are named for the CLI option they create. # "%%bcond_without" means "ENABLE by default and create a --without option" +# Whether to use RPM build wheels from the python-{pip,setuptools}-wheel package +# Uses upstream bundled prebuilt wheels otherwise +%bcond_without rpmwheels + # Expensive optimizations (mainly, profile-guided optimizations) %ifarch %{ix86} x86_64 %bcond_without optimizations @@ -37,9 +41,6 @@ License: Python # Run the test suite in %%check %bcond_without tests -# Ability to reuse RPM-installed pip using rewheel -%bcond_without rewheel - # Extra build for debugging the interpreter or C-API extensions # (the -debug subpackages) %bcond_without debug_build @@ -188,6 +189,7 @@ BuildRequires: ncurses-devel BuildRequires: openssl-devel BuildRequires: pkgconfig BuildRequires: readline-devel +BuildRequires: redhat-rpm-config >= 118 BuildRequires: sqlite-devel BuildRequires: gdb @@ -208,9 +210,9 @@ BuildRequires: /usr/bin/dtrace # workaround http://bugs.python.org/issue19804 (test_uuid requires ifconfig) BuildRequires: /usr/sbin/ifconfig -%if %{with rewheel} -BuildRequires: python3-setuptools -BuildRequires: python3-pip +%if %{with rpmwheels} +BuildRequires: python3-setuptools-wheel +BuildRequires: python3-pip-wheel # Verify that the BuildRoot includes python36. # Not actually needed for build. @@ -320,10 +322,9 @@ Patch170: 00170-gc-assertions.patch Patch178: 00178-dont-duplicate-flags-in-sysconfig.patch # 00189 # -# Add the rewheel module, allowing to recreate wheels from already installed -# ones -# https://github.com/bkabrda/rewheel -Patch189: 00189-add-rewheel-module.patch +# Instead of bundled wheels, use our RPM packaged wheels from +# /usr/share/python3-wheels +Patch189: 00189-use-rpm-wheels.patch # 00205 # # LIBPL variable in makefile takes LIBPL from configure.ac @@ -356,14 +357,77 @@ Patch274: 00274-fix-arch-names.patch # See also: https://bugzilla.redhat.com/show_bug.cgi?id=1489816 Patch294: 00294-define-TLS-cipher-suite-on-build-time.patch +# 00317 # +# Security fix for CVE-2019-5010: Fix segfault in ssl's cert parser +# https://bugzilla.redhat.com/show_bug.cgi?id=1666789 +# Fixed upstream: https://bugs.python.org/issue35746 +Patch317: 00317-CVE-2019-5010.patch + +# 00318 # +# test_ssl fixes for TLS 1.3 and OpenSSL 1.1.1 +# https://bugzilla.redhat.com/show_bug.cgi?id=1639531 +# https://bugs.python.org/issue32947#msg333990 +# https://github.com/python/cpython/pull/11612 +Patch318: 00318-test-ssl-fix-for-tls-13.patch + +# 00319 # +# Fix test_tarfile on ppc64 +# https://bugzilla.redhat.com/show_bug.cgi?id=1639490 +# https://bugs.python.org/issue35772 +Patch319: 00319-test_tarfile_ppc64.patch + # 00320 # # Security fix for CVE-2019-9636 and CVE-2019-10160: Information Disclosure due to urlsplit improper NFKC normalization # Fixed upstream: https://bugs.python.org/issue36216 and https://bugs.python.org/issue36742 -# Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1693973 -# and https://bugzilla.redhat.com/show_bug.cgi?id=1714756 +# Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1689318 Patch320: 00320-CVE-2019-9636-and-CVE-2019-10160.patch -Patch9900: 9900-centos-fix-ssl-connection-resets.patch +# 00324 # +# Disallow control chars in http URLs +# Security fix for CVE-2019-9740 and CVE-2019-9947 +# Fixed upstream: https://bugs.python.org/issue30458 +# Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1704365 +# and https://bugzilla.redhat.com/show_bug.cgi?id=1703531 +Patch324: 00324-disallow-control-chars-in-http-urls.patch + +# 00325 # +# Unnecessary URL scheme exists to allow local_file:// reading file in urllib +# Security fix for CVE-2019-9948 +# Fixed upstream: https://bugs.python.org/issue35907 +# Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1714643 +Patch325: 00325-CVE-2019-9948.patch + +# 00326 # +# Don't set the post-handshake authentication verify flag on client side +# on TLS 1.3, as it also implicitly enables cert chain validation and an +# SSL/TLS connection will fail when verify mode is set to CERT_NONE. +# Fixed upstream: https://bugs.python.org/issue37428 +# Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1725721 +Patch326: 00326-do-not-set-PHA-verify-flag-on-client-side.patch + +# 00327 # +# Enable TLS 1.3 post-handshake authentication in http.client for default +# context or if a cert_file is passed to HTTPSConnection +# Fixed upstream: https://bugs.python.org/issue37440 +# Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1671353 +Patch327: 00327-enable-tls-1.3-PHA-in-http.client.patch + +# 00329 # +# Support OpenSSL FIPS mode +# - Fallback implementations md5, sha1, sha256, sha512 are removed in favor of OpenSSL wrappers +# - In FIPS mode, OpenSSL wrappers are always used in hashlib +# - add a new "usedforsecurity" keyword argument to the various digest +# algorithms in hashlib so that you can whitelist a callsite with +# "usedforsecurity=False" +# - OpenSSL wrappers for the hashes blake2{b512,s256}, +# sha3_{224,256,384,512}, shake_{128,256} are now exported from _hashlib +# - In FIPS mode, the blake2, sha3 and shake hashes use OpenSSL wrappers +# and do not offer extended functionality (keys, tree hashing, custom digest size) +# - In FIPS mode, hmac.HMAC can only be instantiated with an OpenSSL wrapper +# or an string with OpenSSL hash name as the "digestmod" argument. +# The argument must be specified (instead of defaulting to ‘md5’). +# Resolves: rhbz#1731424 +Patch329: 00329-fips.patch # (New patches go here ^^^) # @@ -390,9 +454,21 @@ Provides: python(abi) = %{pybasever} Requires: %{name}-libs%{?_isa} = %{version}-%{release} -%if %{with rewheel} +%if %{with rpmwheels} + +# RHEL8 was forked from F28 and thus required python3-setuptools/pip here +# for the rewheel module to work. We've since backported the use of RPM +# prepared wheels from F29+ into RHEL8, and thus this dependency isn't +# strictly needed. +# However, it is possible, that some packages in BaseOS actually depend on +# setuptools/pip without declaring the dependency in their spec file. Thus +# we're keeping these dependencies here to avoid the possibility of breaking +# them. Requires: platform-python-setuptools Requires: platform-python-pip + +Requires: python3-setuptools-wheel +Requires: python3-pip-wheel %endif # Runtime require alternatives @@ -441,6 +517,14 @@ Requires: chkconfig Requires: gdbm%{?_isa} >= 1:1.13 %endif +%if %{with rpmwheels} +Requires: python3-setuptools-wheel +Requires: python3-pip-wheel +%else +Provides: bundled(python3-pip) = 18.1 +Provides: bundled(python3-setuptools) = 40.6.2 +%endif + # There are files in the standard library that have python shebang. # We've filtered the automatic requirement out so libs are installable without # the main package. This however makes it pulled in by default. @@ -496,11 +580,14 @@ Requires: python-rpm-macros Requires: python3-rpm-macros Requires: python3-rpm-generators -# https://bugzilla.redhat.com/show_bug.cgi?id=1217376 -# https://bugzilla.redhat.com/show_bug.cgi?id=1496757 -# https://bugzilla.redhat.com/show_bug.cgi?id=1218294 -# TODO change to a specific subpackage once available (#1218294) -Requires: redhat-rpm-config +# This is not "API" (packages that need setuptools should still BuildRequire it) +# However some packages apparently can build both with and without setuptools +# producing egg-info as file or directory (depending on setuptools presence). +# Directory-to-file updates are problematic in RPM, so we ensure setuptools is +# installed when -devel is required. +# See https://bugzilla.redhat.com/show_bug.cgi?id=1623914 +# See https://fedoraproject.org/wiki/Packaging:Directory_Replacement +Requires: platform-python-setuptools Provides: %{name}-2to3 = %{version}-%{release} Provides: 2to3 = %{version}-%{release} @@ -614,11 +701,6 @@ so extensions for both versions can co-exist in the same directory. rm -r Modules/expat rm -r Modules/zlib -%if %{with rewheel} -%global pip_version %(pip%{pybasever} --version | cut -d' ' -f2) -sed -r -i s/'_PIP_VERSION = "[0-9.]+"'/'_PIP_VERSION = "%{pip_version}"'/ Lib/ensurepip/__init__.py -%endif - # # Apply patches: # @@ -635,8 +717,9 @@ sed -r -i s/'_PIP_VERSION = "[0-9.]+"'/'_PIP_VERSION = "%{pip_version}"'/ Lib/en %patch170 -p1 %patch178 -p1 -%if %{with rewheel} +%if %{with rpmwheels} %patch189 -p1 +rm Lib/ensurepip/_bundled/*.whl %endif %patch205 -p1 @@ -644,9 +727,16 @@ sed -r -i s/'_PIP_VERSION = "[0-9.]+"'/'_PIP_VERSION = "%{pip_version}"'/ Lib/en %patch262 -p1 %patch274 -p1 %patch294 -p1 +%patch317 -p1 +%patch318 -p1 +%patch319 -p1 %patch320 -p1 +%patch324 -p1 +%patch325 -p1 +%patch326 -p1 +%patch327 -p1 +%patch329 -p1 -%patch9900 -p1 # Remove files that should be generated by the build # (This is after patching, so that we can use patches directly from upstream) @@ -681,13 +771,21 @@ topdir=$(pwd) %endif # Set common compiler/linker flags -export CFLAGS="$RPM_OPT_FLAGS -D_GNU_SOURCE -fPIC -fwrapv" -export CXXFLAGS="$RPM_OPT_FLAGS -D_GNU_SOURCE -fPIC -fwrapv" +# We utilize the %%extension_...flags macros here so users building C/C++ +# extensions with our python won't get all the compiler/linker flags used +# in RHEL RPMs. +# Standard library built here will still use the %%build_...flags, +# RHEL packages utilizing %%py3_build will use them as well +# https://fedoraproject.org/wiki/Changes/Python_Extension_Flags +export CFLAGS="%{extension_cflags} -D_GNU_SOURCE -fPIC -fwrapv" +export CFLAGS_NODIST="%{build_cflags} -D_GNU_SOURCE -fPIC -fwrapv" +export CXXFLAGS="%{extension_cxxflags} -D_GNU_SOURCE -fPIC -fwrapv" export CPPFLAGS="$(pkg-config --cflags-only-I libffi)" -export OPT="$RPM_OPT_FLAGS -D_GNU_SOURCE -fPIC -fwrapv" +export OPT="%{extension_cflags} -D_GNU_SOURCE -fPIC -fwrapv" export LINKCC="gcc" export CFLAGS="$CFLAGS $(pkg-config --cflags openssl)" -export LDFLAGS="$RPM_LD_FLAGS -g $(pkg-config --libs-only-L openssl)" +export LDFLAGS="%{extension_ldflags} -g $(pkg-config --libs-only-L openssl)" +export LDFLAGS_NODIST="%{build_ldflags} -g $(pkg-config --libs-only-L openssl)" # We can build several different configurations of Python: regular and debug. # Define a common function that does one build: @@ -997,7 +1095,7 @@ rm %{buildroot}%{_mandir}/man1/python3.1 # Install the unversioned-python script and its man page install -m 755 %{SOURCE12} %{buildroot}%{_libexecdir}/no-python -install -m 644 %{SOURCE13} %{buildroot}%{_mandir}/man1/unversioned-python.1.gz +install -m 644 %{SOURCE13} %{buildroot}%{_mandir}/man1/unversioned-python.1 # Touch the files that are controlled by `alternatives` so we can declare them # as ghosts in the files section touch %{buildroot}%{_bindir}/unversioned-python @@ -1066,16 +1164,14 @@ CheckPython() { -wW --slowest --findleaks \ -x test_distutils \ -x test_bdist_rpm \ - %ifarch %{arm} - -x test_gdb \ - %endif %ifarch %{mips64} -x test_ctypes \ %endif + %ifarch s390x + -x test_gdb \ + %endif %ifarch ppc64le - -x test_buffer \ - -x test_tarfile \ - -x test_ssl \ + -x test_gdb \ %endif echo FINISHED: CHECKING OF PYTHON FOR CONFIGURATION: $ConfName @@ -1143,7 +1239,9 @@ fi %exclude %{_bindir}/pyvenv %{_bindir}/pyvenv-%{pybasever} -%{_mandir}/*/* + +%{_mandir}/man1/python3.6.1* +%{_mandir}/man1/unversioned-python.1* %files libs %license LICENSE @@ -1178,13 +1276,11 @@ fi %dir %{pylibdir}/ensurepip/__pycache__/ %{pylibdir}/ensurepip/*.py %{pylibdir}/ensurepip/__pycache__/*%{bytecode_suffixes} +%if %{with rpmwheels} %exclude %{pylibdir}/ensurepip/_bundled - -%if %{with rewheel} -%dir %{pylibdir}/ensurepip/rewheel/ -%dir %{pylibdir}/ensurepip/rewheel/__pycache__/ -%{pylibdir}/ensurepip/rewheel/*.py -%{pylibdir}/ensurepip/rewheel/__pycache__/*%{bytecode_suffixes} +%else +%dir %{pylibdir}/ensurepip/_bundled +%{pylibdir}/ensurepip/_bundled/*.whl %endif # The majority of the test module lives in the test subpackage @@ -1212,11 +1308,8 @@ fi %{pylibdir}/pydoc_data %{dynload_dir}/_blake2.%{SOABI_optimized}.so -%{dynload_dir}/_md5.%{SOABI_optimized}.so -%{dynload_dir}/_sha1.%{SOABI_optimized}.so -%{dynload_dir}/_sha256.%{SOABI_optimized}.so %{dynload_dir}/_sha3.%{SOABI_optimized}.so -%{dynload_dir}/_sha512.%{SOABI_optimized}.so +%{dynload_dir}/_hmacopenssl.%{SOABI_optimized}.so %{dynload_dir}/_asyncio.%{SOABI_optimized}.so %{dynload_dir}/_bisect.%{SOABI_optimized}.so @@ -1451,11 +1544,8 @@ fi # ...with debug builds of the built-in "extension" modules: %{dynload_dir}/_blake2.%{SOABI_debug}.so -%{dynload_dir}/_md5.%{SOABI_debug}.so -%{dynload_dir}/_sha1.%{SOABI_debug}.so -%{dynload_dir}/_sha256.%{SOABI_debug}.so %{dynload_dir}/_sha3.%{SOABI_debug}.so -%{dynload_dir}/_sha512.%{SOABI_debug}.so +%{dynload_dir}/_hmacopenssl.%{SOABI_debug}.so %{dynload_dir}/_asyncio.%{SOABI_debug}.so %{dynload_dir}/_bisect.%{SOABI_debug}.so @@ -1570,15 +1660,66 @@ fi # ====================================================== %changelog -* Fri Jun 07 2019 Charalampos Stratakis - 3.6.8-4 +* Fri Oct 11 2019 Tomas Orsava - 3.6.8-15.1 +- Patch 329 (FIPS) modified: Added workaround for mod_ssl: + Skip error checking in _Py_hashlib_fips_error +Resolves: rhbz#1760106 + +* Thu Aug 29 2019 Tomas Orsava - 3.6.8-15 +- Patch 329 that adds support for OpenSSL FIPS mode has been improved and + bugfixed +Resolves: rhbz#1744670 rhbz#1745499 rhbz#1745685 + +* Tue Aug 06 2019 Tomas Orsava - 3.6.8-14 +- Adding a new patch 329 that adds support for OpenSSL FIPS mode +- Explicitly listing man pages in files section to fix an RPM warning +Resolves: rhbz#1731424 + +* Tue Jul 02 2019 Charalampos Stratakis - 3.6.8-13 +- Do not set PHA verify flag on client side (rhbz#1725721) +- Enable TLS 1.3 post-handshake authentication in http.client (rhbz#1671353) + +* Fri Jun 21 2019 Miro Hrončok - 3.6.8-12 +- Use RPM built wheels of pip and setuptools in ensurepip instead of our rewheel patch +- Require platform-python-setuptools from platform-python-devel to prevent packaging errors +Resolves: rhbz#1701286 + +* Fri Jun 07 2019 Charalampos Stratakis - 3.6.8-11 - Fix for CVE-2019-10160 -Resolves: rhbz#1714756 +Resolves: rhbz#1689318 + +* Wed May 29 2019 Charalampos Stratakis - 3.6.8-10 +- Security fix for CVE-2019-9948 +Resolves: rhbz#1714643 + +* Tue May 21 2019 Miro Hrončok - 3.6.8-9 +- Reduced default build flags used to build extension modules + https://fedoraproject.org/wiki/Changes/Python_Extension_Flags +Resolves: rhbz#1634784 + +* Mon May 13 2019 Tomas Orsava - 3.6.8-8 +- gzip the unversioned-python man page +Resolves: rhbz#1665514 + +* Wed May 08 2019 Charalampos Stratakis - 3.6.8-7 +- Disallow control chars in http URLs +- Fixes CVE-2019-9740 and CVE-2019-9947 +Resolves: rhbz#1704365 and rhbz#1703531 + +* Fri May 03 2019 Charalampos Stratakis - 3.6.8-6 +- Updated fix for CVE-2019-9636 (rhbz#1689318) + +* Wed Apr 3 2019 Miro Hrončok - 3.6.8-5 +- Security fix for CVE-2019-9636 (rhbz#1689318) + +* Wed Mar 20 2019 Victor Stinner - 3.6.8-4 +- Security fix for CVE-2019-5010 (rhbz#1666789) -* Fri May 03 2019 Charalampos Stratakis - 3.6.8-3 -- Updated fix for CVE-2019-9636 (rhbz#1714756) +* Wed Mar 13 2019 Victor Stinner - 3.6.8-3 +- Fix test_tarfile on ppc64 (rhbz#1639490) -* Wed Apr 3 2019 Miro Hrončok - 3.6.8-2 -- Security fix for CVE-2019-9636 (rhbz#1693973) +* Fri Jan 18 2019 Victor Stinner - 3.6.8-2 +- test_ssl fixes for TLS 1.3 and OpenSSL 1.1.1 (rhbz#1639531) * Wed Jan 09 2019 Charalampos Stratakis - 3.6.8-1 - Update to 3.6.8