From 227248ee53096848a11b28e8296291ba71542c93 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 25 Sep 2019 09:12:59 -0700 Subject: [PATCH 01/47] bpo-38271: encrypt private key test files with AES256 (GH-16385) The private keys for test_ssl were encrypted with 3DES in traditional PKCSGH-5 format. 3DES and the digest algorithm of PKCSGH-5 are blocked by some strict crypto policies. Use PKCSGH-8 format with AES256 encryption instead. Signed-off-by: Christian Heimes https://bugs.python.org/issue38271 Automerge-Triggered-By: @tiran (cherry picked from commit bfd0c963d88f3df69489ee250655e2b8f3d235bd) Co-authored-by: Christian Heimes --- Lib/test/keycert.passwd.pem | 85 ++++++++++--------- Lib/test/make_ssl_certs.py | 4 +- Lib/test/ssl_key.passwd.pem | 84 +++++++++--------- .../2019-09-25-13-11-29.bpo-38271.iHXNIg.rst | 4 + 4 files changed, 91 insertions(+), 86 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst diff --git a/Lib/test/keycert.passwd.pem b/Lib/test/keycert.passwd.pem index cbb3c3b..c330c36 100644 --- a/Lib/test/keycert.passwd.pem +++ b/Lib/test/keycert.passwd.pem @@ -1,45 +1,45 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,D134E931C96D9DEC - -nuGFEej7vIjkYWSMz5OJeVTNntDRQi6ZM4DBm3g8T7i/0odr3WFqGMMKZcIhLYQf -rgRq7RSKtrJ1y5taVucMV+EuCjyfzDo0TsYt+ZrXv/D08eZhjRmkhoHnGVF0TqQm -nQEXM/ERT4J2RM78dnG+homMkI76qOqxgGbRqQqJo6AiVRcAZ45y8s96bru2TAB8 -+pWjO/v0Je7AFVdwSU52N8OOY6uoSAygW+0UY1WVxbVGJF2XfRsNpPX+YQHYl6e+ -3xM5XBVCgr6kmdAyub5qUJ38X3TpdVGoR0i+CVS9GTr2pSRib1zURAeeHnlqiUZM -4m0Gn9s72nJevU1wxED8pwOhR8fnHEmMKGD2HPhKoOCbzDhwwBZO27TNa1uWeM3f -M5oixKDi2PqMn3y2cDx1NjJtP661688EcJ5a2Ih9BgO9xpnhSyzBWEKcAn0tJB0H -/56M0FW6cdOOIzMveGGL7sHW5E+iOdI1n5e7C6KJUzew78Y9qJnhS53EdI6qTz9R -wsIsj1i070Fk6RbPo6zpLlF6w7Zj8GlZaZA7OZZv9wo5VEV/0ST8gmiiBOBc4C6Y -u9hyLIIu4dFEBKyQHRvBnQSLNpKx6or1OGFDVBay2In9Yh2BHh1+vOj/OIz/wq48 -EHOIV27fRJxLu4jeK5LIGDhuPnMJ8AJYQ0bQOUP6fd7p+TxWkAQZPB/Dx/cs3hxr -nFEdzx+eO+IAsObx/b1EGZyEJyETBslu4GwYX7/KK3HsJhDJ1bdZ//28jOCaoir6 -ZOMT72GRwmVoQTJ0XpccfjHfKJDRLT7C1xvzo4Eibth0hpTZkA75IUYUp6qK/PuJ -kH/qdiC7QIkRKtsrawW4vEDna3YtxIYhQqz9+KwO6u/0gzooZtv1RU4U3ifMDB5u -5P5GAzACRqlY8QYBkM869lvWqzQPHvybC4ak9Yx6/heMO9ddjdIW9BaK8BLxvN/6 -UCD936Y4fWltt09jHZIoxWFykouBwmd7bXooNYXmDRNmjTdVhKJuOEOQw8hDzx7e -pWFJ9Z/V4Qm1tvXbCD7QFqMCDoY3qFvVG8DBqXpmxe1yPfz21FWrT7IuqDXAD3ns -vxfN/2a+Cy04U9FBNVCvWqWIs5AgNpdCMJC2FlXKTy+H3/7rIjNyFyvbX0vxIXtK -liOVNXiyVM++KZXqktqMUDlsJENmIHV9B046luqbgW018fHkyEYlL3iRZGbYegwr -XO9VVIKVPw1BEvJ8VNdGFGuZGepd8qX2ezfYADrNR+4t85HDm8inbjTobSjWuljs -ftUNkOeCHqAvWCFQTLCfdykvV08EJfVY79y7yFPtfRV2gxYokXFifjo3su9sVQr1 -UiIS5ZAsIC1hBXWeXoBN7QVTkFi7Yto6E1q2k10LiT3obpUUUQ/oclhrJOCJVjrS -oRcj2QBy8OT4T9slJr5maTWdgd7Lt6+I6cGQXPaDvjGOJl0eBYM14vhx4rRQWytJ -k07hhHFO4+9CGCuHS8AAy2gR6acYFWt2ZiiNZ0z/iPIHNK4YEyy9aLf6uZH/KQjE -jmHToo7XD6QvCAEC5qTHby3o3LfHIhyZi/4L+AhS4FKUHF6M0peeyYt4z3HaK2d2 -N6mHLPdjwNjra7GOmcns4gzcrdfoF+R293KpPal4PjknvR3dZL4kKP/ougTAM5zv -qDIvRbkHzjP8ChTpoLcJsNVXykNcNkjcSi0GHtIpYjh6QX6P2uvR/S4+Bbb9p9rn -hIy/ovu9tWN2hiPxGPe6torF6BulAxsTYlDercC204AyzsrdA0pr6HBgJH9C6ML1 -TchwodbFJqn9rSv91i1liusAGoOvE81AGBdrXY7LxfSNhYY1IK6yR/POJPTd53sA -uX2/j6Rtoksd/2BHPM6AUnI/2B9slhuzWX2aCtWLeuwvXDS6rYuTigaQmLkzTRfM -dlMI3s9KLXxgi5YVumUZleJWXwBNP7KiKajd+VTSD+7WAhyhM5FIG5wVOaxmy4G2 -TyqZ/Ax9d2VEjTQHWvQlLPQ4Mp0EIz0aEl94K/S8CK8bJRH6+PRkar+dJi1xqlL+ -BYb42At9mEJ8odLlFikvNi1+t7jqXk5jRi5C0xFKx3nTtzoH2zNUeuA3R6vSocVK -45jnze9IkKmxMlJ4loR5sgszdpDCD3kXqjtCcbMTmcrGyzJek3HSOTpiEORoTFOe -Rhg6jH5lm+QcC263oipojS0qEQcnsWJP2CylNYMYHR9O/9NQxT3o2lsRHqZTMELV -uQa/SFH+paQNbZOj8MRwPSqqiIxJFuLswKte1R+W7LKn1yBSM7Pp39lNbzGvJD2E -YRfnCwFpJ54voVAuQ4jXJvigCW2qeCjXlxeD6K2j4eGJEEOmIjIW1wjubyBY6OI3 ------END RSA PRIVATE KEY----- +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIhD+rJdxqb6ECAggA +MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDTdyjCP3riOSUfxix4aXEvBIIH +ECGkbsFabrcFMZcplw5jHMaOlG7rYjUzwDJ80JM8uzbv2Jb8SvNlns2+xmnEvH/M +mNvRmnXmplbVjH3XBMK8o2Psnr2V/a0j7/pgqpRxHykG+koOY4gzdt3MAg8JPbS2 +hymSl+Y5EpciO3xLfz4aFL1ZNqspQbO/TD13Ij7DUIy7xIRBMp4taoZCrP0cEBAZ ++wgu9m23I4dh3E8RUBzWyFFNic2MVVHrui6JbHc4dIHfyKLtXJDhUcS0vIC9PvcV +jhorh3UZC4lM+/jjXV5AhzQ0VrJ2tXAUX2dA144XHzkSH2QmwfnajPsci7BL2CGC +rjyTy4NfB/lDwU+55dqJZQSKXMxAapJMrtgw7LD5CKQcN6zmfhXGssJ7HQUXKkaX +I1YOFzuUD7oo56BVCnVswv0jX9RxrE5QYNreMlOP9cS+kIYH65N+PAhlURuQC14K +PgDkHn5knSa2UQA5tc5f7zdHOZhGRUfcjLP+KAWA3nh+/2OKw/X3zuPx75YT/FKe +tACPw5hjEpl62m9Xa0eWepZXwqkIOkzHMmCyNCsbC0mmRoEjmvfnslfsmnh4Dg/c +4YsTYMOLLIeCa+WIc38aA5W2lNO9lW0LwLhX1rP+GRVPv+TVHXlfoyaI+jp0iXrJ +t3xxT0gaiIR/VznyS7Py68QV/zB7VdqbsNzS7LdquHK1k8+7OYiWjY3gqyU40Iu2 +d1eSnIoDvQJwyYp7XYXbOlXNLY+s1Qb7yxcW3vXm0Bg3gKT8r1XHWJ9rj+CxAn5r +ysfkPs1JsesxzzQjwTiDNvHnBnZnwxuxfBr26ektEHmuAXSl8V6dzLN/aaPjpTj4 +CkE7KyqX3U9bLkp+ztl4xWKEmW44nskzm0+iqrtrxMyTfvvID4QrABjZL4zmWIqc +e3ZfA3AYk9VDIegk/YKGC5VZ8YS7ZXQ0ASK652XqJ7QlMKTxxV7zda6Fp4uW6/qN +ezt5wgbGGhZQXj2wDQmWNQYyG/juIgYTpCUA54U5XBIjuR6pg+Ytm0UrvNjsUoAC +wGelyqaLDq8U8jdIFYVTJy9aJjQOYXjsUJ0dZN2aGHSlju0ZGIZc49cTIVQ9BTC5 +Yc0Vlwzpl+LuA25DzKZNSb/ci0lO/cQGJ2uXQQgaNgdsHlu8nukENGJhnIzx4fzK +wEh3yHxhTRCzPPwDfXmx0IHXrPqJhSpAgaXBVIm8OjvmMxO+W75W4uLfNY/B7e2H +3cjklGuvkofOf7sEOrGUYf4cb6Obg8FpvHgpKo5Twwmoh/qvEKckBFqNhZXDDl88 +GbGlSEgyaAV1Ig8s1NJKBolWFa0juyPAwJ8vT1T4iwW7kQ7KXKt2UNn96K/HxkLu +pikvukz8oRHMlfVHa0R48UB1fFHwZLzPmwkpu6ancIxk3uO3yfhf6iDk3bmnyMlz +g3k/b6MrLYaOVByRxay85jH3Vvgqfgn6wa6BJ7xQ81eZ8B45gFuTH0J5JtLL7SH8 +darRPLCYfA+Ums9/H6pU5EXfd3yfjMIbvhCXHkJrrljkZ+th3p8dyto6wmYqIY6I +qR9sU+o6DhRaiP8tCICuhHxQpXylUM6WeJkJwduTJ8KWIvzsj4mReIKOl/oC2jSd +gIdKhb9Q3zj9ce4N5m6v66tyvjxGZ+xf3BvUPDD+LwZeXgf7OBsNVbXzQbzto594 +nbCzPocFi3gERE50ru4K70eQCy08TPG5NpOz+DDdO5vpAuMLYEuI7O3L+3GjW40Q +G5bu7H5/i7o/RWR67qhG/7p9kPw3nkUtYgnvnWaPMIuTfb4c2d069kjlfgWjIbbI +tpSKmm5DHlqTE4/ECAbIEDtSaw9dXHCdL3nh5+n428xDdGbjN4lT86tfu17EYKzl +ydH1RJ1LX3o3TEj9UkmDPt7LnftvwybMFEcP7hM2xD4lC++wKQs7Alg6dTkBnJV4 +5xU78WRntJkJTU7kFkpPKA0QfyCuSF1fAMoukDBkqUdOj6jE0BlJQlHk5iwgnJlt +uEdkTjHZEjIUxWC6llPcAzaPNlmnD45AgfEW+Jn21IvutmJiQAz5lm9Z9PXaR0C8 +hXB6owRY67C0YKQwXhoNf6xQun2xGBGYy5rPEEezX1S1tUH5GR/KW1Lh+FzFqHXI +ZEb5avfDqHKehGAjPON+Br7akuQ125M9LLjKuSyPaQzeeCAy356Xd7XzVwbPddbm +9S9WSPqzaPgh10chIHoNoC8HMd33dB5j9/Q6jrbU/oPlptu/GlorWblvJdcTuBGI +IVn45RFnkG8hCz0GJSNzW7+70YdESQbfJW79vssWMaiSjFE0pMyFXrFR5lBywBTx +PiGEUWtvrKG94X1TMlGUzDzDJOQNZ9dT94bonNe9pVmP5BP4/DzwwiWh6qrzWk6p +j8OE4cfCSh2WvHnhJbH7/N0v+JKjtxeIeJ16jx/K2oK5 +-----END ENCRYPTED PRIVATE KEY----- -----BEGIN CERTIFICATE----- MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u @@ -66,3 +66,4 @@ jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu 9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW HcVKQHyOeyvnINuBAQ== -----END CERTIFICATE----- + diff --git a/Lib/test/make_ssl_certs.py b/Lib/test/make_ssl_certs.py index 3622765..41b5f46 100644 --- a/Lib/test/make_ssl_certs.py +++ b/Lib/test/make_ssl_certs.py @@ -206,8 +206,8 @@ if __name__ == '__main__': with open('ssl_key.pem', 'w') as f: f.write(key) print("password protecting ssl_key.pem in ssl_key.passwd.pem") - check_call(['openssl','rsa','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-des3','-passout','pass:somepass']) - check_call(['openssl','rsa','-in','ssl_key.pem','-out','keycert.passwd.pem','-des3','-passout','pass:somepass']) + check_call(['openssl','pkey','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-aes256','-passout','pass:somepass']) + check_call(['openssl','pkey','-in','ssl_key.pem','-out','keycert.passwd.pem','-aes256','-passout','pass:somepass']) with open('keycert.pem', 'w') as f: f.write(key) diff --git a/Lib/test/ssl_key.passwd.pem b/Lib/test/ssl_key.passwd.pem index e4f1370..46de61a 100644 --- a/Lib/test/ssl_key.passwd.pem +++ b/Lib/test/ssl_key.passwd.pem @@ -1,42 +1,42 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,8064BE1494B24B13 - -KJrffOMbo8M0I3PzcYxRZGMpKD1yB3Ii4+bT5XoanxjIJ+4fdx6LfZ0Rsx+riyzs -tymsQu/iYY9j+4rCvN9+eetsL1X6iZpiimKsLexcid9M3fb0vxED5Sgw0dvunCUA -xhqjLIKR92MKbODHf6KrDKCpsiPbjq4gZ7P+uCGXAMHL3MXIJSC0hW9rK7Ce6oyO -CjpIcgB8x+GUWZZZhAFdlzIHMZrteNP2P5HK6QcaT71P034Dz1hhqoj4Q0t+Fta2 -4tfsM/bnTR/l6hwlhPa1e3Uj322tDTDWBScgWANn5+sEWldLmozMaWhZsn22pfk2 -KjRMGXG024JVheV882nbdOBvG7oq+lxkZ/ZP+vvqJqnvYtf7WtM8UivzYpe5Hz5b -kVvWzPjBLUSZ9whM9rDLqSSqMPyPvDTuEmLkuq+xm7pYJmsLqIMP2klZLqRxLX6K -uqwplb8UG440qauxgnQ905PId1l2fJEnRtV+7vXprA0L0QotgXLVHBhLmTFM+3PH -9H3onf31dionUAPrn3nfVE36HhvVgRyvDBnBzJSIMighgq21Qx/d1dk0DRYi1hUI -nCHl0YJPXheVcXR7JiSF2XQCAaFuS1Mr7NCXfWZOZQC/0dkvmHnl9DUAhuqq9BNZ -1cKhZXcKHadg2/r0Zup/oDzmHPUEfTAXT0xbqoWlhkdwbF2veWQ96A/ncx3ISTb4 -PkXBlX9rdia8nmtyQDQRn4NuvchbaGkj4WKFC8pF8Hn7naHqwjpHaDUimBc0CoQW -edNJqruKWwtSVLuwKHCC2gZFX9AXSKJXJz/QRSUlhFGOhuF/J6yKaXj6n5lxWNiQ -54J+OP/hz2aS95CD2+Zf1SKpxdWiLZSIQqESpmmUrXROixNJZ/Z7gI74Dd9dSJOH -W+3AU03vrrFZVrJVZhjcINHoH1Skh6JKscH18L6x4U868nSr4SrRLX8BhHllOQyD -bmU+PZAjF8ZBIaCtTGulDXD29F73MeAZeTSsgQjFu0iKLj1wPiphbx8i/SUtR4YP -X6PVA04g66r1NBw+3RQASVorZ3g1MSFvITHXcbKkBDeJH2z1+c6t/VVyTONnQhM5 -lLgRSk6HCbetvT9PKxWrWutA12pdBYEHdZhMHVf2+xclky7l09w8hg2/qqcdGRGe -oAOZ72t0l5ObNyaruDKUS6f4AjOyWq/Xj5xuFtf1n3tQHyslSyCTPcAbQhDfTHUx -vixb/V9qvYPt7OCn8py7v1M69NH42QVFAvwveDIFjZdqfIKBoJK2V4qPoevJI6uj -Q5ByMt8OXOjSXNpHXpYQWUiWeCwOEBXJX8rzCHdMtg37jJ0zCmeErR1NTdg+EujM -TWYgd06jlT67tURST0aB2kg4ijKgUJefD313LW1zC6gVsTbjSZxYyRbPfSP6flQB -yCi1C19E2OsgleqbkBVC5GlYUzaJT7SGjCRmGx1eqtbrALu+LVH24Wceexlpjydl -+s2nf/DZlKun/tlPh6YioifPCJjByZMQOCEfIox6BkemZETz8uYA4TTWimG13Z03 -gyDGC2jdpEW414J2qcQDvrdUgJ+HlhrAAHaWpMQDbXYxBGoZ+3+ORvQV4kAsCwL8 -k3EIrVpePdik+1xgOWsyLj6QxFXlTMvL6Wc5pnArFPORsgHEolJvxSPTf9aAHNPn -V2WBvxiLBtYpGrujAUM40Syx/aN2RPtcXYPAusHUBw+S8/p+/8Kg8GZmnIXG3F89 -45Eepl2quZYIrou7a1fwIpIIZ0hFiBQ1mlHVMFtxwVHS1bQb3SU2GeO+JcGjdVXc -04qeGuQ5M164eQ5C0T7ZQ1ULiUlFWKD30m+cjqmZzt3d7Q0mKpMKuESIuZJo/wpD -Nas432aLKUhcNx/pOYLkKJRpGZKOupQoD5iUj/j44o8JoFkDK33v2S57XB5QGz28 -9Zuhx49b3W8mbM6EBanlQKLWJGCxXqc/jhYhFWn+b0MhidynFgA0oeWvf6ZDyt6H -Yi5Etxsar09xp0Do3NxtQXLuSUu0ji2pQzSIKuoqQWKqldm6VrpwojiqJhy4WQBQ -aVVyFeWBC7G3Zj76dO+yp2sfJ0itJUQ8AIB9Cg0f34rEZu+r9luPmqBoUeL95Tk7 -YvCOU3Jl8Iqysv8aNpVXT8sa8rrSbruWCByEePZ37RIdHLMVBwVY0eVaFQjrjU7E -mXmM9eaoYLfXOllsQ+M2+qPFUITr/GU3Qig13DhK/+yC1R6V2a0l0WRhMltIPYKW -Ztvvr4hK5LcYCeS113BLiMbDIMMZZYGDZGMdC8DnnVbT2loF0Rfmp80Af31KmMQ4 -6XvMatW9UDjBoY5a/YMpdm7SRwm+MgV2KNPpc2kST87/yi9oprGAb8qiarHiHTM0 ------END RSA PRIVATE KEY----- +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQI072N7W+PDDMCAggA +MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBA/AuaRNi4vE4KGqI4In+70BIIH +ENGS5Vex5NID873frmd1UZEHZ+O/Bd0wDb+NUpIqesHkRYf7kKi6Gnr+nKQ/oVVn +Lm3JjE7c8ECP0OkOOXmiXuWL1SkzBBWqCI4stSGUPvBiHsGwNnvJAaGjUffgMlcC +aJOA2+dnejLkzblq4CB2LQdm06N3Xoe9tyqtQaUHxfzJAf5Ydd8uj7vpKN2MMhY7 +icIPJwSyh0N7S6XWVtHEokr9Kp4y2hS5a+BgCWV1/1z0aF7agnSVndmT1VR+nWmc +lM14k+lethmHMB+fsNSjnqeJ7XOPlOTHqhiZ9bBSTgF/xr5Bck/NiKRzHjdovBox +TKg+xchaBhpRh7wBPBIlNJeHmIjv+8obOKjKU98Ig/7R9+IryZaNcKAH0PuOT+Sw +QHXiCGQbOiYHB9UyhDTWiB7YVjd8KHefOFxfHzOQb/iBhbv1x3bTl3DgepvRN6VO +dIsPLoIZe42sdf9GeMsk8mGJyZUQ6AzsfhWk3grb/XscizPSvrNsJ2VL1R7YTyT3 +3WA4ZXR1EqvXnWL7N/raemQjy62iOG6t7fcF5IdP9CMbWP+Plpsz4cQW7FtesCTq +a5ZXraochQz361ODFNIeBEGU+0qqXUtZDlmos/EySkZykSeU/L0bImS62VGE3afo +YXBmznTTT9kkFkqv7H0MerfJsrE/wF8puP3GM01DW2JRgXRpSWlvbPV/2LnMtRuD +II7iH4rWDtTjCN6BWKAgDOnPkc9sZ4XulqT32lcUeV6LTdMBfq8kMEc8eDij1vUT +maVCRpuwaq8EIT3lVgNLufHiG96ojlyYtj3orzw22IjkgC/9ee8UDik9CqbMVmFf +fVHhsw8LNSg8Q4bmwm5Eg2w2it2gtI68+mwr75oCxuJ/8OMjW21Prj8XDh5reie2 +c0lDKQOFZ9UnLU1bXR/6qUM+JFKR4DMq+fOCuoQSVoyVUEOsJpvBOYnYZN9cxsZm +vh9dKafMEcKZ8flsbr+gOmOw7+Py2ifSlf25E/Frb1W4gtbTb0LQVHb6+drutrZj +8HEu4CnHYFCD4ZnOJb26XlZCb8GFBddW86yJYyUqMMV6Q1aJfAOAglsTo1LjIMOZ +byo0BTAmwUevU/iuOXQ4qRBXXcoidDcTCrxfUSPG9wdt9l+m5SdQpWqfQ+fx5O7m +SLlrHyZCiPSFMtC9DxqjIklHjf5W3wslGLgaD30YXa4VDYkRihf3CNsxGQ+tVvef +l0ZjoAitF7Gaua06IESmKnpHe23dkr1cjYq+u2IV+xGH8LeExdwsQ9kpuTeXPnQs +JOA99SsFx1ct32RrwjxnDDsiNkaViTKo9GDkV3jQTfoFgAVqfSgg9wGXpqUqhNG7 +TiSIHCowllLny2zn4XrXCy2niD3VDt0skb3l/PaegHE2z7S5YY85nQtYwpLiwB9M +SQ08DYKxPBZYKtS2iZ/fsA1gjSRQDPg/SIxMhUC3M3qH8iWny1Lzl25F2Uq7VVEX +LdTUtaby49jRTT3CQGr5n6z7bMbUegiY7h8WmOekuThGDH+4xZp6+rDP4GFk4FeK +JcF70vMQYIjQZhadic6olv+9VtUP42ltGG/yP9a3eWRkzfAf2eCh6B1rYdgEWwE8 +rlcZzwM+y6eUmeNF2FVWB8iWtTMQHy+dYNPM+Jtus1KQKxiiq/yCRs7nWvzWRFWA +HRyqV0J6/lqgm4FvfktFt1T0W+mDoLJOR2/zIwMy2lgL5zeHuR3SaMJnCikJbqKS +HB3UvrhAWUcZqdH29+FhVWeM7ybyF1Wccmf+IIC/ePLa6gjtqPV8lG/5kbpcpnB6 +UQY8WWaKMxyr3jJ9bAX5QKshchp04cDecOLZrpFGNNQngR8RxSEkiIgAqNxWunIu +KrdBDrupv/XAgEOclmgToY3iywLJSV5gHAyHWDUhRH4cFCLiGPl4XIcnXOuTze3H +3j+EYSiS3v3DhHjp33YU2pXlJDjiYsKzAXejEh66++Y8qaQdCAad3ruWRCzW3kgk +Md0A1VGzntTnQsewvExQEMZH2LtYIsPv3KCYGeSAuLabX4tbGk79PswjnjLLEOr0 +Ghf6RF6qf5/iFyJoG4vrbKT8kx6ywh0InILCdjUunuDskIBxX6tEcr9XwajoIvb2 +kcmGdjam5kKLS7QOWQTl8/r/cuFes0dj34cX5Qpq+Gd7tRq/D+b0207926Cxvftv +qQ1cVn8HiLxKkZzd3tpf2xnoV1zkTL0oHrNg+qzxoxXUTUcwtIf1d/HRbYEAhi/d +bBBoFeftEHWNq+sJgS9bH+XNzo/yK4u04B5miOq8v4CSkJdzu+ZdF22d4cjiGmtQ +8BTmcn0Unzm+u5H0+QSZe54QBHJGNXXOIKMTkgnOdW27g4DbI1y7fCqJiSMbRW6L +oHmMfbdB3GWqGbsUkhY8i6h9op0MU6WOX7ea2Rxyt4t6 +-----END ENCRYPTED PRIVATE KEY----- diff --git a/Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst b/Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst new file mode 100644 index 0000000..8f43d32 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst @@ -0,0 +1,4 @@ +The private keys for test_ssl were encrypted with 3DES in traditional +PKCS#5 format. 3DES and the digest algorithm of PKCS#5 are blocked by +some strict crypto policies. Use PKCS#8 format with AES256 encryption +instead. -- 2.25.4 From 661938db95a1613526d42a7c34639e2307851fab Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Tue, 26 Nov 2019 23:57:21 +0100 Subject: [PATCH 02/47] Use PROTOCOL_TLS_CLIENT/SERVER Replaces PROTOCOL_TLSv* and PROTOCOL_SSLv23 with PROTOCOL_TLS_CLIENT and PROTOCOL_TLS_SERVER. Partially backports a170fa162dc03f0a014373349e548954fff2e567 --- Lib/ssl.py | 7 +- Lib/test/test_logging.py | 2 +- Lib/test/test_ssl.py | 169 +++++++++++++++++++-------------------- 3 files changed, 87 insertions(+), 91 deletions(-) diff --git a/Lib/ssl.py b/Lib/ssl.py index 0114387..c5c5529 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -473,7 +473,7 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None, context.load_default_certs(purpose) return context -def _create_unverified_context(protocol=PROTOCOL_TLS, *, cert_reqs=None, +def _create_unverified_context(protocol=PROTOCOL_TLS, *, cert_reqs=CERT_NONE, check_hostname=False, purpose=Purpose.SERVER_AUTH, certfile=None, keyfile=None, cafile=None, capath=None, cadata=None): @@ -492,9 +492,12 @@ def _create_unverified_context(protocol=PROTOCOL_TLS, *, cert_reqs=None, # by default. context = SSLContext(protocol) + if not check_hostname: + context.check_hostname = False if cert_reqs is not None: context.verify_mode = cert_reqs - context.check_hostname = check_hostname + if check_hostname: + context.check_hostname = True if keyfile and not certfile: raise ValueError("certfile must be specified") diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 763a5d1..d5c63b4 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -1830,7 +1830,7 @@ class HTTPHandlerTest(BaseTest): else: here = os.path.dirname(__file__) localhost_cert = os.path.join(here, "keycert.pem") - sslctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) sslctx.load_cert_chain(localhost_cert) context = ssl.create_default_context(cafile=localhost_cert) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 4f75f72..93c486a 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -155,6 +155,8 @@ def test_wrap_socket(sock, ssl_version=ssl.PROTOCOL_TLS, *, **kwargs): context = ssl.SSLContext(ssl_version) if cert_reqs is not None: + if cert_reqs == ssl.CERT_NONE: + context.check_hostname = False context.verify_mode = cert_reqs if ca_certs is not None: context.load_verify_locations(ca_certs) @@ -1377,7 +1379,7 @@ class ContextTests(unittest.TestCase): self._assert_context_options(ctx) def test_check_hostname(self): - ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) self.assertFalse(ctx.check_hostname) # Requires CERT_REQUIRED or CERT_OPTIONAL @@ -2371,17 +2373,13 @@ if _have_threads: server_params_test(context, context, chatty=True, connectionchatty=True) - client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - client_context.load_verify_locations(SIGNING_CA) - server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - # server_context.load_verify_locations(SIGNING_CA) - server_context.load_cert_chain(SIGNED_CERTFILE2) + client_context, server_context, hostname = testing_context() with self.subTest(client=ssl.PROTOCOL_TLS_CLIENT, server=ssl.PROTOCOL_TLS_SERVER): server_params_test(client_context=client_context, server_context=server_context, chatty=True, connectionchatty=True, - sni_name='fakehostname') + sni_name='localhost') client_context.check_hostname = False with self.subTest(client=ssl.PROTOCOL_TLS_SERVER, server=ssl.PROTOCOL_TLS_CLIENT): @@ -2389,7 +2387,7 @@ if _have_threads: server_params_test(client_context=server_context, server_context=client_context, chatty=True, connectionchatty=True, - sni_name='fakehostname') + sni_name='localhost') self.assertIn('called a function you should not call', str(e.exception)) @@ -2454,39 +2452,38 @@ if _have_threads: if support.verbose: sys.stdout.write("\n") - server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - server_context.load_cert_chain(SIGNED_CERTFILE) + client_context, server_context, hostname = testing_context() - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - context.verify_mode = ssl.CERT_REQUIRED - context.load_verify_locations(SIGNING_CA) tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) - self.assertEqual(context.verify_flags, ssl.VERIFY_DEFAULT | tf) + self.assertEqual(client_context.verify_flags, ssl.VERIFY_DEFAULT | tf) # VERIFY_DEFAULT should pass server = ThreadedEchoServer(context=server_context, chatty=True) with server: - with context.wrap_socket(socket.socket()) as s: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: s.connect((HOST, server.port)) cert = s.getpeercert() self.assertTrue(cert, "Can't get peer certificate.") # VERIFY_CRL_CHECK_LEAF without a loaded CRL file fails - context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + client_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF server = ThreadedEchoServer(context=server_context, chatty=True) with server: - with context.wrap_socket(socket.socket()) as s: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: with self.assertRaisesRegex(ssl.SSLError, "certificate verify failed"): s.connect((HOST, server.port)) # now load a CRL file. The CRL file is signed by the CA. - context.load_verify_locations(CRLFILE) + client_context.load_verify_locations(CRLFILE) server = ThreadedEchoServer(context=server_context, chatty=True) with server: - with context.wrap_socket(socket.socket()) as s: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: s.connect((HOST, server.port)) cert = s.getpeercert() self.assertTrue(cert, "Can't get peer certificate.") @@ -2495,19 +2492,13 @@ if _have_threads: if support.verbose: sys.stdout.write("\n") - server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - server_context.load_cert_chain(SIGNED_CERTFILE) - - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - context.verify_mode = ssl.CERT_REQUIRED - context.check_hostname = True - context.load_verify_locations(SIGNING_CA) + client_context, server_context, hostname = testing_context() # correct hostname should verify server = ThreadedEchoServer(context=server_context, chatty=True) with server: - with context.wrap_socket(socket.socket(), - server_hostname="localhost") as s: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: s.connect((HOST, server.port)) cert = s.getpeercert() self.assertTrue(cert, "Can't get peer certificate.") @@ -2515,7 +2506,7 @@ if _have_threads: # incorrect hostname should raise an exception server = ThreadedEchoServer(context=server_context, chatty=True) with server: - with context.wrap_socket(socket.socket(), + with client_context.wrap_socket(socket.socket(), server_hostname="invalid") as s: with self.assertRaisesRegex(ssl.CertificateError, "hostname 'invalid' doesn't match 'localhost'"): @@ -2527,7 +2518,7 @@ if _have_threads: with socket.socket() as s: with self.assertRaisesRegex(ValueError, "check_hostname requires server_hostname"): - context.wrap_socket(s) + client_context.wrap_socket(s) def test_wrong_cert(self): """Connecting when the server rejects the client's certificate @@ -2752,7 +2743,6 @@ if _have_threads: msgs = (b"msg 1", b"MSG 2", b"STARTTLS", b"MSG 3", b"msg 4", b"ENDTLS", b"msg 5", b"msg 6") server = ThreadedEchoServer(CERTFILE, - ssl_version=ssl.PROTOCOL_TLSv1, starttls_server=True, chatty=True, connectionchatty=True) @@ -2780,7 +2770,7 @@ if _have_threads: sys.stdout.write( " client: read %r from server, starting TLS...\n" % msg) - conn = test_wrap_socket(s, ssl_version=ssl.PROTOCOL_TLSv1) + conn = test_wrap_socket(s) wrapped = True elif indata == b"ENDTLS" and msg.startswith(b"ok"): # ENDTLS ok, switch back to clear text @@ -2867,7 +2857,7 @@ if _have_threads: server = ThreadedEchoServer(CERTFILE, certreqs=ssl.CERT_NONE, - ssl_version=ssl.PROTOCOL_TLSv1, + ssl_version=ssl.PROTOCOL_TLS_SERVER, cacerts=CERTFILE, chatty=True, connectionchatty=False) @@ -2877,7 +2867,7 @@ if _have_threads: certfile=CERTFILE, ca_certs=CERTFILE, cert_reqs=ssl.CERT_NONE, - ssl_version=ssl.PROTOCOL_TLSv1) + ssl_version=ssl.PROTOCOL_TLS_CLIENT) s.connect((HOST, server.port)) # helper methods for standardising recv* method signatures def _recv_into(): @@ -3019,7 +3009,7 @@ if _have_threads: def test_nonblocking_send(self): server = ThreadedEchoServer(CERTFILE, certreqs=ssl.CERT_NONE, - ssl_version=ssl.PROTOCOL_TLSv1, + ssl_version=ssl.PROTOCOL_TLS_SERVER, cacerts=CERTFILE, chatty=True, connectionchatty=False) @@ -3029,7 +3019,7 @@ if _have_threads: certfile=CERTFILE, ca_certs=CERTFILE, cert_reqs=ssl.CERT_NONE, - ssl_version=ssl.PROTOCOL_TLSv1) + ssl_version=ssl.PROTOCOL_TLS_CLIENT) s.connect((HOST, server.port)) s.setblocking(False) @@ -3175,9 +3165,11 @@ if _have_threads: Basic tests for SSLSocket.version(). More tests are done in the test_protocol_*() methods. """ - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE with ThreadedEchoServer(CERTFILE, - ssl_version=ssl.PROTOCOL_TLSv1, + ssl_version=ssl.PROTOCOL_TLS_SERVER, chatty=False) as server: with context.wrap_socket(socket.socket()) as s: self.assertIs(s.version(), None) @@ -3232,7 +3224,7 @@ if _have_threads: server = ThreadedEchoServer(CERTFILE, certreqs=ssl.CERT_NONE, - ssl_version=ssl.PROTOCOL_TLSv1, + ssl_version=ssl.PROTOCOL_TLS_SERVER, cacerts=CERTFILE, chatty=True, connectionchatty=False) @@ -3242,7 +3234,7 @@ if _have_threads: certfile=CERTFILE, ca_certs=CERTFILE, cert_reqs=ssl.CERT_NONE, - ssl_version=ssl.PROTOCOL_TLSv1) + ssl_version=ssl.PROTOCOL_TLS_CLIENT) s.connect((HOST, server.port)) # get the data cb_data = s.get_channel_binding("tls-unique") @@ -3267,7 +3259,7 @@ if _have_threads: certfile=CERTFILE, ca_certs=CERTFILE, cert_reqs=ssl.CERT_NONE, - ssl_version=ssl.PROTOCOL_TLSv1) + ssl_version=ssl.PROTOCOL_TLS_CLIENT) s.connect((HOST, server.port)) new_cb_data = s.get_channel_binding("tls-unique") if support.verbose: @@ -3284,32 +3276,35 @@ if _have_threads: s.close() def test_compression(self): - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - context.load_cert_chain(CERTFILE) - stats = server_params_test(context, context, - chatty=True, connectionchatty=True) + client_context, server_context, hostname = testing_context() + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) if support.verbose: sys.stdout.write(" got compression: {!r}\n".format(stats['compression'])) self.assertIn(stats['compression'], { None, 'ZLIB', 'RLE' }) + @unittest.skipUnless(hasattr(ssl, 'OP_NO_COMPRESSION'), "ssl.OP_NO_COMPRESSION needed for this test") def test_compression_disabled(self): - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - context.load_cert_chain(CERTFILE) - context.options |= ssl.OP_NO_COMPRESSION - stats = server_params_test(context, context, - chatty=True, connectionchatty=True) + client_context, server_context, hostname = testing_context() + client_context.options |= ssl.OP_NO_COMPRESSION + server_context.options |= ssl.OP_NO_COMPRESSION + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) self.assertIs(stats['compression'], None) def test_dh_params(self): # Check we can get a connection with ephemeral Diffie-Hellman - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - context.load_cert_chain(CERTFILE) - context.load_dh_params(DHFILE) - context.set_ciphers("kEDH") - stats = server_params_test(context, context, - chatty=True, connectionchatty=True) + client_context, server_context, hostname = testing_context() + server_context.load_dh_params(DHFILE) + server_context.set_ciphers("kEDH") + server_context.options |= ssl.OP_NO_TLSv1_3 + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) cipher = stats["cipher"][0] parts = cipher.split("-") if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: @@ -3317,22 +3312,20 @@ if _have_threads: def test_selected_alpn_protocol(self): # selected_alpn_protocol() is None unless ALPN is used. - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - context.load_cert_chain(CERTFILE) - stats = server_params_test(context, context, - chatty=True, connectionchatty=True) + client_context, server_context, hostname = testing_context() + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) self.assertIs(stats['client_alpn_protocol'], None) @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support required") def test_selected_alpn_protocol_if_server_uses_alpn(self): # selected_alpn_protocol() is None unless ALPN is used by the client. - client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - client_context.load_verify_locations(CERTFILE) - server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - server_context.load_cert_chain(CERTFILE) + client_context, server_context, hostname = testing_context() server_context.set_alpn_protocols(['foo', 'bar']) stats = server_params_test(client_context, server_context, - chatty=True, connectionchatty=True) + chatty=True, connectionchatty=True, + sni_name=hostname) self.assertIs(stats['client_alpn_protocol'], None) @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support needed for this test") @@ -3379,10 +3372,10 @@ if _have_threads: def test_selected_npn_protocol(self): # selected_npn_protocol() is None unless NPN is used - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - context.load_cert_chain(CERTFILE) - stats = server_params_test(context, context, - chatty=True, connectionchatty=True) + client_context, server_context, hostname = testing_context() + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) self.assertIs(stats['client_npn_protocol'], None) @unittest.skipUnless(ssl.HAS_NPN, "NPN support needed for this test") @@ -3415,12 +3408,11 @@ if _have_threads: self.assertEqual(server_result, expected, msg % (server_result, "server")) def sni_contexts(self): - server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) server_context.load_cert_chain(SIGNED_CERTFILE) - other_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + other_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) other_context.load_cert_chain(SIGNED_CERTFILE2) - client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - client_context.verify_mode = ssl.CERT_REQUIRED + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) client_context.load_verify_locations(SIGNING_CA) return server_context, other_context, client_context @@ -3433,6 +3425,8 @@ if _have_threads: calls = [] server_context, other_context, client_context = self.sni_contexts() + client_context.check_hostname = False + def servername_cb(ssl_sock, server_name, initial_context): calls.append((server_name, initial_context)) if server_name is not None: @@ -3518,11 +3512,7 @@ if _have_threads: self.assertIn("TypeError", stderr.getvalue()) def test_shared_ciphers(self): - server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - server_context.load_cert_chain(SIGNED_CERTFILE) - client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - client_context.verify_mode = ssl.CERT_REQUIRED - client_context.load_verify_locations(SIGNING_CA) + client_context, server_context, hostname = testing_context() if ssl.OPENSSL_VERSION_INFO >= (1, 0, 2): client_context.set_ciphers("AES128:AES256") server_context.set_ciphers("AES256") @@ -3540,7 +3530,8 @@ if _have_threads: # TLS 1.3 ciphers are always enabled expected_algs.extend(["TLS_CHACHA20", "TLS_AES"]) - stats = server_params_test(client_context, server_context) + stats = server_params_test(client_context, server_context, + sni_name=hostname) ciphers = stats['server_shared_ciphers'][0] self.assertGreater(len(ciphers), 0) for name, tls_version, bits in ciphers: @@ -3580,14 +3571,13 @@ if _have_threads: self.assertEqual(s.recv(1024), TEST_DATA) def test_session(self): - server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - server_context.load_cert_chain(SIGNED_CERTFILE) - client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - client_context.verify_mode = ssl.CERT_REQUIRED - client_context.load_verify_locations(SIGNING_CA) + client_context, server_context, hostname = testing_context() + # TODO: sessions aren't compatible with TLSv1.3 yet + client_context.options |= ssl.OP_NO_TLSv1_3 # first connection without session - stats = server_params_test(client_context, server_context) + stats = server_params_test(client_context, server_context, + sni_name=hostname) session = stats['session'] self.assertTrue(session.id) self.assertGreater(session.time, 0) @@ -3601,7 +3591,8 @@ if _have_threads: self.assertEqual(sess_stat['hits'], 0) # reuse session - stats = server_params_test(client_context, server_context, session=session) + stats = server_params_test(client_context, server_context, + session=session, sni_name=hostname) sess_stat = server_context.session_stats() self.assertEqual(sess_stat['accept'], 2) self.assertEqual(sess_stat['hits'], 1) @@ -3614,7 +3605,8 @@ if _have_threads: self.assertGreaterEqual(session2.timeout, session.timeout) # another one without session - stats = server_params_test(client_context, server_context) + stats = server_params_test(client_context, server_context, + sni_name=hostname) self.assertFalse(stats['session_reused']) session3 = stats['session'] self.assertNotEqual(session3.id, session.id) @@ -3624,7 +3616,8 @@ if _have_threads: self.assertEqual(sess_stat['hits'], 1) # reuse session again - stats = server_params_test(client_context, server_context, session=session) + stats = server_params_test(client_context, server_context, + session=session, sni_name=hostname) self.assertTrue(stats['session_reused']) session4 = stats['session'] self.assertEqual(session4.id, session.id) -- 2.25.4 From 1f5baa42a46189eae205c007525f0c9825da05ae Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Wed, 27 Nov 2019 00:01:17 +0100 Subject: [PATCH 03/47] Adjust some tests for TLS 1.3 compatibility Partially backports some changes from 529525fb5a8fd9b96ab4021311a598c77588b918 and 2614ed4c6e4b32eafb683f2378ed20e87d42976d --- Lib/test/test_ssl.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 93c486a..0768e53 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -3174,7 +3174,12 @@ if _have_threads: with context.wrap_socket(socket.socket()) as s: self.assertIs(s.version(), None) s.connect((HOST, server.port)) - self.assertEqual(s.version(), 'TLSv1') + if IS_OPENSSL_1_1: + self.assertEqual(s.version(), 'TLSv1.3') + elif ssl.OPENSSL_VERSION_INFO >= (1, 0, 2): + self.assertEqual(s.version(), 'TLSv1.2') + else: # 0.9.8 to 1.0.1 + self.assertIn(s.version(), ('TLSv1', 'TLSv1.2')) self.assertIs(s.version(), None) @unittest.skipUnless(ssl.HAS_TLSv1_3, @@ -3244,7 +3249,10 @@ if _have_threads: # check if it is sane self.assertIsNotNone(cb_data) - self.assertEqual(len(cb_data), 12) # True for TLSv1 + if s.version() == 'TLSv1.3': + self.assertEqual(len(cb_data), 48) + else: + self.assertEqual(len(cb_data), 12) # True for TLSv1 # and compare with the peers version s.write(b"CB tls-unique\n") @@ -3268,7 +3276,10 @@ if _have_threads: # is it really unique self.assertNotEqual(cb_data, new_cb_data) self.assertIsNotNone(cb_data) - self.assertEqual(len(cb_data), 12) # True for TLSv1 + if s.version() == 'TLSv1.3': + self.assertEqual(len(cb_data), 48) + else: + self.assertEqual(len(cb_data), 12) # True for TLSv1 s.write(b"CB tls-unique\n") peer_data_repr = s.read().strip() self.assertEqual(peer_data_repr, -- 2.25.4 From 8456f457519a5f52c136e6a96863aa90f8aeca40 Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Tue, 26 Nov 2019 23:18:10 +0100 Subject: [PATCH 04/47] Skip the ssl tests that rely on TLSv1 and TLSv1.1 availability --- Lib/test/test_ssl.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 0768e53..cd425eb 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -39,6 +39,13 @@ IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL') IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0) PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') +# On RHEL8 openssl disables TLSv1 and TLSv1.1 on runtime. +# Since we don't have a good way to detect runtime changes +# on the allowed protocols, we hardcode the default config +# with those flags. +TLSv1_enabled = False +TLSv1_1_enabled = False + def data_file(*name): return os.path.join(os.path.dirname(__file__), *name) @@ -2365,7 +2372,8 @@ if _have_threads: if support.verbose: sys.stdout.write("\n") for protocol in PROTOCOLS: - if protocol in {ssl.PROTOCOL_TLS_CLIENT, ssl.PROTOCOL_TLS_SERVER}: + if protocol in {ssl.PROTOCOL_TLS_CLIENT, ssl.PROTOCOL_TLS_SERVER, + ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1}: continue with self.subTest(protocol=ssl._PROTOCOL_NAMES[protocol]): context = ssl.SSLContext(protocol) @@ -2635,17 +2643,20 @@ if _have_threads: if hasattr(ssl, 'PROTOCOL_SSLv3'): try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False) try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1') + if TLSv1_enabled: + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1') if hasattr(ssl, 'PROTOCOL_SSLv3'): try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL) try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_OPTIONAL) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + if TLSv1_enabled: + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) if hasattr(ssl, 'PROTOCOL_SSLv3'): try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED) try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_REQUIRED) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + if TLSv1_enabled: + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) # Server with specific SSL options if hasattr(ssl, 'PROTOCOL_SSLv3'): @@ -2683,9 +2694,10 @@ if _have_threads: """Connecting to a TLSv1 server with various client options""" if support.verbose: sys.stdout.write("\n") - try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1') - try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) - try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + if TLSv1_enabled: + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1') + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) if hasattr(ssl, 'PROTOCOL_SSLv2'): try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False) if hasattr(ssl, 'PROTOCOL_SSLv3'): @@ -2701,7 +2713,8 @@ if _have_threads: Testing against older TLS versions.""" if support.verbose: sys.stdout.write("\n") - try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + if TLSv1_1_enabled: + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') if hasattr(ssl, 'PROTOCOL_SSLv2'): try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False) if hasattr(ssl, 'PROTOCOL_SSLv3'): @@ -2709,7 +2722,8 @@ if _have_threads: try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv23, False, client_options=ssl.OP_NO_TLSv1_1) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + if TLSv1_1_enabled: + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False) try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False) -- 2.25.4 From 5d8786dbf7b869d1aa86b6741c41585084e77380 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 25 Sep 2019 08:50:31 -0700 Subject: [PATCH 05/47] bpo-38270: Check for hash digest algorithms and avoid MD5 (GH-16382) (GH-16393) Make it easier to run and test Python on systems with restrict crypto policies: * add requires_hashdigest to test.support to check if a hash digest algorithm is available and working * avoid MD5 in test_hmac * replace MD5 with SHA256 in test_tarfile * mark network tests that require MD5 for MD5-based digest auth or CRAM-MD5 https://bugs.python.org/issue38270 (cherry picked from commit c64a1a61e6fc542cada40eb069a239317e1af36e) Co-authored-by: Christian Heimes https://bugs.python.org/issue38270 Automerge-Triggered-By: @tiran --- Lib/test/support/__init__.py | 22 ++++++++++ Lib/test/test_hmac.py | 67 +++++++++++++++++++------------ Lib/test/test_imaplib.py | 6 ++- Lib/test/test_poplib.py | 2 + Lib/test/test_smtplib.py | 11 ++++- Lib/test/test_tarfile.py | 56 ++++++++++++++------------ Lib/test/test_urllib2_localnet.py | 1 + 7 files changed, 112 insertions(+), 53 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 66c0fed..a438199 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -11,6 +11,7 @@ import faulthandler import fnmatch import functools import gc +import hashlib import importlib import importlib.util import io @@ -627,6 +628,27 @@ def requires_mac_ver(*min_version): return wrapper return decorator +def requires_hashdigest(digestname): + """Decorator raising SkipTest if a hashing algorithm is not available + + The hashing algorithm could be missing or blocked by a strict crypto + policy. + + ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS + ValueError: unsupported hash type md4 + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + hashlib.new(digestname) + except ValueError: + raise unittest.SkipTest( + f"hash digest '{digestname}' is not available." + ) + return func(*args, **kwargs) + return wrapper + return decorator # Don't use "localhost", since resolving it uses the DNS under recent # Windows versions (see issue #18792). diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 067e13f..81c3485 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -4,6 +4,8 @@ import hashlib import unittest import warnings +from test.support import requires_hashdigest + def ignore_warning(func): @functools.wraps(func) @@ -17,6 +19,7 @@ def ignore_warning(func): class TestVectorsTestCase(unittest.TestCase): + @requires_hashdigest('md5') def test_md5_vectors(self): # Test the HMAC module against test vectors from the RFC. @@ -63,6 +66,7 @@ class TestVectorsTestCase(unittest.TestCase): b"and Larger Than One Block-Size Data"), "6f630fad67cda0ee1fb1f562db3aa53e") + @requires_hashdigest('sha1') def test_sha_vectors(self): def shatest(key, data, digest): h = hmac.HMAC(key, data, digestmod=hashlib.sha1) @@ -230,23 +234,28 @@ class TestVectorsTestCase(unittest.TestCase): '134676fb6de0446065c97440fa8c6a58', }) + @requires_hashdigest('sha224') def test_sha224_rfc4231(self): self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64) + @requires_hashdigest('sha256') def test_sha256_rfc4231(self): self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64) + @requires_hashdigest('sha384') def test_sha384_rfc4231(self): self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128) + @requires_hashdigest('sha512') def test_sha512_rfc4231(self): self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128) + @requires_hashdigest('sha256') def test_legacy_block_size_warnings(self): class MockCrazyHash(object): """Ain't no block_size attribute here.""" def __init__(self, *args): - self._x = hashlib.sha1(*args) + self._x = hashlib.sha256(*args) self.digest_size = self._x.digest_size def update(self, v): self._x.update(v) @@ -273,76 +282,80 @@ class TestVectorsTestCase(unittest.TestCase): self.assertEqual(h.hexdigest().upper(), digest) + class ConstructorTestCase(unittest.TestCase): + expected = ( + "6c845b47f52b3b47f6590c502db7825aad757bf4fadc8fa972f7cd2e76a5bdeb" + ) @ignore_warning + @requires_hashdigest('sha256') def test_normal(self): # Standard constructor call. - failed = 0 try: - h = hmac.HMAC(b"key") + hmac.HMAC(b"key", digestmod='sha256') except Exception: self.fail("Standard constructor call raised exception.") @ignore_warning + @requires_hashdigest('sha256') def test_with_str_key(self): # Pass a key of type str, which is an error, because it expects a key # of type bytes with self.assertRaises(TypeError): - h = hmac.HMAC("key") + h = hmac.HMAC("key", digestmod='sha256') - @ignore_warning + @requires_hashdigest('sha256') def test_dot_new_with_str_key(self): # Pass a key of type str, which is an error, because it expects a key # of type bytes with self.assertRaises(TypeError): - h = hmac.new("key") + h = hmac.HMAC("key", digestmod='sha256') @ignore_warning + @requires_hashdigest('sha256') def test_withtext(self): # Constructor call with text. try: - h = hmac.HMAC(b"key", b"hash this!") + h = hmac.HMAC(b"key", b"hash this!", digestmod='sha256') except Exception: self.fail("Constructor call with text argument raised exception.") - self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864') + self.assertEqual(h.hexdigest(), self.expected) + @requires_hashdigest('sha256') def test_with_bytearray(self): try: h = hmac.HMAC(bytearray(b"key"), bytearray(b"hash this!"), - digestmod="md5") + digestmod="sha256") except Exception: self.fail("Constructor call with bytearray arguments raised exception.") - self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864') + self.assertEqual(h.hexdigest(), self.expected) + @requires_hashdigest('sha256') def test_with_memoryview_msg(self): try: - h = hmac.HMAC(b"key", memoryview(b"hash this!"), digestmod="md5") + h = hmac.HMAC(b"key", memoryview(b"hash this!"), digestmod="sha256") except Exception: self.fail("Constructor call with memoryview msg raised exception.") - self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864') + self.assertEqual(h.hexdigest(), self.expected) + @requires_hashdigest('sha256') def test_withmodule(self): # Constructor call with text and digest module. try: - h = hmac.HMAC(b"key", b"", hashlib.sha1) + h = hmac.HMAC(b"key", b"", hashlib.sha256) except Exception: - self.fail("Constructor call with hashlib.sha1 raised exception.") + self.fail("Constructor call with hashlib.sha256 raised exception.") -class SanityTestCase(unittest.TestCase): - @ignore_warning - def test_default_is_md5(self): - # Testing if HMAC defaults to MD5 algorithm. - # NOTE: this whitebox test depends on the hmac class internals - h = hmac.HMAC(b"key") - self.assertEqual(h.digest_cons, hashlib.md5) +class SanityTestCase(unittest.TestCase): + @requires_hashdigest('sha256') def test_exercise_all_methods(self): # Exercising all methods once. # This must not raise any exceptions try: - h = hmac.HMAC(b"my secret key", digestmod="md5") + h = hmac.HMAC(b"my secret key", digestmod="sha256") h.update(b"compute the hash of this text!") dig = h.digest() dig = h.hexdigest() @@ -350,11 +363,13 @@ class SanityTestCase(unittest.TestCase): except Exception: self.fail("Exception raised during normal usage of HMAC class.") + class CopyTestCase(unittest.TestCase): + @requires_hashdigest('sha256') def test_attributes(self): # Testing if attributes are of same type. - h1 = hmac.HMAC(b"key", digestmod="md5") + h1 = hmac.HMAC(b"key", digestmod="sha256") h2 = h1.copy() self.assertTrue(h1.digest_cons == h2.digest_cons, "digest constructors don't match.") @@ -363,9 +378,10 @@ class CopyTestCase(unittest.TestCase): self.assertEqual(type(h1.outer), type(h2.outer), "Types of outer don't match.") + @requires_hashdigest('sha256') def test_realcopy(self): # Testing if the copy method created a real copy. - h1 = hmac.HMAC(b"key", digestmod="md5") + h1 = hmac.HMAC(b"key", digestmod="sha256") h2 = h1.copy() # Using id() in case somebody has overridden __eq__/__ne__. self.assertTrue(id(h1) != id(h2), "No real copy of the HMAC instance.") @@ -374,9 +390,10 @@ class CopyTestCase(unittest.TestCase): self.assertTrue(id(h1.outer) != id(h2.outer), "No real copy of the attribute 'outer'.") + @requires_hashdigest('sha256') def test_equality(self): # Testing if the copy has the same digests. - h1 = hmac.HMAC(b"key", digestmod="md5") + h1 = hmac.HMAC(b"key", digestmod="sha256") h1.update(b"some random text") h2 = h1.copy() self.assertEqual(h1.digest(), h2.digest(), diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 0593a37..086cc0a 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -14,7 +14,8 @@ import calendar import inspect from test.support import (reap_threads, verbose, transient_internet, - run_with_tz, run_with_locale, cpython_only) + run_with_tz, run_with_locale, cpython_only, + requires_hashdigest) import unittest from unittest import mock from datetime import datetime, timezone, timedelta @@ -369,6 +370,7 @@ class NewIMAPTestsMixin(): self.assertEqual(code, 'OK') self.assertEqual(server.response, b'ZmFrZQ==\r\n') # b64 encoded 'fake' + @requires_hashdigest('md5') def test_login_cram_md5_bytes(self): class AuthHandler(SimpleIMAPHandler): capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' @@ -386,6 +388,7 @@ class NewIMAPTestsMixin(): ret, _ = client.login_cram_md5("tim", b"tanstaaftanstaaf") self.assertEqual(ret, "OK") + @requires_hashdigest('md5') def test_login_cram_md5_plain_text(self): class AuthHandler(SimpleIMAPHandler): capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' @@ -797,6 +800,7 @@ class ThreadedNetworkedTests(unittest.TestCase): b'ZmFrZQ==\r\n') # b64 encoded 'fake' @reap_threads + @requires_hashdigest('md5') def test_login_cram_md5(self): class AuthHandler(SimpleIMAPHandler): diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 234c855..b8146be 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -304,9 +304,11 @@ class TestPOP3Class(TestCase): def test_rpop(self): self.assertOK(self.client.rpop('foo')) + @test_support.requires_hashdigest('md5') def test_apop_normal(self): self.assertOK(self.client.apop('foo', 'dummypassword')) + @test_support.requires_hashdigest('md5') def test_apop_REDOS(self): # Replace welcome with very long evil welcome. # NB The upper bound on welcome length is currently 2048. diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 8704751..64b3201 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -4,6 +4,7 @@ import email.mime.text from email.message import EmailMessage from email.base64mime import body_encode as encode_base64 import email.utils +import hashlib import hmac import socket import smtpd @@ -18,6 +19,7 @@ import textwrap import unittest from test import support, mock_socket +from test.support import requires_hashdigest from unittest.mock import Mock HOST = "localhost" @@ -968,6 +970,7 @@ class SMTPSimTests(unittest.TestCase): self.assertEqual(resp, (235, b'Authentication Succeeded')) smtp.close() + @requires_hashdigest('md5') def testAUTH_CRAM_MD5(self): self.serv.add_feature("AUTH CRAM-MD5") smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) @@ -984,7 +987,13 @@ class SMTPSimTests(unittest.TestCase): smtp.close() def test_auth_function(self): - supported = {'CRAM-MD5', 'PLAIN', 'LOGIN'} + supported = {'PLAIN', 'LOGIN'} + try: + hashlib.md5() + except ValueError: + pass + else: + supported.add('CRAM-MD5') for mechanism in supported: self.serv.add_feature("AUTH {}".format(mechanism)) for mechanism in supported: diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 4cd7d53..b5e855e 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1,7 +1,7 @@ import sys import os import io -from hashlib import md5 +from hashlib import sha256 from contextlib import contextmanager from random import Random import pathlib @@ -11,7 +11,7 @@ import unittest.mock import tarfile from test import support -from test.support import script_helper +from test.support import script_helper, requires_hashdigest # Check for our compression modules. try: @@ -27,8 +27,8 @@ try: except ImportError: lzma = None -def md5sum(data): - return md5(data).hexdigest() +def sha256sum(data): + return sha256(data).hexdigest() TEMPDIR = os.path.abspath(support.TESTFN) + "-tardir" tarextdir = TEMPDIR + '-extract-test' @@ -39,8 +39,12 @@ xzname = os.path.join(TEMPDIR, "testtar.tar.xz") tmpname = os.path.join(TEMPDIR, "tmp.tar") dotlessname = os.path.join(TEMPDIR, "testtar") -md5_regtype = "65f477c818ad9e15f7feab0c6d37742f" -md5_sparse = "a54fbc4ca4f4399a90e1b27164012fc6" +sha256_regtype = ( + "e09e4bc8b3c9d9177e77256353b36c159f5f040531bbd4b024a8f9b9196c71ce" +) +sha256_sparse = ( + "4f05a776071146756345ceee937b33fc5644f5a96b9780d1c7d6a32cdf164d7b" +) class TarTest: @@ -95,7 +99,7 @@ class UstarReadTest(ReadTest, unittest.TestCase): data = fobj.read() self.assertEqual(len(data), tarinfo.size, "regular file extraction failed") - self.assertEqual(md5sum(data), md5_regtype, + self.assertEqual(sha256sum(data), sha256_regtype, "regular file extraction failed") def test_fileobj_readlines(self): @@ -180,7 +184,7 @@ class UstarReadTest(ReadTest, unittest.TestCase): with self.tar.extractfile("ustar/regtype") as fobj: fobj = io.TextIOWrapper(fobj) data = fobj.read().encode("iso8859-1") - self.assertEqual(md5sum(data), md5_regtype) + self.assertEqual(sha256sum(data), sha256_regtype) try: fobj.seek(100) except AttributeError: @@ -546,13 +550,13 @@ class MiscReadTestBase(CommonReadTest): self.addCleanup(support.unlink, os.path.join(TEMPDIR, "ustar/lnktype")) with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f: data = f.read() - self.assertEqual(md5sum(data), md5_regtype) + self.assertEqual(sha256sum(data), sha256_regtype) tar.extract("ustar/symtype", TEMPDIR) self.addCleanup(support.unlink, os.path.join(TEMPDIR, "ustar/symtype")) with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f: data = f.read() - self.assertEqual(md5sum(data), md5_regtype) + self.assertEqual(sha256sum(data), sha256_regtype) def test_extractall(self): # Test if extractall() correctly restores directory permissions @@ -687,7 +691,7 @@ class StreamReadTest(CommonReadTest, unittest.TestCase): data = fobj.read() self.assertEqual(len(data), tarinfo.size, "regular file extraction failed") - self.assertEqual(md5sum(data), md5_regtype, + self.assertEqual(sha256sum(data), sha256_regtype, "regular file extraction failed") def test_provoke_stream_error(self): @@ -799,8 +803,8 @@ class MemberReadTest(ReadTest, unittest.TestCase): def _test_member(self, tarinfo, chksum=None, **kwargs): if chksum is not None: with self.tar.extractfile(tarinfo) as f: - self.assertEqual(md5sum(f.read()), chksum, - "wrong md5sum for %s" % tarinfo.name) + self.assertEqual(sha256sum(f.read()), chksum, + "wrong sha256sum for %s" % tarinfo.name) kwargs["mtime"] = 0o7606136617 kwargs["uid"] = 1000 @@ -815,11 +819,11 @@ class MemberReadTest(ReadTest, unittest.TestCase): def test_find_regtype(self): tarinfo = self.tar.getmember("ustar/regtype") - self._test_member(tarinfo, size=7011, chksum=md5_regtype) + self._test_member(tarinfo, size=7011, chksum=sha256_regtype) def test_find_conttype(self): tarinfo = self.tar.getmember("ustar/conttype") - self._test_member(tarinfo, size=7011, chksum=md5_regtype) + self._test_member(tarinfo, size=7011, chksum=sha256_regtype) def test_find_dirtype(self): tarinfo = self.tar.getmember("ustar/dirtype") @@ -851,28 +855,28 @@ class MemberReadTest(ReadTest, unittest.TestCase): def test_find_sparse(self): tarinfo = self.tar.getmember("ustar/sparse") - self._test_member(tarinfo, size=86016, chksum=md5_sparse) + self._test_member(tarinfo, size=86016, chksum=sha256_sparse) def test_find_gnusparse(self): tarinfo = self.tar.getmember("gnu/sparse") - self._test_member(tarinfo, size=86016, chksum=md5_sparse) + self._test_member(tarinfo, size=86016, chksum=sha256_sparse) def test_find_gnusparse_00(self): tarinfo = self.tar.getmember("gnu/sparse-0.0") - self._test_member(tarinfo, size=86016, chksum=md5_sparse) + self._test_member(tarinfo, size=86016, chksum=sha256_sparse) def test_find_gnusparse_01(self): tarinfo = self.tar.getmember("gnu/sparse-0.1") - self._test_member(tarinfo, size=86016, chksum=md5_sparse) + self._test_member(tarinfo, size=86016, chksum=sha256_sparse) def test_find_gnusparse_10(self): tarinfo = self.tar.getmember("gnu/sparse-1.0") - self._test_member(tarinfo, size=86016, chksum=md5_sparse) + self._test_member(tarinfo, size=86016, chksum=sha256_sparse) def test_find_umlauts(self): tarinfo = self.tar.getmember("ustar/umlauts-" "\xc4\xd6\xdc\xe4\xf6\xfc\xdf") - self._test_member(tarinfo, size=7011, chksum=md5_regtype) + self._test_member(tarinfo, size=7011, chksum=sha256_regtype) def test_find_ustar_longname(self): name = "ustar/" + "12345/" * 39 + "1234567/longname" @@ -880,7 +884,7 @@ class MemberReadTest(ReadTest, unittest.TestCase): def test_find_regtype_oldv7(self): tarinfo = self.tar.getmember("misc/regtype-old-v7") - self._test_member(tarinfo, size=7011, chksum=md5_regtype) + self._test_member(tarinfo, size=7011, chksum=sha256_regtype) def test_find_pax_umlauts(self): self.tar.close() @@ -888,7 +892,7 @@ class MemberReadTest(ReadTest, unittest.TestCase): encoding="iso8859-1") tarinfo = self.tar.getmember("pax/umlauts-" "\xc4\xd6\xdc\xe4\xf6\xfc\xdf") - self._test_member(tarinfo, size=7011, chksum=md5_regtype) + self._test_member(tarinfo, size=7011, chksum=sha256_regtype) class LongnameTest: @@ -950,8 +954,8 @@ class GNUReadTest(LongnameTest, ReadTest, unittest.TestCase): filename = os.path.join(TEMPDIR, name) with open(filename, "rb") as fobj: data = fobj.read() - self.assertEqual(md5sum(data), md5_sparse, - "wrong md5sum for %s" % name) + self.assertEqual(sha256sum(data), sha256_sparse, + "wrong sha256sum for %s" % name) if self._fs_supports_holes(): s = os.stat(filename) @@ -2431,7 +2435,7 @@ class LinkEmulationTest(ReadTest, unittest.TestCase): self.tar.extract(name, TEMPDIR) with open(os.path.join(TEMPDIR, name), "rb") as f: data = f.read() - self.assertEqual(md5sum(data), md5_regtype) + self.assertEqual(sha256sum(data), sha256_regtype) # See issues #1578269, #8879, and #17689 for some history on these skips @unittest.skipIf(hasattr(os.path, "islink"), diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index ef0091c..895f97c 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -325,6 +325,7 @@ class ProxyAuthTests(unittest.TestCase): PASSWD = "test123" REALM = "TestRealm" + @support.requires_hashdigest("md5") def setUp(self): super(ProxyAuthTests, self).setUp() # Ignore proxy bypass settings in the environment. -- 2.25.4 From 031e1a8cdbd297a350764ec199812370890d6b6d Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 30 Sep 2019 09:10:38 +0200 Subject: [PATCH 06/47] bpo-38270: More fixes for strict crypto policy (GH-16418) (#16437) test_hmac and test_hashlib test built-in hashing implementations and OpenSSL-based hashing implementations. Add more checks to skip OpenSSL implementations when a strict crypto policy is active. Use EVP_DigestInit_ex() instead of EVP_DigestInit() to initialize the EVP context. The EVP_DigestInit() function clears alls flags and breaks usedforsecurity flag again. Signed-off-by: Christian Heimes https://bugs.python.org/issue38270. (cherry picked from commit 90558158093c0ad893102158fd3c2dd9f864e82e) Co-authored-by: Christian Heimes --- Lib/test/support/__init__.py | 19 ++++++++++++++++--- Lib/test/test_hashlib.py | 3 +++ Lib/test/test_hmac.py | 12 ++++++------ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index a438199..3342218 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -71,6 +71,11 @@ try: except ImportError: resource = None +try: + import _hashlib +except ImportError: + _hashlib = None + __all__ = [ # globals "PIPE_MAX_SIZE", "verbose", "max_memuse", "use_resources", "failfast", @@ -88,7 +93,8 @@ __all__ = [ "create_empty_file", "can_symlink", "fs_is_case_insensitive", # unittest "is_resource_enabled", "requires", "requires_freebsd_version", - "requires_linux_version", "requires_mac_ver", "check_syntax_error", + "requires_linux_version", "requires_mac_ver", "requires_hashdigest", + "check_syntax_error", "TransientResource", "time_out", "socket_peer_reset", "ioerror_peer_reset", "transient_internet", "BasicTestRunner", "run_unittest", "run_doctest", "skip_unless_symlink", "requires_gzip", "requires_bz2", "requires_lzma", @@ -628,12 +634,16 @@ def requires_mac_ver(*min_version): return wrapper return decorator -def requires_hashdigest(digestname): +def requires_hashdigest(digestname, openssl=None): """Decorator raising SkipTest if a hashing algorithm is not available The hashing algorithm could be missing or blocked by a strict crypto policy. + If 'openssl' is True, then the decorator checks that OpenSSL provides + the algorithm. Otherwise the check falls back to built-in + implementations. + ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS ValueError: unsupported hash type md4 """ @@ -641,7 +651,10 @@ def requires_hashdigest(digestname): @functools.wraps(func) def wrapper(*args, **kwargs): try: - hashlib.new(digestname) + if openssl and _hashlib is not None: + _hashlib.new(digestname) + else: + hashlib.new(digestname) except ValueError: raise unittest.SkipTest( f"hash digest '{digestname}' is not available." diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 9711856..1b1b4d5 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -8,6 +8,7 @@ import array from binascii import unhexlify +import functools import hashlib import importlib import itertools @@ -21,6 +22,7 @@ import unittest import warnings from test import support from test.support import _4G, bigmemtest, import_fresh_module +from test.support import requires_hashdigest from http.client import HTTPException # Were we compiled --with-pydebug or with #define Py_DEBUG? @@ -117,6 +119,7 @@ class HashLibTestCase(unittest.TestCase): constructors.add(_test_algorithm_via_hashlib_new) _hashlib = self._conditional_import_module('_hashlib') + self._hashlib = _hashlib if _hashlib: # These two algorithms should always be present when this module # is compiled. If not, something was compiled wrong. diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 81c3485..2a5a0d3 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -19,7 +19,7 @@ def ignore_warning(func): class TestVectorsTestCase(unittest.TestCase): - @requires_hashdigest('md5') + @requires_hashdigest('md5', openssl=True) def test_md5_vectors(self): # Test the HMAC module against test vectors from the RFC. @@ -66,7 +66,7 @@ class TestVectorsTestCase(unittest.TestCase): b"and Larger Than One Block-Size Data"), "6f630fad67cda0ee1fb1f562db3aa53e") - @requires_hashdigest('sha1') + @requires_hashdigest('sha1', openssl=True) def test_sha_vectors(self): def shatest(key, data, digest): h = hmac.HMAC(key, data, digestmod=hashlib.sha1) @@ -234,19 +234,19 @@ class TestVectorsTestCase(unittest.TestCase): '134676fb6de0446065c97440fa8c6a58', }) - @requires_hashdigest('sha224') + @requires_hashdigest('sha224', openssl=True) def test_sha224_rfc4231(self): self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64) - @requires_hashdigest('sha256') + @requires_hashdigest('sha256', openssl=True) def test_sha256_rfc4231(self): self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64) - @requires_hashdigest('sha384') + @requires_hashdigest('sha384', openssl=True) def test_sha384_rfc4231(self): self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128) - @requires_hashdigest('sha512') + @requires_hashdigest('sha512', openssl=True) def test_sha512_rfc4231(self): self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128) -- 2.25.4 From 3a665eca6fb236aca440209c097627dd1aa94ea6 Mon Sep 17 00:00:00 2001 From: stratakis Date: Tue, 3 Dec 2019 16:35:54 +0100 Subject: [PATCH 07/47] bpo-38270: Fix indentation of test_hmac assertions (GH-17446) Since https://github.com/python/cpython/commit/c64a1a61e6fc542cada40eb069a239317e1af36e two assertions were indented and thus ignored when running test_hmac. This PR fixes it. As the change is quite trivial I didn't add a NEWS entry. https://bugs.python.org/issue38270 --- Lib/test/test_hmac.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 2a5a0d3..338e021 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -329,7 +329,7 @@ class ConstructorTestCase(unittest.TestCase): digestmod="sha256") except Exception: self.fail("Constructor call with bytearray arguments raised exception.") - self.assertEqual(h.hexdigest(), self.expected) + self.assertEqual(h.hexdigest(), self.expected) @requires_hashdigest('sha256') def test_with_memoryview_msg(self): @@ -337,7 +337,7 @@ class ConstructorTestCase(unittest.TestCase): h = hmac.HMAC(b"key", memoryview(b"hash this!"), digestmod="sha256") except Exception: self.fail("Constructor call with memoryview msg raised exception.") - self.assertEqual(h.hexdigest(), self.expected) + self.assertEqual(h.hexdigest(), self.expected) @requires_hashdigest('sha256') def test_withmodule(self): -- 2.25.4 From 157bba013612bf2e7ec3d8018a38543cfcd62e17 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 25 Jul 2019 16:19:52 +0200 Subject: [PATCH 08/47] Expose OpenSSL FIPS_mode() as hashlib.get_fips_mode() --- Lib/hashlib.py | 5 +++++ Modules/_hashopenssl.c | 36 +++++++++++++++++++++++++++++++++ Modules/clinic/_hashopenssl.c.h | 25 ++++++++++++++++++++++- 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 98d2d79..ae17c58 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -236,6 +236,11 @@ try: except ImportError: pass +try: + from _hashlib import get_fips_mode +except ImportError: + pass + for __func_name in __always_supported: # try them all, some may not work due to the OpenSSL diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 84edd72..4876a7f 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -25,6 +25,9 @@ #include #include "openssl/err.h" +/* Expose FIPS_mode */ +#include + #include "clinic/_hashopenssl.c.h" /*[clinic input] module _hashlib @@ -987,6 +990,38 @@ GEN_CONSTRUCTOR(sha256) GEN_CONSTRUCTOR(sha384) GEN_CONSTRUCTOR(sha512) +/*[clinic input] +_hashlib.get_fips_mode + +Determine the OpenSSL FIPS mode of operation. + +Effectively any non-zero return value indicates FIPS mode; +values other than 1 may have additional significance. + +See OpenSSL documentation for the FIPS_mode() function for details. +[clinic start generated code]*/ + +static PyObject * +_hashlib_get_fips_mode_impl(PyObject *module) +/*[clinic end generated code: output=ad8a7793310d3f98 input=f42a2135df2a5e11]*/ +{ + int result = FIPS_mode(); + if (result == 0) { + // "If the library was built without support of the FIPS Object Module, + // then the function will return 0 with an error code of + // CRYPTO_R_FIPS_MODE_NOT_SUPPORTED (0x0f06d065)." + // But 0 is also a valid result value. + + unsigned long errcode = ERR_peek_last_error(); + if (errcode) { + _setException(PyExc_ValueError); + return NULL; + } + } + return PyLong_FromLong(result); +} + + /* List of functions exported by this module */ static struct PyMethodDef EVP_functions[] = { @@ -996,6 +1031,7 @@ static struct PyMethodDef EVP_functions[] = { pbkdf2_hmac__doc__}, #endif _HASHLIB_SCRYPT_METHODDEF + _HASHLIB_GET_FIPS_MODE_METHODDEF CONSTRUCTOR_METH_DEF(md5), CONSTRUCTOR_METH_DEF(sha1), CONSTRUCTOR_METH_DEF(sha224), diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 0445352..8828e27 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -54,7 +54,30 @@ exit: #endif /* (OPENSSL_VERSION_NUMBER > 0x10100000L && !defined(OPENSSL_NO_SCRYPT) && !defined(LIBRESSL_VERSION_NUMBER)) */ +PyDoc_STRVAR(_hashlib_get_fips_mode__doc__, +"get_fips_mode($module, /)\n" +"--\n" +"\n" +"Determine the OpenSSL FIPS mode of operation.\n" +"\n" +"Effectively any non-zero return value indicates FIPS mode;\n" +"values other than 1 may have additional significance.\n" +"\n" +"See OpenSSL documentation for the FIPS_mode() function for details."); + +#define _HASHLIB_GET_FIPS_MODE_METHODDEF \ + {"get_fips_mode", (PyCFunction)_hashlib_get_fips_mode, METH_NOARGS, _hashlib_get_fips_mode__doc__}, + +static PyObject * +_hashlib_get_fips_mode_impl(PyObject *module); + +static PyObject * +_hashlib_get_fips_mode(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _hashlib_get_fips_mode_impl(module); +} + #ifndef _HASHLIB_SCRYPT_METHODDEF #define _HASHLIB_SCRYPT_METHODDEF #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=118cd7036fa0fb52 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7d683c930bbb7c36 input=a9049054013a1b77]*/ -- 2.25.4 From 4a6d42d03fae55a61f0642b069710c6f0479f507 Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Thu, 25 Jul 2019 17:04:06 +0200 Subject: [PATCH 09/47] Use python's fall backs for the crypto it implements only if we are not in FIPS mode --- Lib/hashlib.py | 209 +++++++++++++++------------------------ Lib/test/test_hashlib.py | 1 + 2 files changed, 81 insertions(+), 129 deletions(-) diff --git a/Lib/hashlib.py b/Lib/hashlib.py index ae17c58..7db1e02 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -67,56 +67,64 @@ algorithms_available = set(__always_supported) __all__ = __always_supported + ('new', 'algorithms_guaranteed', 'algorithms_available', 'pbkdf2_hmac') - -__builtin_constructor_cache = {} - -def __get_builtin_constructor(name): - cache = __builtin_constructor_cache - constructor = cache.get(name) - if constructor is not None: - return constructor - try: - if name in ('SHA1', 'sha1'): - import _sha1 - cache['SHA1'] = cache['sha1'] = _sha1.sha1 - elif name in ('MD5', 'md5'): - import _md5 - cache['MD5'] = cache['md5'] = _md5.md5 - elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): - import _sha256 - cache['SHA224'] = cache['sha224'] = _sha256.sha224 - cache['SHA256'] = cache['sha256'] = _sha256.sha256 - elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): - import _sha512 - cache['SHA384'] = cache['sha384'] = _sha512.sha384 - cache['SHA512'] = cache['sha512'] = _sha512.sha512 - elif name in ('blake2b', 'blake2s'): - import _blake2 - cache['blake2b'] = _blake2.blake2b - cache['blake2s'] = _blake2.blake2s - elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', - 'shake_128', 'shake_256'}: - import _sha3 - cache['sha3_224'] = _sha3.sha3_224 - cache['sha3_256'] = _sha3.sha3_256 - cache['sha3_384'] = _sha3.sha3_384 - cache['sha3_512'] = _sha3.sha3_512 - cache['shake_128'] = _sha3.shake_128 - cache['shake_256'] = _sha3.shake_256 - except ImportError: - pass # no extension module, this hash is unsupported. - - constructor = cache.get(name) - if constructor is not None: - return constructor - - raise ValueError('unsupported hash type ' + name) +try: + from _hashlib import get_fips_mode +except ImportError: + def get_fips_mode(): + return 0 + + +if not get_fips_mode(): + __builtin_constructor_cache = {} + + def __get_builtin_constructor(name): + cache = __builtin_constructor_cache + constructor = cache.get(name) + if constructor is not None: + return constructor + try: + if name in ('SHA1', 'sha1'): + import _sha1 + cache['SHA1'] = cache['sha1'] = _sha1.sha1 + elif name in ('MD5', 'md5'): + import _md5 + cache['MD5'] = cache['md5'] = _md5.md5 + elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): + import _sha256 + cache['SHA224'] = cache['sha224'] = _sha256.sha224 + cache['SHA256'] = cache['sha256'] = _sha256.sha256 + elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): + import _sha512 + cache['SHA384'] = cache['sha384'] = _sha512.sha384 + cache['SHA512'] = cache['sha512'] = _sha512.sha512 + elif name in ('blake2b', 'blake2s'): + import _blake2 + cache['blake2b'] = _blake2.blake2b + cache['blake2s'] = _blake2.blake2s + elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', + 'shake_128', 'shake_256'}: + import _sha3 + cache['sha3_224'] = _sha3.sha3_224 + cache['sha3_256'] = _sha3.sha3_256 + cache['sha3_384'] = _sha3.sha3_384 + cache['sha3_512'] = _sha3.sha3_512 + cache['shake_128'] = _sha3.shake_128 + cache['shake_256'] = _sha3.shake_256 + except ImportError: + pass # no extension module, this hash is unsupported. + + constructor = cache.get(name) + if constructor is not None: + return constructor + + raise ValueError('unsupported hash type ' + name) def __get_openssl_constructor(name): - if name in {'blake2b', 'blake2s'}: - # Prefer our blake2 implementation. - return __get_builtin_constructor(name) + if not get_fips_mode(): + if name in {'blake2b', 'blake2s'}: + # Prefer our blake2 implementation. + return __get_builtin_constructor(name) try: f = getattr(_hashlib, 'openssl_' + name) # Allow the C module to raise ValueError. The function will be @@ -125,27 +133,30 @@ def __get_openssl_constructor(name): # Use the C function directly (very fast) return f except (AttributeError, ValueError): + if get_fips_mode(): + raise return __get_builtin_constructor(name) - -def __py_new(name, data=b'', **kwargs): - """new(name, data=b'', **kwargs) - Return a new hashing object using the - named algorithm; optionally initialized with data (which must be - a bytes-like object). - """ - return __get_builtin_constructor(name)(data, **kwargs) +if not get_fips_mode(): + def __py_new(name, data=b'', **kwargs): + """new(name, data=b'', **kwargs) - Return a new hashing object using the + named algorithm; optionally initialized with data (which must be + a bytes-like object). + """ + return __get_builtin_constructor(name)(data, **kwargs) def __hash_new(name, data=b'', **kwargs): """new(name, data=b'') - Return a new hashing object using the named algorithm; optionally initialized with data (which must be a bytes-like object). """ - if name in {'blake2b', 'blake2s'}: - # Prefer our blake2 implementation. - # OpenSSL 1.1.0 comes with a limited implementation of blake2b/s. - # It does neither support keyed blake2 nor advanced features like - # salt, personal, tree hashing or SSE. - return __get_builtin_constructor(name)(data, **kwargs) + if not get_fips_mode(): + if name in {'blake2b', 'blake2s'}: + # Prefer our blake2 implementation. + # OpenSSL 1.1.0 comes with a limited implementation of blake2b/s. + # It does neither support keyed blake2 nor advanced features like + # salt, personal, tree hashing or SSE. + return __get_builtin_constructor(name)(data, **kwargs) try: return _hashlib.new(name, data) except ValueError: @@ -153,6 +164,8 @@ def __hash_new(name, data=b'', **kwargs): # hash, try using our builtin implementations. # This allows for SHA224/256 and SHA384/512 support even though # the OpenSSL library prior to 0.9.8 doesn't provide them. + if get_fips_mode(): + raise return __get_builtin_constructor(name)(data) @@ -163,72 +176,14 @@ try: algorithms_available = algorithms_available.union( _hashlib.openssl_md_meth_names) except ImportError: + if get_fips_mode(): + raise new = __py_new __get_hash = __get_builtin_constructor -try: - # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA - from _hashlib import pbkdf2_hmac -except ImportError: - _trans_5C = bytes((x ^ 0x5C) for x in range(256)) - _trans_36 = bytes((x ^ 0x36) for x in range(256)) - - def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None): - """Password based key derivation function 2 (PKCS #5 v2.0) - This Python implementations based on the hmac module about as fast - as OpenSSL's PKCS5_PBKDF2_HMAC for short passwords and much faster - for long passwords. - """ - if not isinstance(hash_name, str): - raise TypeError(hash_name) - - if not isinstance(password, (bytes, bytearray)): - password = bytes(memoryview(password)) - if not isinstance(salt, (bytes, bytearray)): - salt = bytes(memoryview(salt)) - - # Fast inline HMAC implementation - inner = new(hash_name) - outer = new(hash_name) - blocksize = getattr(inner, 'block_size', 64) - if len(password) > blocksize: - password = new(hash_name, password).digest() - password = password + b'\x00' * (blocksize - len(password)) - inner.update(password.translate(_trans_36)) - outer.update(password.translate(_trans_5C)) - - def prf(msg, inner=inner, outer=outer): - # PBKDF2_HMAC uses the password as key. We can re-use the same - # digest objects and just update copies to skip initialization. - icpy = inner.copy() - ocpy = outer.copy() - icpy.update(msg) - ocpy.update(icpy.digest()) - return ocpy.digest() - - if iterations < 1: - raise ValueError(iterations) - if dklen is None: - dklen = outer.digest_size - if dklen < 1: - raise ValueError(dklen) - - dkey = b'' - loop = 1 - from_bytes = int.from_bytes - while len(dkey) < dklen: - prev = prf(salt + loop.to_bytes(4, 'big')) - # endianess doesn't matter here as long to / from use the same - rkey = int.from_bytes(prev, 'big') - for i in range(iterations - 1): - prev = prf(prev) - # rkey = rkey ^ prev - rkey ^= from_bytes(prev, 'big') - loop += 1 - dkey += rkey.to_bytes(inner.digest_size, 'big') - - return dkey[:dklen] +# OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA +from _hashlib import pbkdf2_hmac try: # OpenSSL's scrypt requires OpenSSL 1.1+ @@ -236,12 +191,6 @@ try: except ImportError: pass -try: - from _hashlib import get_fips_mode -except ImportError: - pass - - for __func_name in __always_supported: # try them all, some may not work due to the OpenSSL # version not supporting that algorithm. @@ -254,4 +203,6 @@ for __func_name in __always_supported: # Cleanup locals() del __always_supported, __func_name, __get_hash -del __py_new, __hash_new, __get_openssl_constructor +del __hash_new, __get_openssl_constructor +if not get_fips_mode(): + del __py_new diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 1b1b4d5..4e6b5b6 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -933,6 +933,7 @@ class KDFTests(unittest.TestCase): iterations=1, dklen=None) self.assertEqual(out, self.pbkdf2_results['sha1'][0][0]) + @unittest.skip("The python implementation of pbkdf2_hmac has been removed") def test_pbkdf2_hmac_py(self): self._test_pbkdf2_hmac(py_hashlib.pbkdf2_hmac) -- 2.25.4 From 2ae118ab024f6c0f0969eca5576125f6cee27dff Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 25 Jul 2019 17:19:06 +0200 Subject: [PATCH 10/47] Disable Python's hash implementations in FIPS mode, forcing OpenSSL --- Include/_hashopenssl.h | 66 ++++++++++++++++++++++++++++++++++ Modules/_blake2/blake2b_impl.c | 5 +++ Modules/_blake2/blake2module.c | 3 ++ Modules/_blake2/blake2s_impl.c | 5 +++ Modules/_hashopenssl.c | 36 +------------------ Modules/_sha3/sha3module.c | 5 +++ setup.py | 31 ++++++++-------- 7 files changed, 101 insertions(+), 50 deletions(-) create mode 100644 Include/_hashopenssl.h diff --git a/Include/_hashopenssl.h b/Include/_hashopenssl.h new file mode 100644 index 0000000..a726c0d --- /dev/null +++ b/Include/_hashopenssl.h @@ -0,0 +1,66 @@ +#ifndef Py_HASHOPENSSL_H +#define Py_HASHOPENSSL_H + +#include "Python.h" +#include +#include + +/* LCOV_EXCL_START */ +static PyObject * +_setException(PyObject *exc) +{ + unsigned long errcode; + const char *lib, *func, *reason; + + errcode = ERR_peek_last_error(); + if (!errcode) { + PyErr_SetString(exc, "unknown reasons"); + return NULL; + } + ERR_clear_error(); + + lib = ERR_lib_error_string(errcode); + func = ERR_func_error_string(errcode); + reason = ERR_reason_error_string(errcode); + + if (lib && func) { + PyErr_Format(exc, "[%s: %s] %s", lib, func, reason); + } + else if (lib) { + PyErr_Format(exc, "[%s] %s", lib, reason); + } + else { + PyErr_SetString(exc, reason); + } + return NULL; +} +/* LCOV_EXCL_STOP */ + + +__attribute__((__unused__)) +static int +_Py_hashlib_fips_error(char *name) { + int result = FIPS_mode(); + if (result == 0) { + // "If the library was built without support of the FIPS Object Module, + // then the function will return 0 with an error code of + // CRYPTO_R_FIPS_MODE_NOT_SUPPORTED (0x0f06d065)." + // But 0 is also a valid result value. + + unsigned long errcode = ERR_peek_last_error(); + if (errcode) { + _setException(PyExc_ValueError); + return 1; + } + return 0; + } + PyErr_Format(PyExc_ValueError, "%s is not available in FIPS mode", + name); + return 1; +} + +#define FAIL_RETURN_IN_FIPS_MODE(name) do { \ + if (_Py_hashlib_fips_error(name)) return NULL; \ +} while (0) + +#endif // !Py_HASHOPENSSL_H diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c index 418c018..341e67a 100644 --- a/Modules/_blake2/blake2b_impl.c +++ b/Modules/_blake2/blake2b_impl.c @@ -14,6 +14,7 @@ */ #include "Python.h" +#include "_hashopenssl.h" #include "pystrhex.h" #ifdef WITH_THREAD #include "pythread.h" @@ -103,6 +104,8 @@ py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, unsigned long leaf_size = 0; unsigned long long node_offset = 0; + FAIL_RETURN_IN_FIPS_MODE("_blake2"); + self = new_BLAKE2bObject(type); if (self == NULL) { goto error; @@ -293,6 +296,8 @@ _blake2_blake2b_update(BLAKE2bObject *self, PyObject *data) { Py_buffer buf; + FAIL_RETURN_IN_FIPS_MODE("_blake2"); + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); #ifdef WITH_THREAD diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c index e2a3d42..817b716 100644 --- a/Modules/_blake2/blake2module.c +++ b/Modules/_blake2/blake2module.c @@ -9,6 +9,7 @@ */ #include "Python.h" +#include "_hashopenssl.h" #include "impl/blake2.h" @@ -57,6 +58,8 @@ PyInit__blake2(void) PyObject *m; PyObject *d; + FAIL_RETURN_IN_FIPS_MODE("blake2"); + m = PyModule_Create(&blake2_module); if (m == NULL) return NULL; diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c index 24e529b..0dfe058 100644 --- a/Modules/_blake2/blake2s_impl.c +++ b/Modules/_blake2/blake2s_impl.c @@ -14,6 +14,7 @@ */ #include "Python.h" +#include "_hashopenssl.h" #include "pystrhex.h" #ifdef WITH_THREAD #include "pythread.h" @@ -103,6 +104,8 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, unsigned long leaf_size = 0; unsigned long long node_offset = 0; + FAIL_RETURN_IN_FIPS_MODE("_blake2"); + self = new_BLAKE2sObject(type); if (self == NULL) { goto error; @@ -293,6 +296,8 @@ _blake2_blake2s_update(BLAKE2sObject *self, PyObject *data) { Py_buffer buf; + FAIL_RETURN_IN_FIPS_MODE("_blake2"); + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); #ifdef WITH_THREAD diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 4876a7f..02cf608 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -17,16 +17,13 @@ #include "structmember.h" #include "hashlib.h" #include "pystrhex.h" +#include "_hashopenssl.h" /* EVP is the preferred interface to hashing in OpenSSL */ #include /* We use the object interface to discover what hashes OpenSSL supports. */ #include -#include "openssl/err.h" - -/* Expose FIPS_mode */ -#include #include "clinic/_hashopenssl.c.h" /*[clinic input] @@ -77,37 +74,6 @@ DEFINE_CONSTS_FOR_NEW(sha384) DEFINE_CONSTS_FOR_NEW(sha512) -/* LCOV_EXCL_START */ -static PyObject * -_setException(PyObject *exc) -{ - unsigned long errcode; - const char *lib, *func, *reason; - - errcode = ERR_peek_last_error(); - if (!errcode) { - PyErr_SetString(exc, "unknown reasons"); - return NULL; - } - ERR_clear_error(); - - lib = ERR_lib_error_string(errcode); - func = ERR_func_error_string(errcode); - reason = ERR_reason_error_string(errcode); - - if (lib && func) { - PyErr_Format(exc, "[%s: %s] %s", lib, func, reason); - } - else if (lib) { - PyErr_Format(exc, "[%s] %s", lib, reason); - } - else { - PyErr_SetString(exc, reason); - } - return NULL; -} -/* LCOV_EXCL_STOP */ - static EVPobject * newEVPobject(PyObject *name) { diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c index 2c2b2db..624f7f2 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/_sha3/sha3module.c @@ -18,6 +18,7 @@ #include "Python.h" #include "pystrhex.h" #include "../hashlib.h" +#include "_hashopenssl.h" /* ************************************************************************** * SHA-3 (Keccak) and SHAKE @@ -162,6 +163,7 @@ static PyTypeObject SHAKE256type; static SHA3object * newSHA3object(PyTypeObject *type) { + FAIL_RETURN_IN_FIPS_MODE("_sha3"); SHA3object *newobj; newobj = (SHA3object *)PyObject_New(SHA3object, type); if (newobj == NULL) { @@ -177,6 +179,7 @@ newSHA3object(PyTypeObject *type) static PyObject * py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { + FAIL_RETURN_IN_FIPS_MODE("_sha3"); SHA3object *self = NULL; Py_buffer buf = {NULL, NULL}; HashReturn res; @@ -724,6 +727,8 @@ PyInit__sha3(void) { PyObject *m = NULL; + FAIL_RETURN_IN_FIPS_MODE("_sha3"); + if ((m = PyModule_Create(&_SHA3module)) == NULL) { return NULL; } diff --git a/setup.py b/setup.py index 73f7298..52ee704 100644 --- a/setup.py +++ b/setup.py @@ -901,31 +901,30 @@ class PyBuildExt(build_ext): have_usable_openssl = (have_any_openssl and openssl_ver >= min_openssl_ver) + if not have_usable_openssl: + raise ValueError('Cannot build for RHEL without OpenSSL') + + ssl_args = { + 'include_dirs': ssl_incs, + 'library_dirs': ssl_libs, + 'libraries': ['ssl', 'crypto'], + } + if have_any_openssl: if have_usable_openssl: # The _hashlib module wraps optimized implementations # of hash functions from the OpenSSL library. exts.append( Extension('_hashlib', ['_hashopenssl.c'], depends = ['hashlib.h'], - include_dirs = ssl_incs, - library_dirs = ssl_libs, - libraries = ['ssl', 'crypto']) ) + **ssl_args)) else: print("warning: openssl 0x%08x is too old for _hashlib" % openssl_ver) missing.append('_hashlib') - # We always compile these even when OpenSSL is available (issue #14693). - # It's harmless and the object code is tiny (40-50 KB per module, - # only loaded when actually used). - exts.append( Extension('_sha256', ['sha256module.c'], - depends=['hashlib.h']) ) - exts.append( Extension('_sha512', ['sha512module.c'], - depends=['hashlib.h']) ) - exts.append( Extension('_md5', ['md5module.c'], - depends=['hashlib.h']) ) - exts.append( Extension('_sha1', ['sha1module.c'], - depends=['hashlib.h']) ) + # RHEL: Always force OpenSSL for md5, sha1, sha256, sha512; + # don't build Python's implementations. + # sha3 and blake2 have extra functionality, so do build those: blake2_deps = glob(os.path.join(os.getcwd(), srcdir, 'Modules/_blake2/impl/*')) @@ -944,6 +943,7 @@ class PyBuildExt(build_ext): '_blake2/blake2b_impl.c', '_blake2/blake2s_impl.c'], define_macros=blake2_macros, + **ssl_args, depends=blake2_deps) ) sha3_deps = glob(os.path.join(os.getcwd(), srcdir, @@ -951,7 +951,8 @@ class PyBuildExt(build_ext): sha3_deps.append('hashlib.h') exts.append( Extension('_sha3', ['_sha3/sha3module.c'], - depends=sha3_deps)) + **ssl_args, + depends=sha3_deps + ['hashlib.h'])) # Modules that provide persistent dictionary-like semantics. You will # probably want to arrange for at least one of them to be available on -- 2.25.4 From c3a0e3ecf72d34178901d2f83de15c3c21fe6d01 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 25 Jul 2019 17:35:27 +0200 Subject: [PATCH 11/47] Expose all hashes available to OpenSSL, using a list --- Modules/_hashopenssl.c | 44 ++++++++++++++----------------------- Modules/_hashopenssl_list.h | 21 ++++++++++++++++++ 2 files changed, 37 insertions(+), 28 deletions(-) create mode 100644 Modules/_hashopenssl_list.h diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 02cf608..7dfd708 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -66,12 +66,9 @@ static PyTypeObject EVPtype; static PyObject *CONST_ ## Name ## _name_obj = NULL; \ static EVP_MD_CTX *CONST_new_ ## Name ## _ctx_p = NULL; -DEFINE_CONSTS_FOR_NEW(md5) -DEFINE_CONSTS_FOR_NEW(sha1) -DEFINE_CONSTS_FOR_NEW(sha224) -DEFINE_CONSTS_FOR_NEW(sha256) -DEFINE_CONSTS_FOR_NEW(sha384) -DEFINE_CONSTS_FOR_NEW(sha512) +#define _HASH(py_name, openssl_name) DEFINE_CONSTS_FOR_NEW(py_name) +#include "_hashopenssl_list.h" +#undef _HASH static EVPobject * @@ -896,7 +893,7 @@ generate_hash_name_list(void) * The first call will lazy-initialize, which reports an exception * if initialization fails. */ -#define GEN_CONSTRUCTOR(NAME) \ +#define GEN_CONSTRUCTOR(NAME, SSL_NAME) \ static PyObject * \ EVP_new_ ## NAME (PyObject *self, PyObject *args) \ { \ @@ -910,8 +907,8 @@ generate_hash_name_list(void) \ if (CONST_new_ ## NAME ## _ctx_p == NULL) { \ EVP_MD_CTX *ctx_p = EVP_MD_CTX_new(); \ - if (!EVP_get_digestbyname(#NAME) || \ - !EVP_DigestInit(ctx_p, EVP_get_digestbyname(#NAME))) { \ + if (!EVP_get_digestbyname(SSL_NAME) || \ + !EVP_DigestInit(ctx_p, EVP_get_digestbyname(SSL_NAME))) { \ _setException(PyExc_ValueError); \ EVP_MD_CTX_free(ctx_p); \ return NULL; \ @@ -939,7 +936,7 @@ generate_hash_name_list(void) {"openssl_" #NAME, (PyCFunction)EVP_new_ ## NAME, METH_VARARGS, \ PyDoc_STR("Returns a " #NAME \ " hash object; optionally initialized with a string") \ - } + }, /* used in the init function to setup a constructor: initialize OpenSSL constructor constants if they haven't been initialized already. */ @@ -949,12 +946,9 @@ generate_hash_name_list(void) } \ } while (0); -GEN_CONSTRUCTOR(md5) -GEN_CONSTRUCTOR(sha1) -GEN_CONSTRUCTOR(sha224) -GEN_CONSTRUCTOR(sha256) -GEN_CONSTRUCTOR(sha384) -GEN_CONSTRUCTOR(sha512) +#define _HASH(py_name, openssl_name) GEN_CONSTRUCTOR(py_name, openssl_name) +#include "_hashopenssl_list.h" +#undef _HASH /*[clinic input] _hashlib.get_fips_mode @@ -998,12 +992,9 @@ static struct PyMethodDef EVP_functions[] = { #endif _HASHLIB_SCRYPT_METHODDEF _HASHLIB_GET_FIPS_MODE_METHODDEF - CONSTRUCTOR_METH_DEF(md5), - CONSTRUCTOR_METH_DEF(sha1), - CONSTRUCTOR_METH_DEF(sha224), - CONSTRUCTOR_METH_DEF(sha256), - CONSTRUCTOR_METH_DEF(sha384), - CONSTRUCTOR_METH_DEF(sha512), +#define _HASH(py_name, openssl_name) CONSTRUCTOR_METH_DEF(py_name) +#include "_hashopenssl_list.h" +#undef _HASH {NULL, NULL} /* Sentinel */ }; @@ -1061,11 +1052,8 @@ PyInit__hashlib(void) PyModule_AddObject(m, "HASH", (PyObject *)&EVPtype); /* these constants are used by the convenience constructors */ - INIT_CONSTRUCTOR_CONSTANTS(md5); - INIT_CONSTRUCTOR_CONSTANTS(sha1); - INIT_CONSTRUCTOR_CONSTANTS(sha224); - INIT_CONSTRUCTOR_CONSTANTS(sha256); - INIT_CONSTRUCTOR_CONSTANTS(sha384); - INIT_CONSTRUCTOR_CONSTANTS(sha512); +#define _HASH(py_name, openssl_name) INIT_CONSTRUCTOR_CONSTANTS(py_name) +#include "_hashopenssl_list.h" +#undef _HASH return m; } diff --git a/Modules/_hashopenssl_list.h b/Modules/_hashopenssl_list.h new file mode 100644 index 0000000..3c11b2e --- /dev/null +++ b/Modules/_hashopenssl_list.h @@ -0,0 +1,21 @@ +/* Call the _HASH macro with all the hashes exported by OpenSSL, + * at compile time. + * + * This file is meant to be included multiple times, with different values of + * _HASH. + */ + +_HASH(md5, "md5") +_HASH(sha1, "sha1") +_HASH(sha224, "sha224") +_HASH(sha256, "sha256") +_HASH(sha384, "sha384") +_HASH(sha512, "sha512") +_HASH(blake2b, "blake2b512") +_HASH(blake2s, "blake2s256") +_HASH(sha3_224, "sha3-224") +_HASH(sha3_256, "sha3-256") +_HASH(sha3_384, "sha3-384") +_HASH(sha3_512, "sha3-512") +_HASH(shake_128, "shake128") +_HASH(shake_256, "shake256") -- 2.25.4 From d33327e4272ba64f81cc460129d35c7986c4a84d Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 25 Jul 2019 18:13:45 +0200 Subject: [PATCH 12/47] Fix tests --- Lib/hashlib.py | 5 +++- Lib/test/test_hashlib.py | 58 +++++++++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 7db1e02..2def0a3 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -122,7 +122,10 @@ if not get_fips_mode(): def __get_openssl_constructor(name): if not get_fips_mode(): - if name in {'blake2b', 'blake2s'}: + if name in { + 'blake2b', 'blake2s', 'shake_256', 'shake_128', + #'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', + }: # Prefer our blake2 implementation. return __get_builtin_constructor(name) try: diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 4e6b5b6..e57c93b 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -182,7 +182,9 @@ class HashLibTestCase(unittest.TestCase): a = array.array("b", range(10)) for cons in self.hash_constructors: c = cons(a) - if c.name in self.shakes: + if (c.name in self.shakes + and not cons.__name__.startswith('openssl_') + ): c.hexdigest(16) else: c.hexdigest() @@ -229,7 +231,9 @@ class HashLibTestCase(unittest.TestCase): def test_hexdigest(self): for cons in self.hash_constructors: h = cons() - if h.name in self.shakes: + if (h.name in self.shakes + and not cons.__name__.startswith('openssl_') + ): self.assertIsInstance(h.digest(16), bytes) self.assertEqual(hexstr(h.digest(16)), h.hexdigest(16)) else: @@ -243,6 +247,8 @@ class HashLibTestCase(unittest.TestCase): h = cons() if h.name not in self.shakes: continue + if cons.__name__.startswith('openssl_'): + continue for digest in h.digest, h.hexdigest: with self.assertRaises((ValueError, OverflowError)): digest(-10) @@ -272,7 +278,9 @@ class HashLibTestCase(unittest.TestCase): m1.update(bees) m1.update(cees) m1.update(dees) - if m1.name in self.shakes: + if (m1.name in self.shakes + and not cons.__name__.startswith('openssl_') + ): args = (16,) else: args = () @@ -299,15 +307,36 @@ class HashLibTestCase(unittest.TestCase): # 2 is for hashlib.name(...) and hashlib.new(name, ...) self.assertGreaterEqual(len(constructors), 2) for hash_object_constructor in constructors: + if ( + kwargs + and hash_object_constructor.__name__.startswith('openssl_') + ): + return m = hash_object_constructor(data, **kwargs) - computed = m.hexdigest() if not shake else m.hexdigest(length) + if shake: + if hash_object_constructor.__name__.startswith('openssl_'): + if length > m.digest_size: + # OpenSSL doesn't give long digests + return + computed = m.hexdigest()[:length*2] + hexdigest = hexdigest[:length*2] + else: + computed = m.hexdigest(length) + else: + computed = m.hexdigest() self.assertEqual( computed, hexdigest, "Hash algorithm %s constructed using %s returned hexdigest" " %r for %d byte input data that should have hashed to %r." % (name, hash_object_constructor, computed, len(data), hexdigest)) - computed = m.digest() if not shake else m.digest(length) + if shake: + if hash_object_constructor.__name__.startswith('openssl_'): + computed = m.digest()[:length] + else: + computed = m.digest(length) + else: + computed = m.digest() digest = bytes.fromhex(hexdigest) self.assertEqual(computed, digest) if not shake: @@ -347,12 +376,14 @@ class HashLibTestCase(unittest.TestCase): for hash_object_constructor in constructors: m = hash_object_constructor() self.assertEqual(m.block_size, block_size) - self.assertEqual(m.digest_size, digest_size) + if not hash_object_constructor.__name__.startswith('openssl_'): + self.assertEqual(m.digest_size, digest_size) if digest_length: - self.assertEqual(len(m.digest(digest_length)), - digest_length) - self.assertEqual(len(m.hexdigest(digest_length)), - 2*digest_length) + if not hash_object_constructor.__name__.startswith('openssl_'): + self.assertEqual(len(m.digest(digest_length)), + digest_length) + self.assertEqual(len(m.hexdigest(digest_length)), + 2*digest_length) else: self.assertEqual(len(m.digest()), digest_size) self.assertEqual(len(m.hexdigest()), 2*digest_size) @@ -382,9 +413,10 @@ class HashLibTestCase(unittest.TestCase): for hash_object_constructor in constructors: m = hash_object_constructor() self.assertEqual(capacity + rate, 1600) - self.assertEqual(m._capacity_bits, capacity) - self.assertEqual(m._rate_bits, rate) - self.assertEqual(m._suffix, suffix) + if not hash_object_constructor.__name__.startswith('openssl_'): + self.assertEqual(m._capacity_bits, capacity) + self.assertEqual(m._rate_bits, rate) + self.assertEqual(m._suffix, suffix) @requires_sha3 def test_extra_sha3(self): -- 2.25.4 From 8666ef6a6480ffd88f97aa4c7248b30503a92931 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 26 Jul 2019 11:27:57 +0200 Subject: [PATCH 13/47] Change FIPS exceptions from _blake2, _sha3 module init to ImportError --- Include/_hashopenssl.h | 11 +++++------ Modules/_blake2/blake2b_impl.c | 4 ++-- Modules/_blake2/blake2module.c | 2 +- Modules/_blake2/blake2s_impl.c | 4 ++-- Modules/_sha3/sha3module.c | 6 +++--- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/Include/_hashopenssl.h b/Include/_hashopenssl.h index a726c0d..47ed003 100644 --- a/Include/_hashopenssl.h +++ b/Include/_hashopenssl.h @@ -39,7 +39,7 @@ _setException(PyObject *exc) __attribute__((__unused__)) static int -_Py_hashlib_fips_error(char *name) { +_Py_hashlib_fips_error(PyObject *exc, char *name) { int result = FIPS_mode(); if (result == 0) { // "If the library was built without support of the FIPS Object Module, @@ -49,18 +49,17 @@ _Py_hashlib_fips_error(char *name) { unsigned long errcode = ERR_peek_last_error(); if (errcode) { - _setException(PyExc_ValueError); + _setException(exc); return 1; } return 0; } - PyErr_Format(PyExc_ValueError, "%s is not available in FIPS mode", - name); + PyErr_Format(exc, "%s is not available in FIPS mode", name); return 1; } -#define FAIL_RETURN_IN_FIPS_MODE(name) do { \ - if (_Py_hashlib_fips_error(name)) return NULL; \ +#define FAIL_RETURN_IN_FIPS_MODE(exc, name) do { \ + if (_Py_hashlib_fips_error(exc, name)) return NULL; \ } while (0) #endif // !Py_HASHOPENSSL_H diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c index 341e67a..f6bfce8 100644 --- a/Modules/_blake2/blake2b_impl.c +++ b/Modules/_blake2/blake2b_impl.c @@ -104,7 +104,7 @@ py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, unsigned long leaf_size = 0; unsigned long long node_offset = 0; - FAIL_RETURN_IN_FIPS_MODE("_blake2"); + FAIL_RETURN_IN_FIPS_MODE(PyExc_ValueError, "_blake2"); self = new_BLAKE2bObject(type); if (self == NULL) { @@ -296,7 +296,7 @@ _blake2_blake2b_update(BLAKE2bObject *self, PyObject *data) { Py_buffer buf; - FAIL_RETURN_IN_FIPS_MODE("_blake2"); + FAIL_RETURN_IN_FIPS_MODE(PyExc_ValueError, "_blake2"); GET_BUFFER_VIEW_OR_ERROUT(data, &buf); diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c index 817b716..a9c7cbc 100644 --- a/Modules/_blake2/blake2module.c +++ b/Modules/_blake2/blake2module.c @@ -58,7 +58,7 @@ PyInit__blake2(void) PyObject *m; PyObject *d; - FAIL_RETURN_IN_FIPS_MODE("blake2"); + FAIL_RETURN_IN_FIPS_MODE(PyExc_ImportError, "blake2"); m = PyModule_Create(&blake2_module); if (m == NULL) diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c index 0dfe058..28ae5b6 100644 --- a/Modules/_blake2/blake2s_impl.c +++ b/Modules/_blake2/blake2s_impl.c @@ -104,7 +104,7 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, unsigned long leaf_size = 0; unsigned long long node_offset = 0; - FAIL_RETURN_IN_FIPS_MODE("_blake2"); + FAIL_RETURN_IN_FIPS_MODE(PyExc_ValueError, "_blake2"); self = new_BLAKE2sObject(type); if (self == NULL) { @@ -296,7 +296,7 @@ _blake2_blake2s_update(BLAKE2sObject *self, PyObject *data) { Py_buffer buf; - FAIL_RETURN_IN_FIPS_MODE("_blake2"); + FAIL_RETURN_IN_FIPS_MODE(PyExc_ValueError, "_blake2"); GET_BUFFER_VIEW_OR_ERROUT(data, &buf); diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c index 624f7f2..2783a75 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/_sha3/sha3module.c @@ -163,7 +163,7 @@ static PyTypeObject SHAKE256type; static SHA3object * newSHA3object(PyTypeObject *type) { - FAIL_RETURN_IN_FIPS_MODE("_sha3"); + FAIL_RETURN_IN_FIPS_MODE(PyExc_ValueError, "_sha3"); SHA3object *newobj; newobj = (SHA3object *)PyObject_New(SHA3object, type); if (newobj == NULL) { @@ -179,7 +179,7 @@ newSHA3object(PyTypeObject *type) static PyObject * py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - FAIL_RETURN_IN_FIPS_MODE("_sha3"); + FAIL_RETURN_IN_FIPS_MODE(PyExc_ValueError, "_sha3"); SHA3object *self = NULL; Py_buffer buf = {NULL, NULL}; HashReturn res; @@ -727,7 +727,7 @@ PyInit__sha3(void) { PyObject *m = NULL; - FAIL_RETURN_IN_FIPS_MODE("_sha3"); + FAIL_RETURN_IN_FIPS_MODE(PyExc_ImportError, "_sha3"); if ((m = PyModule_Create(&_SHA3module)) == NULL) { return NULL; -- 2.25.4 From 26a1fc2de2d6323d37b22f1615b42e16debb27fa Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 26 Jul 2019 11:24:09 +0200 Subject: [PATCH 14/47] Make hashlib importable under FIPS mode --- Lib/hashlib.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 2def0a3..ca1dd20 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -132,12 +132,14 @@ def __get_openssl_constructor(name): f = getattr(_hashlib, 'openssl_' + name) # Allow the C module to raise ValueError. The function will be # defined but the hash not actually available thanks to OpenSSL. - f() + if not get_fips_mode(): + # N.B. In "FIPS mode", there is no fallback. + # If this test fails, we want to export the broken hash + # constructor anyway. + f() # Use the C function directly (very fast) return f except (AttributeError, ValueError): - if get_fips_mode(): - raise return __get_builtin_constructor(name) if not get_fips_mode(): -- 2.25.4 From e9667f72ccaabdd9696520ec80af4642753addd7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 26 Jul 2019 15:41:10 +0200 Subject: [PATCH 15/47] Implement hmac.new using new built-in module, _hmacopenssl --- Lib/hmac.py | 33 ++- Modules/_hmacopenssl.c | 396 ++++++++++++++++++++++++++++++++ Modules/clinic/_hmacopenssl.c.h | 133 +++++++++++ setup.py | 4 + 4 files changed, 565 insertions(+), 1 deletion(-) create mode 100644 Modules/_hmacopenssl.c create mode 100644 Modules/clinic/_hmacopenssl.c.h diff --git a/Lib/hmac.py b/Lib/hmac.py index 121029a..ed98406 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -6,6 +6,8 @@ Implements the HMAC algorithm as described by RFC 2104. import warnings as _warnings from _operator import _compare_digest as compare_digest import hashlib as _hashlib +import _hashlib as _hashlibopenssl +import _hmacopenssl trans_5C = bytes((x ^ 0x5C) for x in range(256)) trans_36 = bytes((x ^ 0x36) for x in range(256)) @@ -37,6 +39,11 @@ class HMAC: Note: key and msg must be a bytes or bytearray objects. """ + if _hashlib.get_fips_mode(): + raise ValueError( + 'hmac.HMAC is not available in FIPS mode. ' + + 'Use hmac.new().' + ) if not isinstance(key, (bytes, bytearray)): raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) @@ -90,6 +97,8 @@ class HMAC: def update(self, msg): """Update this hashing object with the string msg. """ + if _hashlib.get_fips_mode(): + raise ValueError('hmac.HMAC is not available in FIPS mode') self.inner.update(msg) def copy(self): @@ -130,6 +139,19 @@ class HMAC: h = self._current() return h.hexdigest() + +def _get_openssl_name(digestmod): + if isinstance(digestmod, str): + return digestmod.lower() + elif callable(digestmod): + digestmod = digestmod(b'') + + if not isinstance(digestmod, _hashlibopenssl.HASH): + raise TypeError( + 'Only OpenSSL hashlib hashes are accepted in FIPS mode.') + + return digestmod.name.lower().replace('_', '-') + def new(key, msg = None, digestmod = None): """Create a new hashing object and return it. @@ -141,4 +163,13 @@ def new(key, msg = None, digestmod = None): method, and can ask for the hash value at any time by calling its digest() method. """ - return HMAC(key, msg, digestmod) + if _hashlib.get_fips_mode(): + if digestmod is None: + digestmod = 'md5' + name = _get_openssl_name(digestmod) + result = _hmacopenssl.new(key, digestmod=name) + if msg: + result.update(msg) + return result + else: + return HMAC(key, msg, digestmod) diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c new file mode 100644 index 0000000..ca95d72 --- /dev/null +++ b/Modules/_hmacopenssl.c @@ -0,0 +1,396 @@ +/* Module that wraps all OpenSSL MHAC algorithm */ + +/* Copyright (C) 2019 Red Hat, Inc. Red Hat, Inc. and/or its affiliates + * + * Based on _hashopenssl.c, which is: + * Copyright (C) 2005-2010 Gregory P. Smith (greg@krypto.org) + * Licensed to PSF under a Contributor Agreement. + * + * Derived from a skeleton of shamodule.c containing work performed by: + * + * Andrew Kuchling (amk@amk.ca) + * Greg Stein (gstein@lyra.org) + * + */ + +#define PY_SSIZE_T_CLEAN + +#include "Python.h" +#include "structmember.h" +#include "hashlib.h" +#include "pystrhex.h" +#include "_hashopenssl.h" + + +#include + +static PyTypeObject HmacType; + +typedef struct { + PyObject_HEAD + PyObject *name; /* name of the hash algorithm */ + HMAC_CTX *ctx; /* OpenSSL hmac context */ + PyThread_type_lock lock; /* HMAC context lock */ +} HmacObject; + +#include "clinic/_hmacopenssl.c.h" +/*[clinic input] +module _hmacopenssl +class _hmacopenssl.HMAC "HmacObject *" "&HmacType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c98d3f2af591c085]*/ + + +/*[clinic input] +_hmacopenssl.new + + key: Py_buffer + * + digestmod: str + +Return a new hmac object. +[clinic start generated code]*/ + +static PyObject * +_hmacopenssl_new_impl(PyObject *module, Py_buffer *key, + const char *digestmod) +/*[clinic end generated code: output=46f1cb4e02921922 input=be8c0c2e4fad508c]*/ +{ + if (digestmod == NULL) { + PyErr_SetString(PyExc_ValueError, "digestmod must be specified"); + return NULL; + } + + /* name mut be lowercase */ + for (int i=0; digestmod[i]; i++) { + if ( + ((digestmod[i] < 'a') || (digestmod[i] > 'z')) + && ((digestmod[i] < '0') || (digestmod[i] > '9')) + && digestmod[i] != '-' + ) { + PyErr_SetString(PyExc_ValueError, "digestmod must be lowercase"); + return NULL; + } + } + + const EVP_MD *digest = EVP_get_digestbyname(digestmod); + if (!digest) { + PyErr_SetString(PyExc_ValueError, "unknown hash function"); + return NULL; + } + + PyObject *name = NULL; + HMAC_CTX *ctx = NULL; + HmacObject *retval = NULL; + + name = PyUnicode_FromFormat("hmac-%s", digestmod); + if (name == NULL) { + goto error; + } + + ctx = HMAC_CTX_new(); + if (ctx == NULL) { + _setException(PyExc_ValueError); + goto error; + } + + int r = HMAC_Init_ex( + ctx, + (const char*)key->buf, + key->len, + digest, + NULL /*impl*/); + if (r == 0) { + _setException(PyExc_ValueError); + goto error; + } + + retval = (HmacObject *)PyObject_New(HmacObject, &HmacType); + if (retval == NULL) { + goto error; + } + + retval->name = name; + retval->ctx = ctx; + retval->lock = NULL; + + return (PyObject*)retval; + +error: + if (ctx) HMAC_CTX_free(ctx); + if (name) Py_DECREF(name); + if (retval) PyObject_Del(name); + return NULL; +} + +/*[clinic input] +_hmacopenssl.HMAC.copy + +Return a copy (“clone”) of the HMAC object. +[clinic start generated code]*/ + +static PyObject * +_hmacopenssl_HMAC_copy_impl(HmacObject *self) +/*[clinic end generated code: output=fe5ee41faf30dcf0 input=f5ed20feec42d8d0]*/ +{ + HmacObject *retval = (HmacObject *)PyObject_New(HmacObject, &HmacType); + if (retval == NULL) { + return NULL; + } + + Py_INCREF(self->name); + retval->name = self->name; + + int r = HMAC_CTX_copy(retval->ctx, self->ctx); + if (r == 0) { + PyObject_Del(retval); + return _setException(PyExc_ValueError); + } + + return (PyObject*)retval; +} + +static void +_hmac_dealloc(HmacObject *self) +{ + if (self->lock != NULL) { + PyThread_free_lock(self->lock); + } + HMAC_CTX_free(self->ctx); + Py_XDECREF(self->name); + PyObject_Del(self); +} + +static PyObject * +_hmac_repr(HmacObject *self) +{ + return PyUnicode_FromFormat("<%U HMAC object @ %p>", self->name, self); +} + +/*[clinic input] +_hmacopenssl.HMAC.update + + msg: Py_buffer + +Update the HMAC object with msg. +[clinic start generated code]*/ + +static PyObject * +_hmacopenssl_HMAC_update_impl(HmacObject *self, Py_buffer *msg) +/*[clinic end generated code: output=0efeee663a98cee5 input=0683d64f35808cb9]*/ +{ + if (self->lock == NULL && msg->len >= HASHLIB_GIL_MINSIZE) { + self->lock = PyThread_allocate_lock(); + /* fail? lock = NULL and we fail over to non-threaded code. */ + } + + int r; + + if (self->lock != NULL) { + Py_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(self->lock, 1); + r = HMAC_Update(self->ctx, (const unsigned char*)msg->buf, msg->len); + PyThread_release_lock(self->lock); + Py_END_ALLOW_THREADS + } else { + r = HMAC_Update(self->ctx, (const unsigned char*)msg->buf, msg->len); + } + + if (r == 0) { + _setException(PyExc_ValueError); + return NULL; + } + Py_RETURN_NONE; +} + +static unsigned int +_digest_size(HmacObject *self) +{ + const EVP_MD *md = HMAC_CTX_get_md(self->ctx); + if (md == NULL) { + _setException(PyExc_ValueError); + return 0; + } + return EVP_MD_size(md); +} + +static int +_digest(HmacObject *self, unsigned char *buf, unsigned int len) +{ + HMAC_CTX *temp_ctx = HMAC_CTX_new(); + if (temp_ctx == NULL) { + PyErr_NoMemory(); + return 0; + } + int r = HMAC_CTX_copy(temp_ctx, self->ctx); + if (r == 0) { + _setException(PyExc_ValueError); + return 0; + } + r = HMAC_Final(temp_ctx, buf, &len); + HMAC_CTX_free(temp_ctx); + if (r == 0) { + _setException(PyExc_ValueError); + return 0; + } + return 1; +} + +/*[clinic input] +_hmacopenssl.HMAC.digest + +Return the digest of the bytes passed to the update() method so far. +[clinic start generated code]*/ + +static PyObject * +_hmacopenssl_HMAC_digest_impl(HmacObject *self) +/*[clinic end generated code: output=3aa6dbfc46ec4957 input=bf769a10b1d9edd9]*/ +{ + unsigned int digest_size = _digest_size(self); + if (digest_size == 0) { + return _setException(PyExc_ValueError); + } + unsigned char buf[digest_size]; /* FIXME: C99 feature */ + int r = _digest(self, buf, digest_size); + if (r == 0) { + return NULL; + } + return PyBytes_FromStringAndSize((const char *)buf, digest_size); +} + +/*[clinic input] +_hmacopenssl.HMAC.hexdigest + +Return hexadecimal digest of the bytes passed to the update() method so far. + +This may be used to exchange the value safely in email or other non-binary +environments. +[clinic start generated code]*/ + +static PyObject * +_hmacopenssl_HMAC_hexdigest_impl(HmacObject *self) +/*[clinic end generated code: output=630f6fa89f9f1e48 input=b8e60ec8b811c4cd]*/ +{ + unsigned int digest_size = _digest_size(self); + if (digest_size == 0) { + return _setException(PyExc_ValueError); + } + unsigned char buf[digest_size]; /* FIXME: C99 feature */ + int r = _digest(self, buf, digest_size); + if (r == 0) { + return NULL; + } + return _Py_strhex((const char *)buf, digest_size); +} + + + +static PyObject * +_hmacopenssl_get_digest_size(HmacObject *self, void *closure) +{ + unsigned int digest_size = _digest_size(self); + if (digest_size == 0) { + return _setException(PyExc_ValueError); + } + return PyLong_FromLong(digest_size); +} + +static PyObject * +_hmacopenssl_get_block_size(HmacObject *self, void *closure) +{ + const EVP_MD *md = HMAC_CTX_get_md(self->ctx); + if (md == NULL) { + return _setException(PyExc_ValueError); + } + return PyLong_FromLong(EVP_MD_size(md)); +} + +static PyMethodDef Hmac_methods[] = { + _HMACOPENSSL_HMAC_UPDATE_METHODDEF + _HMACOPENSSL_HMAC_DIGEST_METHODDEF + _HMACOPENSSL_HMAC_HEXDIGEST_METHODDEF + _HMACOPENSSL_HMAC_COPY_METHODDEF + {NULL, NULL} /* sentinel */ +}; + +static PyGetSetDef Hmac_getset[] = { + {"digest_size", (getter)_hmacopenssl_get_digest_size, NULL, NULL, NULL}, + {"block_size", (getter)_hmacopenssl_get_block_size, NULL, NULL, NULL}, + {NULL} /* Sentinel */ +}; + +static PyMemberDef Hmac_members[] = { + {"name", T_OBJECT, offsetof(HmacObject, name), READONLY, PyDoc_STR("HMAC name")}, +}; + +PyDoc_STRVAR(hmactype_doc, +"The object used to calculate HMAC of a message.\n\ +\n\ +Methods:\n\ +\n\ +update() -- updates the current digest with an additional string\n\ +digest() -- return the current digest value\n\ +hexdigest() -- return the current digest as a string of hexadecimal digits\n\ +copy() -- return a copy of the current hash object\n\ +\n\ +Attributes:\n\ +\n\ +name -- the name, including the hash algorithm used by this object\n\ +digest_size -- number of bytes in digest() output\n"); + +static PyTypeObject HmacType = { + PyVarObject_HEAD_INIT(NULL, 0) + "_hmacopenssl.HMAC", /*tp_name*/ + sizeof(HmacObject), /*tp_basicsize*/ + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = hmactype_doc, + .tp_repr = (reprfunc)_hmac_repr, + .tp_dealloc = (destructor)_hmac_dealloc, + .tp_methods = Hmac_methods, + .tp_getset = Hmac_getset, + .tp_members = Hmac_members, +}; + +static struct PyMethodDef hmacopenssl_functions[] = { + _HMACOPENSSL_NEW_METHODDEF + {NULL, NULL} /* Sentinel */ +}; + + + +/* Initialize this module. */ + + +static struct PyModuleDef _hmacopenssl_module = { + PyModuleDef_HEAD_INIT, + "_hmacopenssl", + NULL, + -1, + hmacopenssl_functions, + NULL, + NULL, + NULL, + NULL +}; + +PyMODINIT_FUNC +PyInit__hmacopenssl(void) +{ + /* TODO build EVP_functions openssl_* entries dynamically based + * on what hashes are supported rather than listing many + * but having some be unsupported. Only init appropriate + * constants. */ + + Py_TYPE(&HmacType) = &PyType_Type; + if (PyType_Ready(&HmacType) < 0) + return NULL; + + PyObject *m = PyModule_Create(&_hmacopenssl_module); + if (m == NULL) + return NULL; + + Py_INCREF((PyObject *)&HmacType); + PyModule_AddObject(m, "HMAC", (PyObject *)&HmacType); + + return m; +} diff --git a/Modules/clinic/_hmacopenssl.c.h b/Modules/clinic/_hmacopenssl.c.h new file mode 100644 index 0000000..b472a6e --- /dev/null +++ b/Modules/clinic/_hmacopenssl.c.h @@ -0,0 +1,133 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_hmacopenssl_new__doc__, +"new($module, /, key, *, digestmod)\n" +"--\n" +"\n" +"Return a new hmac object."); + +#define _HMACOPENSSL_NEW_METHODDEF \ + {"new", (PyCFunction)_hmacopenssl_new, METH_FASTCALL, _hmacopenssl_new__doc__}, + +static PyObject * +_hmacopenssl_new_impl(PyObject *module, Py_buffer *key, + const char *digestmod); + +static PyObject * +_hmacopenssl_new(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"key", "digestmod", NULL}; + static _PyArg_Parser _parser = {"y*$s:new", _keywords, 0}; + Py_buffer key = {NULL, NULL}; + const char *digestmod; + + if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, + &key, &digestmod)) { + goto exit; + } + return_value = _hmacopenssl_new_impl(module, &key, digestmod); + +exit: + /* Cleanup for key */ + if (key.obj) { + PyBuffer_Release(&key); + } + + return return_value; +} + +PyDoc_STRVAR(_hmacopenssl_HMAC_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a copy (“clone”) of the HMAC object."); + +#define _HMACOPENSSL_HMAC_COPY_METHODDEF \ + {"copy", (PyCFunction)_hmacopenssl_HMAC_copy, METH_NOARGS, _hmacopenssl_HMAC_copy__doc__}, + +static PyObject * +_hmacopenssl_HMAC_copy_impl(HmacObject *self); + +static PyObject * +_hmacopenssl_HMAC_copy(HmacObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _hmacopenssl_HMAC_copy_impl(self); +} + +PyDoc_STRVAR(_hmacopenssl_HMAC_update__doc__, +"update($self, /, msg)\n" +"--\n" +"\n" +"Update the HMAC object with msg."); + +#define _HMACOPENSSL_HMAC_UPDATE_METHODDEF \ + {"update", (PyCFunction)_hmacopenssl_HMAC_update, METH_FASTCALL, _hmacopenssl_HMAC_update__doc__}, + +static PyObject * +_hmacopenssl_HMAC_update_impl(HmacObject *self, Py_buffer *msg); + +static PyObject * +_hmacopenssl_HMAC_update(HmacObject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"msg", NULL}; + static _PyArg_Parser _parser = {"y*:update", _keywords, 0}; + Py_buffer msg = {NULL, NULL}; + + if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, + &msg)) { + goto exit; + } + return_value = _hmacopenssl_HMAC_update_impl(self, &msg); + +exit: + /* Cleanup for msg */ + if (msg.obj) { + PyBuffer_Release(&msg); + } + + return return_value; +} + +PyDoc_STRVAR(_hmacopenssl_HMAC_digest__doc__, +"digest($self, /)\n" +"--\n" +"\n" +"Return the digest of the bytes passed to the update() method so far."); + +#define _HMACOPENSSL_HMAC_DIGEST_METHODDEF \ + {"digest", (PyCFunction)_hmacopenssl_HMAC_digest, METH_NOARGS, _hmacopenssl_HMAC_digest__doc__}, + +static PyObject * +_hmacopenssl_HMAC_digest_impl(HmacObject *self); + +static PyObject * +_hmacopenssl_HMAC_digest(HmacObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _hmacopenssl_HMAC_digest_impl(self); +} + +PyDoc_STRVAR(_hmacopenssl_HMAC_hexdigest__doc__, +"hexdigest($self, /)\n" +"--\n" +"\n" +"Return hexadecimal digest of the bytes passed to the update() method so far.\n" +"\n" +"This may be used to exchange the value safely in email or other non-binary\n" +"environments."); + +#define _HMACOPENSSL_HMAC_HEXDIGEST_METHODDEF \ + {"hexdigest", (PyCFunction)_hmacopenssl_HMAC_hexdigest, METH_NOARGS, _hmacopenssl_HMAC_hexdigest__doc__}, + +static PyObject * +_hmacopenssl_HMAC_hexdigest_impl(HmacObject *self); + +static PyObject * +_hmacopenssl_HMAC_hexdigest(HmacObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _hmacopenssl_HMAC_hexdigest_impl(self); +} +/*[clinic end generated code: output=10b6e8cac6d7a2c9 input=a9049054013a1b77]*/ diff --git a/setup.py b/setup.py index 52ee704..ef8e2fe 100644 --- a/setup.py +++ b/setup.py @@ -922,6 +922,10 @@ class PyBuildExt(build_ext): openssl_ver) missing.append('_hashlib') + exts.append( Extension('_hmacopenssl', ['_hmacopenssl.c'], + depends = ['hashlib.h'], + **ssl_args)) + # RHEL: Always force OpenSSL for md5, sha1, sha256, sha512; # don't build Python's implementations. # sha3 and blake2 have extra functionality, so do build those: -- 2.25.4 From b3ca148c1b526817f4573b8b3cb4da8e88e0c61f Mon Sep 17 00:00:00 2001 From: Marcel Plch Date: Mon, 29 Jul 2019 12:45:11 +0200 Subject: [PATCH 16/47] FIPS review * Port _hmacopenssl to multiphase init. * Make _hmacopenssl.HMAC.copy create same type as self. * hmac.py cosmetic nitpick --- Lib/hmac.py | 2 +- Modules/_hmacopenssl.c | 112 +++++++++++++++++++++++++---------------- 2 files changed, 70 insertions(+), 44 deletions(-) diff --git a/Lib/hmac.py b/Lib/hmac.py index ed98406..b9bf16b 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -42,7 +42,7 @@ class HMAC: if _hashlib.get_fips_mode(): raise ValueError( 'hmac.HMAC is not available in FIPS mode. ' - + 'Use hmac.new().' + 'Use hmac.new().' ) if not isinstance(key, (bytes, bytearray)): diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c index ca95d72..216ed04 100644 --- a/Modules/_hmacopenssl.c +++ b/Modules/_hmacopenssl.c @@ -24,7 +24,10 @@ #include -static PyTypeObject HmacType; +typedef struct hmacopenssl_state { + PyTypeObject *HmacType; +} hmacopenssl_state; + typedef struct { PyObject_HEAD @@ -36,9 +39,9 @@ typedef struct { #include "clinic/_hmacopenssl.c.h" /*[clinic input] module _hmacopenssl -class _hmacopenssl.HMAC "HmacObject *" "&HmacType" +class _hmacopenssl.HMAC "HmacObject *" "PyModule_GetState(module)->HmacType" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c98d3f2af591c085]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=204b7f45847f57b4]*/ /*[clinic input] @@ -56,11 +59,18 @@ _hmacopenssl_new_impl(PyObject *module, Py_buffer *key, const char *digestmod) /*[clinic end generated code: output=46f1cb4e02921922 input=be8c0c2e4fad508c]*/ { + hmacopenssl_state *state; + if (digestmod == NULL) { PyErr_SetString(PyExc_ValueError, "digestmod must be specified"); return NULL; } + state = PyModule_GetState(module); + if (state == NULL) { + return NULL; + } + /* name mut be lowercase */ for (int i=0; digestmod[i]; i++) { if ( @@ -105,7 +115,7 @@ _hmacopenssl_new_impl(PyObject *module, Py_buffer *key, goto error; } - retval = (HmacObject *)PyObject_New(HmacObject, &HmacType); + retval = (HmacObject *)PyObject_New(HmacObject, state->HmacType); if (retval == NULL) { goto error; } @@ -133,7 +143,9 @@ static PyObject * _hmacopenssl_HMAC_copy_impl(HmacObject *self) /*[clinic end generated code: output=fe5ee41faf30dcf0 input=f5ed20feec42d8d0]*/ { - HmacObject *retval = (HmacObject *)PyObject_New(HmacObject, &HmacType); + HmacObject *retval; + + retval = (HmacObject *)PyObject_New(HmacObject, (PyTypeObject *)PyObject_Type((PyObject *)self)); if (retval == NULL) { return NULL; } @@ -147,7 +159,7 @@ _hmacopenssl_HMAC_copy_impl(HmacObject *self) return _setException(PyExc_ValueError); } - return (PyObject*)retval; + return (PyObject *)retval; } static void @@ -338,19 +350,24 @@ Attributes:\n\ name -- the name, including the hash algorithm used by this object\n\ digest_size -- number of bytes in digest() output\n"); -static PyTypeObject HmacType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_hmacopenssl.HMAC", /*tp_name*/ - sizeof(HmacObject), /*tp_basicsize*/ - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - .tp_doc = hmactype_doc, - .tp_repr = (reprfunc)_hmac_repr, - .tp_dealloc = (destructor)_hmac_dealloc, - .tp_methods = Hmac_methods, - .tp_getset = Hmac_getset, - .tp_members = Hmac_members, +static PyType_Slot HmacType_slots[] = { + {Py_tp_doc, hmactype_doc}, + {Py_tp_repr, (reprfunc)_hmac_repr}, + {Py_tp_dealloc,(destructor)_hmac_dealloc}, + {Py_tp_methods, Hmac_methods}, + {Py_tp_getset, Hmac_getset}, + {Py_tp_members, Hmac_members}, + {0, NULL} +}; + +PyType_Spec HmacType_spec = { + "_hmacopenssl.HMAC", /* name */ + sizeof(HmacObject), /* basicsize */ + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = HmacType_slots, }; + static struct PyMethodDef hmacopenssl_functions[] = { _HMACOPENSSL_NEW_METHODDEF {NULL, NULL} /* Sentinel */ @@ -360,37 +377,46 @@ static struct PyMethodDef hmacopenssl_functions[] = { /* Initialize this module. */ - -static struct PyModuleDef _hmacopenssl_module = { - PyModuleDef_HEAD_INIT, - "_hmacopenssl", - NULL, - -1, - hmacopenssl_functions, - NULL, - NULL, - NULL, - NULL -}; - -PyMODINIT_FUNC -PyInit__hmacopenssl(void) -{ +static int +hmacopenssl_exec(PyObject *m) { /* TODO build EVP_functions openssl_* entries dynamically based * on what hashes are supported rather than listing many - * but having some be unsupported. Only init appropriate + * and having some unsupported. Only init appropriate * constants. */ + PyObject *temp; - Py_TYPE(&HmacType) = &PyType_Type; - if (PyType_Ready(&HmacType) < 0) - return NULL; + temp = PyType_FromSpec(&HmacType_spec); + if (temp == NULL) { + goto fail; + } - PyObject *m = PyModule_Create(&_hmacopenssl_module); - if (m == NULL) - return NULL; + if (PyModule_AddObject(m, "HMAC", temp) == -1) { + goto fail; + } + + return 0; - Py_INCREF((PyObject *)&HmacType); - PyModule_AddObject(m, "HMAC", (PyObject *)&HmacType); +fail: + Py_XDECREF(temp); + return -1; +} - return m; +static PyModuleDef_Slot hmacopenssl_slots[] = { + {Py_mod_exec, hmacopenssl_exec}, + {0, NULL}, +}; + +static struct PyModuleDef _hmacopenssl_def = { + PyModuleDef_HEAD_INIT, /* m_base */ + .m_name = "_hmacopenssl", + .m_methods = hmacopenssl_functions, + .m_slots = hmacopenssl_slots, + .m_size = sizeof(hmacopenssl_state) +}; + + +PyMODINIT_FUNC +PyInit__hmacopenssl(void) +{ + return PyModuleDef_Init(&_hmacopenssl_def); } -- 2.25.4 From 293bccf9e9b12497621b15540b0d6f95b28731a8 Mon Sep 17 00:00:00 2001 From: Marcel Plch Date: Mon, 29 Jul 2019 13:05:04 +0200 Subject: [PATCH 17/47] revert cosmetic nitpick and remove trailing whitespace --- Lib/hmac.py | 2 +- Modules/_hmacopenssl.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/hmac.py b/Lib/hmac.py index b9bf16b..ed98406 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -42,7 +42,7 @@ class HMAC: if _hashlib.get_fips_mode(): raise ValueError( 'hmac.HMAC is not available in FIPS mode. ' - 'Use hmac.new().' + + 'Use hmac.new().' ) if not isinstance(key, (bytes, bytearray)): diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c index 216ed04..221714c 100644 --- a/Modules/_hmacopenssl.c +++ b/Modules/_hmacopenssl.c @@ -363,7 +363,7 @@ static PyType_Slot HmacType_slots[] = { PyType_Spec HmacType_spec = { "_hmacopenssl.HMAC", /* name */ sizeof(HmacObject), /* basicsize */ - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .slots = HmacType_slots, }; @@ -407,7 +407,7 @@ static PyModuleDef_Slot hmacopenssl_slots[] = { }; static struct PyModuleDef _hmacopenssl_def = { - PyModuleDef_HEAD_INIT, /* m_base */ + PyModuleDef_HEAD_INIT, /* m_base */ .m_name = "_hmacopenssl", .m_methods = hmacopenssl_functions, .m_slots = hmacopenssl_slots, -- 2.25.4 From c686423a619f1939b57ebd1a79707b966b8d80ea Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Wed, 31 Jul 2019 15:43:43 +0200 Subject: [PATCH 18/47] Add initial tests for various hashes under FIPS mode --- Lib/test/test_fips.py | 64 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 Lib/test/test_fips.py diff --git a/Lib/test/test_fips.py b/Lib/test/test_fips.py new file mode 100644 index 0000000..bee911e --- /dev/null +++ b/Lib/test/test_fips.py @@ -0,0 +1,64 @@ +import unittest +import hmac, _hmacopenssl +import hashlib, _hashlib + + + +class HashlibFipsTests(unittest.TestCase): + + @unittest.skipUnless(hashlib.get_fips_mode(), "Test only when FIPS is enabled") + def test_fips_imports(self): + """blake2s and blake2b should fail to import in FIPS mode + """ + with self.assertRaises(ValueError, msg='blake2s not available in FIPS'): + m = hashlib.blake2s() + with self.assertRaises(ValueError, msg='blake2b not available in FIPS'): + m = hashlib.blake2b() + + def compare_hashes(self, python_hash, openssl_hash): + """ + Compare between the python implementation and the openssl one that the digests + are the same + """ + if python_hash.name.startswith('shake_128'): + m = python_hash.hexdigest(16) + elif python_hash.name.startswith('shake_256'): + m = python_hash.hexdigest(32) + else: + m = python_hash.hexdigest() + h = openssl_hash.hexdigest() + + self.assertEqual(m, h) + + @unittest.skipIf(hashlib.get_fips_mode(), "blake2 hashes are not available under FIPS") + def test_blake2_hashes(self): + self.compare_hashes(hashlib.blake2b(b'abc'), _hashlib.openssl_blake2b(b'abc')) + self.compare_hashes(hashlib.blake2s(b'abc'), _hashlib.openssl_blake2s(b'abc')) + + def test_sha3_hashes(self): + self.compare_hashes(hashlib.sha3_224(b'abc'), _hashlib.openssl_sha3_224(b'abc')) + self.compare_hashes(hashlib.sha3_256(b'abc'), _hashlib.openssl_sha3_256(b'abc')) + self.compare_hashes(hashlib.sha3_384(b'abc'), _hashlib.openssl_sha3_384(b'abc')) + self.compare_hashes(hashlib.sha3_512(b'abc'), _hashlib.openssl_sha3_512(b'abc')) + + @unittest.skipIf(hashlib.get_fips_mode(), "shake hashes are not available under FIPS") + def test_shake_hashes(self): + self.compare_hashes(hashlib.shake_128(b'abc'), _hashlib.openssl_shake_128(b'abc')) + self.compare_hashes(hashlib.shake_256(b'abc'), _hashlib.openssl_shake_256(b'abc')) + + def test_sha(self): + self.compare_hashes(hashlib.sha1(b'abc'), _hashlib.openssl_sha1(b'abc')) + self.compare_hashes(hashlib.sha224(b'abc'), _hashlib.openssl_sha224(b'abc')) + self.compare_hashes(hashlib.sha256(b'abc'), _hashlib.openssl_sha256(b'abc')) + self.compare_hashes(hashlib.sha384(b'abc'), _hashlib.openssl_sha384(b'abc')) + self.compare_hashes(hashlib.sha512(b'abc'), _hashlib.openssl_sha512(b'abc')) + + def test_hmac_digests(self): + self.compare_hashes(_hmacopenssl.new(b'My hovercraft is full of eels', digestmod='sha384'), + hmac.new(b'My hovercraft is full of eels', digestmod='sha384')) + + + + +if __name__ == "__main__": + unittest.main() -- 2.25.4 From b6d7bfca3091312b42e29c9f2ee84d50755410be Mon Sep 17 00:00:00 2001 From: Marcel Plch Date: Thu, 1 Aug 2019 16:39:37 +0200 Subject: [PATCH 19/47] Initialize HMAC type. --- Modules/_hmacopenssl.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c index 221714c..239445a 100644 --- a/Modules/_hmacopenssl.c +++ b/Modules/_hmacopenssl.c @@ -22,12 +22,12 @@ #include "_hashopenssl.h" -#include typedef struct hmacopenssl_state { PyTypeObject *HmacType; } hmacopenssl_state; +#include typedef struct { PyObject_HEAD @@ -39,7 +39,7 @@ typedef struct { #include "clinic/_hmacopenssl.c.h" /*[clinic input] module _hmacopenssl -class _hmacopenssl.HMAC "HmacObject *" "PyModule_GetState(module)->HmacType" +class _hmacopenssl.HMAC "HmacObject *" "((hmacopenssl_state *)PyModule_GetState(module))->HmacType" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=204b7f45847f57b4]*/ @@ -71,7 +71,7 @@ _hmacopenssl_new_impl(PyObject *module, Py_buffer *key, return NULL; } - /* name mut be lowercase */ + /* name must be lowercase */ for (int i=0; digestmod[i]; i++) { if ( ((digestmod[i] < 'a') || (digestmod[i] > 'z')) @@ -383,7 +383,8 @@ hmacopenssl_exec(PyObject *m) { * on what hashes are supported rather than listing many * and having some unsupported. Only init appropriate * constants. */ - PyObject *temp; + PyObject *temp = NULL; + hmacopenssl_state *state; temp = PyType_FromSpec(&HmacType_spec); if (temp == NULL) { @@ -394,6 +395,9 @@ hmacopenssl_exec(PyObject *m) { goto fail; } + state = PyModule_GetState(m); + state->HmacType = (PyTypeObject *)temp; + return 0; fail: -- 2.25.4 From 89563cde5051e110028ca89180be6e71fef0070f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 1 Aug 2019 17:57:05 +0200 Subject: [PATCH 20/47] Use a stronger hash in multiprocessing handshake Adapted from patch by David Malcolm, https://bugs.python.org/issue17258 --- Lib/multiprocessing/connection.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index d379750..a0b1538 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -42,6 +42,10 @@ BUFSIZE = 8192 # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. +# The hmac module implicitly defaults to using MD5. +# Support using a stronger algorithm for the challenge/response code: +HMAC_DIGEST_NAME='sha256' + _mmap_counter = itertools.count() default_family = 'AF_INET' @@ -718,7 +722,7 @@ def deliver_challenge(connection, authkey): assert isinstance(authkey, bytes) message = os.urandom(MESSAGE_LENGTH) connection.send_bytes(CHALLENGE + message) - digest = hmac.new(authkey, message, 'md5').digest() + digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest() response = connection.recv_bytes(256) # reject large message if response == digest: connection.send_bytes(WELCOME) @@ -732,7 +736,7 @@ def answer_challenge(connection, authkey): message = connection.recv_bytes(256) # reject large message assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message message = message[len(CHALLENGE):] - digest = hmac.new(authkey, message, 'md5').digest() + digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest() connection.send_bytes(digest) response = connection.recv_bytes(256) # reject large message if response != WELCOME: -- 2.25.4 From f28bbbd23c874db9609450d074d403e0b96ad888 Mon Sep 17 00:00:00 2001 From: Marcel Plch Date: Fri, 2 Aug 2019 17:36:01 +0200 Subject: [PATCH 21/47] Fix refcounting --- Modules/_hmacopenssl.c | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c index 239445a..9c28828 100644 --- a/Modules/_hmacopenssl.c +++ b/Modules/_hmacopenssl.c @@ -373,6 +373,34 @@ static struct PyMethodDef hmacopenssl_functions[] = { {NULL, NULL} /* Sentinel */ }; +static int +hmacopenssl_traverse(PyObject *self, visitproc visit, void *arg) +{ + hmacopenssl_state *state; + + state = PyModule_GetState(self); + + if (state) { + Py_VISIT(state->HmacType); + } + + return 0; +} + +static int +hmacopenssl_clear(PyObject *self) +{ + hmacopenssl_state *state; + + state = PyModule_GetState(self); + + if (state) { + Py_CLEAR(state->HmacType); + } + + return 0; +} + /* Initialize this module. */ @@ -396,7 +424,10 @@ hmacopenssl_exec(PyObject *m) { } state = PyModule_GetState(m); + state->HmacType = (PyTypeObject *)temp; + Py_INCREF(temp); + return 0; @@ -415,7 +446,9 @@ static struct PyModuleDef _hmacopenssl_def = { .m_name = "_hmacopenssl", .m_methods = hmacopenssl_functions, .m_slots = hmacopenssl_slots, - .m_size = sizeof(hmacopenssl_state) + .m_size = sizeof(hmacopenssl_state), + .m_traverse = hmacopenssl_traverse, + .m_clear = hmacopenssl_clear }; -- 2.25.4 From 8dcfe3926888571be626959485a8e05a36edd383 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 5 Aug 2019 13:37:05 +0200 Subject: [PATCH 22/47] hmac: Don't default to md5 in FIPS mode --- Lib/hmac.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/hmac.py b/Lib/hmac.py index ed98406..7b8821e 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -165,7 +165,7 @@ def new(key, msg = None, digestmod = None): """ if _hashlib.get_fips_mode(): if digestmod is None: - digestmod = 'md5' + raise ValueError("'digestmod' argument is mandatory in FIPS mode") name = _get_openssl_name(digestmod) result = _hmacopenssl.new(key, digestmod=name) if msg: -- 2.25.4 From 18dfc7c75df80a8f4b43e024c851722ac48bfe18 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 5 Aug 2019 14:20:58 +0200 Subject: [PATCH 23/47] Make _hmacopenssl.HMAC subclassable; subclass it as hmac.HMAC under FIPS This removes the _hmacopenssl.new function. --- Lib/hmac.py | 27 +++++++----- Lib/test/test_fips.py | 2 +- Modules/_hmacopenssl.c | 75 ++++++++++++++++----------------- Modules/clinic/_hmacopenssl.c.h | 39 +---------------- 4 files changed, 56 insertions(+), 87 deletions(-) diff --git a/Lib/hmac.py b/Lib/hmac.py index 7b8821e..d479c5a 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -141,6 +141,8 @@ class HMAC: def _get_openssl_name(digestmod): + if digestmod is None: + raise ValueError("'digestmod' argument is mandatory in FIPS mode") if isinstance(digestmod, str): return digestmod.lower() elif callable(digestmod): @@ -152,6 +154,20 @@ def _get_openssl_name(digestmod): return digestmod.name.lower().replace('_', '-') + +class HMAC_openssl(_hmacopenssl.HMAC): + def __new__(cls, key, msg = None, digestmod = None): + name = _get_openssl_name(digestmod) + result = _hmacopenssl.HMAC.__new__(cls, key, digestmod=name) + if msg: + result.update(msg) + return result + + +if _hashlib.get_fips_mode(): + HMAC = HMAC_openssl + + def new(key, msg = None, digestmod = None): """Create a new hashing object and return it. @@ -163,13 +179,4 @@ def new(key, msg = None, digestmod = None): method, and can ask for the hash value at any time by calling its digest() method. """ - if _hashlib.get_fips_mode(): - if digestmod is None: - raise ValueError("'digestmod' argument is mandatory in FIPS mode") - name = _get_openssl_name(digestmod) - result = _hmacopenssl.new(key, digestmod=name) - if msg: - result.update(msg) - return result - else: - return HMAC(key, msg, digestmod) + return HMAC(key, msg, digestmod) diff --git a/Lib/test/test_fips.py b/Lib/test/test_fips.py index bee911e..34812e6 100644 --- a/Lib/test/test_fips.py +++ b/Lib/test/test_fips.py @@ -54,7 +54,7 @@ class HashlibFipsTests(unittest.TestCase): self.compare_hashes(hashlib.sha512(b'abc'), _hashlib.openssl_sha512(b'abc')) def test_hmac_digests(self): - self.compare_hashes(_hmacopenssl.new(b'My hovercraft is full of eels', digestmod='sha384'), + self.compare_hashes(_hmacopenssl.HMAC(b'My hovercraft is full of eels', digestmod='sha384'), hmac.new(b'My hovercraft is full of eels', digestmod='sha384')) diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c index 9c28828..7d3d973 100644 --- a/Modules/_hmacopenssl.c +++ b/Modules/_hmacopenssl.c @@ -41,33 +41,25 @@ typedef struct { module _hmacopenssl class _hmacopenssl.HMAC "HmacObject *" "((hmacopenssl_state *)PyModule_GetState(module))->HmacType" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=204b7f45847f57b4]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=9fe07a087adc2cf9]*/ -/*[clinic input] -_hmacopenssl.new - - key: Py_buffer - * - digestmod: str - -Return a new hmac object. -[clinic start generated code]*/ - static PyObject * -_hmacopenssl_new_impl(PyObject *module, Py_buffer *key, - const char *digestmod) -/*[clinic end generated code: output=46f1cb4e02921922 input=be8c0c2e4fad508c]*/ +Hmac_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) { - hmacopenssl_state *state; - - if (digestmod == NULL) { - PyErr_SetString(PyExc_ValueError, "digestmod must be specified"); + static char *kwarg_names[] = {"key", "digestmod", NULL}; + Py_buffer key = {NULL, NULL}; + char *digestmod = NULL; + + int ret = PyArg_ParseTupleAndKeywords( + args, kwds, "y*|$s:_hmacopenssl.HMAC", kwarg_names, + &key, &digestmod); + if (ret == 0) { return NULL; } - state = PyModule_GetState(module); - if (state == NULL) { + if (digestmod == NULL) { + PyErr_SetString(PyExc_ValueError, "digestmod must be specified"); return NULL; } @@ -106,8 +98,8 @@ _hmacopenssl_new_impl(PyObject *module, Py_buffer *key, int r = HMAC_Init_ex( ctx, - (const char*)key->buf, - key->len, + (const char*)key.buf, + key.len, digest, NULL /*impl*/); if (r == 0) { @@ -115,7 +107,10 @@ _hmacopenssl_new_impl(PyObject *module, Py_buffer *key, goto error; } - retval = (HmacObject *)PyObject_New(HmacObject, state->HmacType); + PyBuffer_Release(&key); + key.buf = NULL; + + retval = (HmacObject *)subtype->tp_alloc(subtype, 0); if (retval == NULL) { goto error; } @@ -130,6 +125,7 @@ error: if (ctx) HMAC_CTX_free(ctx); if (name) Py_DECREF(name); if (retval) PyObject_Del(name); + if (key.buf) PyBuffer_Release(&key); return NULL; } @@ -145,19 +141,27 @@ _hmacopenssl_HMAC_copy_impl(HmacObject *self) { HmacObject *retval; - retval = (HmacObject *)PyObject_New(HmacObject, (PyTypeObject *)PyObject_Type((PyObject *)self)); + HMAC_CTX *ctx = HMAC_CTX_new(); + if (ctx == NULL) { + return _setException(PyExc_ValueError); + } + + int r = HMAC_CTX_copy(ctx, self->ctx); + if (r == 0) { + HMAC_CTX_free(ctx); + return _setException(PyExc_ValueError); + } + + retval = (HmacObject *)Py_TYPE(self)->tp_alloc(Py_TYPE(self), 0); if (retval == NULL) { + HMAC_CTX_free(ctx); return NULL; } - + retval->ctx = ctx; Py_INCREF(self->name); retval->name = self->name; - int r = HMAC_CTX_copy(retval->ctx, self->ctx); - if (r == 0) { - PyObject_Del(retval); - return _setException(PyExc_ValueError); - } + retval->lock = NULL; return (PyObject *)retval; } @@ -169,8 +173,8 @@ _hmac_dealloc(HmacObject *self) PyThread_free_lock(self->lock); } HMAC_CTX_free(self->ctx); - Py_XDECREF(self->name); - PyObject_Del(self); + Py_CLEAR(self->name); + Py_TYPE(self)->tp_free(self); } static PyObject * @@ -357,6 +361,7 @@ static PyType_Slot HmacType_slots[] = { {Py_tp_methods, Hmac_methods}, {Py_tp_getset, Hmac_getset}, {Py_tp_members, Hmac_members}, + {Py_tp_new, Hmac_new}, {0, NULL} }; @@ -368,11 +373,6 @@ PyType_Spec HmacType_spec = { }; -static struct PyMethodDef hmacopenssl_functions[] = { - _HMACOPENSSL_NEW_METHODDEF - {NULL, NULL} /* Sentinel */ -}; - static int hmacopenssl_traverse(PyObject *self, visitproc visit, void *arg) { @@ -444,7 +444,6 @@ static PyModuleDef_Slot hmacopenssl_slots[] = { static struct PyModuleDef _hmacopenssl_def = { PyModuleDef_HEAD_INIT, /* m_base */ .m_name = "_hmacopenssl", - .m_methods = hmacopenssl_functions, .m_slots = hmacopenssl_slots, .m_size = sizeof(hmacopenssl_state), .m_traverse = hmacopenssl_traverse, diff --git a/Modules/clinic/_hmacopenssl.c.h b/Modules/clinic/_hmacopenssl.c.h index b472a6e..861acc1 100644 --- a/Modules/clinic/_hmacopenssl.c.h +++ b/Modules/clinic/_hmacopenssl.c.h @@ -2,43 +2,6 @@ preserve [clinic start generated code]*/ -PyDoc_STRVAR(_hmacopenssl_new__doc__, -"new($module, /, key, *, digestmod)\n" -"--\n" -"\n" -"Return a new hmac object."); - -#define _HMACOPENSSL_NEW_METHODDEF \ - {"new", (PyCFunction)_hmacopenssl_new, METH_FASTCALL, _hmacopenssl_new__doc__}, - -static PyObject * -_hmacopenssl_new_impl(PyObject *module, Py_buffer *key, - const char *digestmod); - -static PyObject * -_hmacopenssl_new(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - static const char * const _keywords[] = {"key", "digestmod", NULL}; - static _PyArg_Parser _parser = {"y*$s:new", _keywords, 0}; - Py_buffer key = {NULL, NULL}; - const char *digestmod; - - if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, - &key, &digestmod)) { - goto exit; - } - return_value = _hmacopenssl_new_impl(module, &key, digestmod); - -exit: - /* Cleanup for key */ - if (key.obj) { - PyBuffer_Release(&key); - } - - return return_value; -} - PyDoc_STRVAR(_hmacopenssl_HMAC_copy__doc__, "copy($self, /)\n" "--\n" @@ -130,4 +93,4 @@ _hmacopenssl_HMAC_hexdigest(HmacObject *self, PyObject *Py_UNUSED(ignored)) { return _hmacopenssl_HMAC_hexdigest_impl(self); } -/*[clinic end generated code: output=10b6e8cac6d7a2c9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d93ad460795d49b5 input=a9049054013a1b77]*/ -- 2.25.4 From 99f652a5949ba1ac34c5510ea0f47e2da1f3850a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 5 Aug 2019 16:10:36 +0200 Subject: [PATCH 24/47] Fix _hmacopenssl.HMAC.block_size --- Modules/_hmacopenssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c index 7d3d973..a24c8ba 100644 --- a/Modules/_hmacopenssl.c +++ b/Modules/_hmacopenssl.c @@ -318,7 +318,7 @@ _hmacopenssl_get_block_size(HmacObject *self, void *closure) if (md == NULL) { return _setException(PyExc_ValueError); } - return PyLong_FromLong(EVP_MD_size(md)); + return PyLong_FromLong(EVP_MD_block_size(md)); } static PyMethodDef Hmac_methods[] = { -- 2.25.4 From d155dc5c093c30dc83219d043609ebbeab953c0a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 5 Aug 2019 15:02:08 +0200 Subject: [PATCH 25/47] distutils upload: Skip md5 checksum in FIPS mode --- Lib/distutils/command/upload.py | 11 ++++++++++- Lib/distutils/tests/test_upload.py | 13 +++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Lib/distutils/command/upload.py b/Lib/distutils/command/upload.py index 32dda35..0edb39e 100644 --- a/Lib/distutils/command/upload.py +++ b/Lib/distutils/command/upload.py @@ -102,7 +102,6 @@ class upload(PyPIRCCommand): 'content': (os.path.basename(filename),content), 'filetype': command, 'pyversion': pyversion, - 'md5_digest': hashlib.md5(content).hexdigest(), # additional meta-data 'metadata_version': '1.0', @@ -121,6 +120,16 @@ class upload(PyPIRCCommand): 'requires': meta.get_requires(), 'obsoletes': meta.get_obsoletes(), } + try: + digest = hashlib.md5(content).hexdigest() + except ValueError as e: + msg = 'calculating md5 checksum failed: %s' % e + self.announce(msg, log.ERROR) + if not hashlib.get_fips_mode(): + # this really shouldn't fail + raise + else: + data['md5_digest'] = digest comment = '' if command == 'bdist_rpm': dist, version, id = platform.dist() diff --git a/Lib/distutils/tests/test_upload.py b/Lib/distutils/tests/test_upload.py index c17d8e7..b4b64e9 100644 --- a/Lib/distutils/tests/test_upload.py +++ b/Lib/distutils/tests/test_upload.py @@ -3,6 +3,7 @@ import os import unittest import unittest.mock as mock from urllib.request import HTTPError +import hashlib from test.support import run_unittest @@ -130,7 +131,11 @@ class uploadTestCase(BasePyPIRCCommandTestCase): # what did we send ? headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2162') + if hashlib.get_fips_mode(): + # md5 hash is omitted + self.assertEqual(headers['Content-length'], '2020') + else: + self.assertEqual(headers['Content-length'], '2162') content_type = headers['Content-type'] self.assertTrue(content_type.startswith('multipart/form-data')) self.assertEqual(self.last_open.req.get_method(), 'POST') @@ -166,7 +171,11 @@ class uploadTestCase(BasePyPIRCCommandTestCase): cmd.run() headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2172') + if hashlib.get_fips_mode(): + # md5 hash is omitted + self.assertEqual(headers['Content-length'], '2030') + else: + self.assertEqual(headers['Content-length'], '2172') self.assertIn(b'long description\r', self.last_open.req.data) def test_upload_fails(self): -- 2.25.4 From 8d3fc7b8e624d5db6ac9720fed4345e614a1a740 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 5 Aug 2019 15:32:25 +0200 Subject: [PATCH 26/47] Fix HMAC tests on FIPS mode --- Lib/hmac.py | 3 +++ Lib/test/test_hmac.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/Lib/hmac.py b/Lib/hmac.py index d479c5a..7af94c3 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -157,6 +157,9 @@ def _get_openssl_name(digestmod): class HMAC_openssl(_hmacopenssl.HMAC): def __new__(cls, key, msg = None, digestmod = None): + if not isinstance(key, (bytes, bytearray)): + raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) + name = _get_openssl_name(digestmod) result = _hmacopenssl.HMAC.__new__(cls, key, digestmod=name) if msg: diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 338e021..5b09072 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -250,6 +250,7 @@ class TestVectorsTestCase(unittest.TestCase): def test_sha512_rfc4231(self): self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128) + @unittest.skipIf(hashlib.get_fips_mode(), 'MockCrazyHash unacceptable in FIPS mode.') @requires_hashdigest('sha256') def test_legacy_block_size_warnings(self): class MockCrazyHash(object): @@ -298,6 +299,14 @@ class ConstructorTestCase(unittest.TestCase): self.fail("Standard constructor call raised exception.") @ignore_warning + def test_normal_digestmod(self): + # Standard constructor call. + failed = 0 + try: + h = hmac.HMAC(b"key", digestmod='sha1') + except Exception: + self.fail("Standard constructor call raised exception.") + @requires_hashdigest('sha256') def test_with_str_key(self): # Pass a key of type str, which is an error, because it expects a key @@ -366,6 +375,7 @@ class SanityTestCase(unittest.TestCase): class CopyTestCase(unittest.TestCase): + @unittest.skipIf(hashlib.get_fips_mode(), "Internal attributes unavailable in FIPS mode") @requires_hashdigest('sha256') def test_attributes(self): # Testing if attributes are of same type. @@ -378,6 +388,7 @@ class CopyTestCase(unittest.TestCase): self.assertEqual(type(h1.outer), type(h2.outer), "Types of outer don't match.") + @unittest.skipIf(hashlib.get_fips_mode(), "Internal attributes unavailable in FIPS mode") @requires_hashdigest('sha256') def test_realcopy(self): # Testing if the copy method created a real copy. @@ -390,6 +401,21 @@ class CopyTestCase(unittest.TestCase): self.assertTrue(id(h1.outer) != id(h2.outer), "No real copy of the attribute 'outer'.") + def test_realcopy(self): + # Testing if the copy method created a real copy. + h1 = hmac.HMAC(b"key", digestmod="sha1") + h2 = h1.copy() + # Using id() in case somebody has overridden __eq__/__ne__. + self.assertTrue(id(h1) != id(h2), "No real copy of the HMAC instance.") + old_digest = h1.digest() + assert h1.digest() == h2.digest() + h1.update(b'hi') + assert h1.digest() != h2.digest() + assert h2.digest() == old_digest + new_digest = h1.digest() + h2.update(b'hi') + assert h1.digest() == h2.digest() == new_digest + @requires_hashdigest('sha256') def test_equality(self): # Testing if the copy has the same digests. -- 2.25.4 From b0ab74c4edace20f5fda41fe1c4aeec2ba2ec851 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 5 Aug 2019 16:37:12 +0200 Subject: [PATCH 27/47] test_tools: Skip md5sum tests in FIPS mode --- Lib/test/test_tools/test_md5sum.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_tools/test_md5sum.py b/Lib/test/test_tools/test_md5sum.py index fb565b7..7028a4d 100644 --- a/Lib/test/test_tools/test_md5sum.py +++ b/Lib/test/test_tools/test_md5sum.py @@ -4,11 +4,15 @@ import os import unittest from test import support from test.support.script_helper import assert_python_ok, assert_python_failure +import hashlib from test.test_tools import scriptsdir, skip_if_missing skip_if_missing() +if hashlib.get_fips_mode(): + raise unittest.SkipTest("md5sum won't work at all in FIPS mode") + class MD5SumTests(unittest.TestCase): @classmethod def setUpClass(cls): -- 2.25.4 From 3cf34aca2cc87146763d8e2ea83f2f9089beacd8 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 5 Aug 2019 17:21:16 +0200 Subject: [PATCH 28/47] _hashopenssl: Include hash name in the error message --- Modules/_hashopenssl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 7dfd708..0563473 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -431,7 +431,7 @@ EVPnew(PyObject *name_obj, EVPobject *self; if (!digest && !initial_ctx) { - PyErr_SetString(PyExc_ValueError, "unsupported hash type"); + PyErr_Format(PyExc_ValueError, "unsupported hash type %U", name_obj); return NULL; } @@ -622,7 +622,7 @@ pbkdf2_hmac(PyObject *self, PyObject *args, PyObject *kwdict) digest = EVP_get_digestbyname(name); if (digest == NULL) { - PyErr_SetString(PyExc_ValueError, "unsupported hash type"); + PyErr_Format(PyExc_ValueError, "unsupported hash type %s", name); goto end; } -- 2.25.4 From 6198c0fdfd6a10aa2dba4a51c56a44dbc85b18b7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 5 Aug 2019 18:23:57 +0200 Subject: [PATCH 29/47] Make hashlib tests pass in FIPS mode (with some hashlib changes) --- Lib/hashlib.py | 18 +++++++++-- Lib/test/test_hashlib.py | 67 ++++++++++++++++++++++++++++------------ Modules/_hashopenssl.c | 14 +++++++++ 3 files changed, 78 insertions(+), 21 deletions(-) diff --git a/Lib/hashlib.py b/Lib/hashlib.py index ca1dd20..d3344f6 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -155,7 +155,18 @@ def __hash_new(name, data=b'', **kwargs): """new(name, data=b'') - Return a new hashing object using the named algorithm; optionally initialized with data (which must be a bytes-like object). """ - if not get_fips_mode(): + if get_fips_mode(): + # Use OpenSSL names for Python built-in hashes + orig_name = name + name = { + 'sha3_224': "sha3-224", + 'sha3_256': "sha3-256", + 'sha3_384': "sha3-384", + 'sha3_512': "sha3-512", + 'shake_128': "shake128", + 'shake_256': "shake256", + }.get(name, name) + else: if name in {'blake2b', 'blake2s'}: # Prefer our blake2 implementation. # OpenSSL 1.1.0 comes with a limited implementation of blake2b/s. @@ -163,7 +174,10 @@ def __hash_new(name, data=b'', **kwargs): # salt, personal, tree hashing or SSE. return __get_builtin_constructor(name)(data, **kwargs) try: - return _hashlib.new(name, data) + retval = _hashlib.new(name, data) + if get_fips_mode() and name != orig_name: + retval._set_name(orig_name) + return retval except ValueError: # If the _hashlib module (OpenSSL) doesn't support the named # hash, try using our builtin implementations. diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index e57c93b..9745bfd 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -31,6 +31,11 @@ COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount') c_hashlib = import_fresh_module('hashlib', fresh=['_hashlib']) py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) +if hashlib.get_fips_mode(): + FIPS_DISABLED = {'md5', 'MD5', 'blake2b', 'blake2s'} +else: + FIPS_DISABLED = set() + try: import _blake2 except ImportError: @@ -86,6 +91,11 @@ class HashLibTestCase(unittest.TestCase): # Issue #14693: fallback modules are always compiled under POSIX _warn_on_extension_import = os.name == 'posix' or COMPILED_WITH_PYDEBUG + if hashlib.get_fips_mode(): + shakes = set() + supported_hash_names = tuple( + n for n in supported_hash_names if n not in FIPS_DISABLED) + def _conditional_import_module(self, module_name): """Import a module and return a reference to it or None on failure.""" try: @@ -93,8 +103,20 @@ class HashLibTestCase(unittest.TestCase): except ModuleNotFoundError as error: if self._warn_on_extension_import: warnings.warn('Did a C extension fail to compile? %s' % error) + except ImportError as error: + if not hashlib.get_fips_mode(): + raise return None + def _has_shake_extras(self, hasher): + """Return true if the hasher should have "shake" API (digest length)""" + if hasher.name not in self.shakes: + return False + _hashlib = self._conditional_import_module('_hashlib') + if _hashlib and isinstance(hasher, _hashlib.HASH): + return False + return True + def __init__(self, *args, **kwargs): algorithms = set() for algorithm in self.supported_hash_names: @@ -182,15 +204,13 @@ class HashLibTestCase(unittest.TestCase): a = array.array("b", range(10)) for cons in self.hash_constructors: c = cons(a) - if (c.name in self.shakes - and not cons.__name__.startswith('openssl_') - ): + if self._has_shake_extras(c): c.hexdigest(16) else: c.hexdigest() def test_algorithms_guaranteed(self): - self.assertEqual(hashlib.algorithms_guaranteed, + self.assertEqual(hashlib.algorithms_guaranteed - FIPS_DISABLED, set(_algo for _algo in self.supported_hash_names if _algo.islower())) @@ -202,6 +222,12 @@ class HashLibTestCase(unittest.TestCase): self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam') self.assertRaises(TypeError, hashlib.new, 1) + @unittest.skipUnless(hashlib.get_fips_mode(), "Builtin constructor only unavailable in FIPS mode") + def test_get_builtin_constructor_fips(self): + with self.assertRaises(AttributeError): + hashlib.__get_builtin_constructor + + @unittest.skipIf(hashlib.get_fips_mode(), "No builtin constructors in FIPS mode") def test_get_builtin_constructor(self): get_builtin_constructor = getattr(hashlib, '__get_builtin_constructor') @@ -231,9 +257,7 @@ class HashLibTestCase(unittest.TestCase): def test_hexdigest(self): for cons in self.hash_constructors: h = cons() - if (h.name in self.shakes - and not cons.__name__.startswith('openssl_') - ): + if self._has_shake_extras(h): self.assertIsInstance(h.digest(16), bytes) self.assertEqual(hexstr(h.digest(16)), h.hexdigest(16)) else: @@ -245,9 +269,7 @@ class HashLibTestCase(unittest.TestCase): large_sizes = (2**29, 2**32-10, 2**32+10, 2**61, 2**64-10, 2**64+10) for cons in self.hash_constructors: h = cons() - if h.name not in self.shakes: - continue - if cons.__name__.startswith('openssl_'): + if not self._has_shake_extras(h): continue for digest in h.digest, h.hexdigest: with self.assertRaises((ValueError, OverflowError)): @@ -278,9 +300,7 @@ class HashLibTestCase(unittest.TestCase): m1.update(bees) m1.update(cees) m1.update(dees) - if (m1.name in self.shakes - and not cons.__name__.startswith('openssl_') - ): + if self._has_shake_extras(m1): args = (16,) else: args = () @@ -349,7 +369,8 @@ class HashLibTestCase(unittest.TestCase): self.assertRaises(TypeError, hash_object_constructor, 'spam') def test_no_unicode(self): - self.check_no_unicode('md5') + if not hashlib.get_fips_mode(): + self.check_no_unicode('md5') self.check_no_unicode('sha1') self.check_no_unicode('sha224') self.check_no_unicode('sha256') @@ -392,7 +413,8 @@ class HashLibTestCase(unittest.TestCase): self.assertIn(name.split("_")[0], repr(m)) def test_blocksize_name(self): - self.check_blocksize_name('md5', 64, 16) + if not hashlib.get_fips_mode(): + self.check_blocksize_name('md5', 64, 16) self.check_blocksize_name('sha1', 64, 20) self.check_blocksize_name('sha224', 64, 28) self.check_blocksize_name('sha256', 64, 32) @@ -432,22 +454,27 @@ class HashLibTestCase(unittest.TestCase): self.check_blocksize_name('blake2b', 128, 64) self.check_blocksize_name('blake2s', 64, 32) + @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") def test_case_md5_0(self): self.check('md5', b'', 'd41d8cd98f00b204e9800998ecf8427e') + @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") def test_case_md5_1(self): self.check('md5', b'abc', '900150983cd24fb0d6963f7d28e17f72') + @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") def test_case_md5_2(self): self.check('md5', b'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', 'd174ab98d277d9f5a5611c2c9f419d9f') + @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") @unittest.skipIf(sys.maxsize < _4G + 5, 'test cannot run on 32-bit systems') @bigmemtest(size=_4G + 5, memuse=1, dry_run=False) def test_case_md5_huge(self, size): self.check('md5', b'A'*size, 'c9af2dff37468ce5dfee8f2cfc0a9c6d') + @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") @unittest.skipIf(sys.maxsize < _4G - 1, 'test cannot run on 32-bit systems') @bigmemtest(size=_4G - 1, memuse=1, dry_run=False) def test_case_md5_uintmax(self, size): @@ -829,14 +856,16 @@ class HashLibTestCase(unittest.TestCase): m = cons(b'x' * gil_minsize) m.update(b'1') - m = hashlib.md5() + m = hashlib.sha1() m.update(b'1') m.update(b'#' * gil_minsize) m.update(b'1') - self.assertEqual(m.hexdigest(), 'cb1e1a2cbc80be75e19935d621fb9b21') + self.assertEqual(m.hexdigest(), + 'c45f7445ca0ea087d7a1758fbea07935f267c46a') - m = hashlib.md5(b'x' * gil_minsize) - self.assertEqual(m.hexdigest(), 'cfb767f225d58469c5de3632a8803958') + m = hashlib.sha1(b'x' * gil_minsize) + self.assertEqual(m.hexdigest(), + '63fda1efde982ba1ffe9d53035bff5c9ce4758fb') @unittest.skipUnless(threading, 'Threading required for this test.') @support.reap_threads diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 0563473..e330423 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -256,11 +256,25 @@ EVP_update(EVPobject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +EVP_set_name(EVPobject *self, PyObject *new_name) +{ + if (!PyUnicode_CheckExact(new_name)) { + PyErr_SetString(PyExc_TypeError, "expected string"); + return NULL; + } + Py_DECREF(self->name); + Py_INCREF(new_name); + self->name = new_name; + Py_RETURN_NONE; +} + static PyMethodDef EVP_methods[] = { {"update", (PyCFunction)EVP_update, METH_VARARGS, EVP_update__doc__}, {"digest", (PyCFunction)EVP_digest, METH_NOARGS, EVP_digest__doc__}, {"hexdigest", (PyCFunction)EVP_hexdigest, METH_NOARGS, EVP_hexdigest__doc__}, {"copy", (PyCFunction)EVP_copy, METH_NOARGS, EVP_copy__doc__}, + {"_set_name", (PyCFunction)EVP_set_name, METH_O, EVP_copy__doc__}, {NULL, NULL} /* sentinel */ }; -- 2.25.4 From aaced7cd109440a5f1f1eabf271195ce35c69a36 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Wed, 14 Aug 2019 14:43:07 +0200 Subject: [PATCH 30/47] distutils upload: only add md5 if available, but *always* use sha256 --- Lib/distutils/command/upload.py | 3 ++- Lib/distutils/tests/test_upload.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Lib/distutils/command/upload.py b/Lib/distutils/command/upload.py index 0edb39e..170acb3 100644 --- a/Lib/distutils/command/upload.py +++ b/Lib/distutils/command/upload.py @@ -102,6 +102,7 @@ class upload(PyPIRCCommand): 'content': (os.path.basename(filename),content), 'filetype': command, 'pyversion': pyversion, + 'sha256_digest': hashlib.sha256(content).hexdigest(), # additional meta-data 'metadata_version': '1.0', @@ -124,7 +125,7 @@ class upload(PyPIRCCommand): digest = hashlib.md5(content).hexdigest() except ValueError as e: msg = 'calculating md5 checksum failed: %s' % e - self.announce(msg, log.ERROR) + self.announce(msg, log.INFO) if not hashlib.get_fips_mode(): # this really shouldn't fail raise diff --git a/Lib/distutils/tests/test_upload.py b/Lib/distutils/tests/test_upload.py index b4b64e9..f720a79 100644 --- a/Lib/distutils/tests/test_upload.py +++ b/Lib/distutils/tests/test_upload.py @@ -132,10 +132,11 @@ class uploadTestCase(BasePyPIRCCommandTestCase): # what did we send ? headers = dict(self.last_open.req.headers) if hashlib.get_fips_mode(): - # md5 hash is omitted - self.assertEqual(headers['Content-length'], '2020') + # only sha256 hash is used + self.assertEqual(headers['Content-length'], '2197') else: - self.assertEqual(headers['Content-length'], '2162') + # both sha256 and md5 hashes are used + self.assertEqual(headers['Content-length'], '2339') content_type = headers['Content-type'] self.assertTrue(content_type.startswith('multipart/form-data')) self.assertEqual(self.last_open.req.get_method(), 'POST') @@ -172,10 +173,11 @@ class uploadTestCase(BasePyPIRCCommandTestCase): headers = dict(self.last_open.req.headers) if hashlib.get_fips_mode(): - # md5 hash is omitted - self.assertEqual(headers['Content-length'], '2030') + # only sha256 hash is used + self.assertEqual(headers['Content-length'], '2207') else: - self.assertEqual(headers['Content-length'], '2172') + # both sha256 and md5 hashes are used + self.assertEqual(headers['Content-length'], '2349') self.assertIn(b'long description\r', self.last_open.req.data) def test_upload_fails(self): -- 2.25.4 From 3fc31eb1a07366e1b1becc356695ac643dc0577f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2019 15:55:48 +0200 Subject: [PATCH 31/47] Add the usedforsecurity argument back --- Lib/hashlib.py | 4 ++- Modules/_hashopenssl.c | 82 +++++++++++++++++++++++++++++++----------- 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/Lib/hashlib.py b/Lib/hashlib.py index d3344f6..cd3b035 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -174,7 +174,9 @@ def __hash_new(name, data=b'', **kwargs): # salt, personal, tree hashing or SSE. return __get_builtin_constructor(name)(data, **kwargs) try: - retval = _hashlib.new(name, data) + usedforsecurity = kwargs.pop('usedforsecurity', True) + retval = _hashlib.new( + name, data, usedforsecurity=usedforsecurity) if get_fips_mode() and name != orig_name: retval._set_name(orig_name) return retval diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index e330423..b621c33 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -318,6 +318,25 @@ EVP_repr(EVPobject *self) return PyUnicode_FromFormat("<%U HASH object @ %p>", self->name, self); } + +static void +mc_ctx_init(EVP_MD_CTX *ctx, int usedforsecurity) +{ + EVP_MD_CTX_init(ctx); + /* + If the user has declared that this digest is being used in a + non-security role (e.g. indexing into a data structure), set + the exception flag for openssl to allow it + */ + if (!usedforsecurity) { +#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW + EVP_MD_CTX_set_flags(ctx, + EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); +#endif + } +} + + #if HASH_OBJ_CONSTRUCTOR static int EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds) @@ -328,9 +347,10 @@ EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds) Py_buffer view; char *nameStr; const EVP_MD *digest; + int usedforsecurity=1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:HASH", kwlist, - &name_obj, &data_obj)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O$d:HASH", kwlist, + &name_obj, &data_obj, &usedforsecurity)) { return -1; } @@ -351,7 +371,8 @@ EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds) PyBuffer_Release(&view); return -1; } - if (!EVP_DigestInit(self->ctx, digest)) { + mc_ctx_init(&self->ctx, usedforsecurity); + if (!EVP_DigestInit_ex(self->ctx, digest, NULL)) { _setException(PyExc_ValueError); if (data_obj) PyBuffer_Release(&view); @@ -440,7 +461,7 @@ static PyTypeObject EVPtype = { static PyObject * EVPnew(PyObject *name_obj, const EVP_MD *digest, const EVP_MD_CTX *initial_ctx, - const unsigned char *cp, Py_ssize_t len) + const unsigned char *cp, Py_ssize_t len, int usedforsecurity) { EVPobject *self; @@ -455,7 +476,8 @@ EVPnew(PyObject *name_obj, if (initial_ctx) { EVP_MD_CTX_copy(self->ctx, initial_ctx); } else { - if (!EVP_DigestInit(self->ctx, digest)) { + mc_ctx_init(self->ctx, usedforsecurity); + if (!EVP_DigestInit_ex(self->ctx, digest, NULL)) { _setException(PyExc_ValueError); Py_DECREF(self); return NULL; @@ -484,26 +506,35 @@ An optional string argument may be provided and will be\n\ automatically hashed.\n\ \n\ The MD5 and SHA1 algorithms are always supported.\n"); +static PyObject *_EVP_new_impl(PyObject *name_obj, char *name, PyObject *data_obj, int usedforsecurity); static PyObject * EVP_new(PyObject *self, PyObject *args, PyObject *kwdict) { - static char *kwlist[] = {"name", "string", NULL}; + static char *kwlist[] = {"name", "string", "usedforsecurity", NULL}; PyObject *name_obj = NULL; PyObject *data_obj = NULL; - Py_buffer view = { 0 }; - PyObject *ret_obj; - char *name; - const EVP_MD *digest; + int usedforsecurity = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|O:new", kwlist, - &name_obj, &data_obj)) { + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|O$p:new", kwlist, + &name_obj, &data_obj, &usedforsecurity)) { return NULL; } + return _EVP_new_impl(name_obj, NULL, data_obj, usedforsecurity); +} - if (!PyArg_Parse(name_obj, "s", &name)) { - PyErr_SetString(PyExc_TypeError, "name must be a string"); - return NULL; +static PyObject * +_EVP_new_impl(PyObject *name_obj, char *name, PyObject *data_obj, int usedforsecurity) +{ + Py_buffer view = { 0 }; + PyObject *ret_obj; + const EVP_MD *digest; + + if (!name) { + if (!PyArg_Parse(name_obj, "s", &name)) { + PyErr_SetString(PyExc_TypeError, "name must be a string"); + return NULL; + } } if (data_obj) @@ -511,7 +542,7 @@ EVP_new(PyObject *self, PyObject *args, PyObject *kwdict) digest = EVP_get_digestbyname(name); - ret_obj = EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf, view.len); + ret_obj = EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf, view.len, usedforsecurity); if (data_obj) PyBuffer_Release(&view); @@ -906,18 +937,27 @@ generate_hash_name_list(void) * code that wants to make hashes of a bunch of small strings. * The first call will lazy-initialize, which reports an exception * if initialization fails. + * + * Note that for usedforsecurity=0, we fall back to new(). */ #define GEN_CONSTRUCTOR(NAME, SSL_NAME) \ static PyObject * \ - EVP_new_ ## NAME (PyObject *self, PyObject *args) \ + EVP_new_ ## NAME (PyObject *self, PyObject *args, PyObject *kw) \ { \ PyObject *data_obj = NULL; \ Py_buffer view = { 0 }; \ PyObject *ret_obj; \ + int usedforsecurity = 1; \ + char *kwnames[] = {"", "usedforsecurity", NULL}; \ \ - if (!PyArg_ParseTuple(args, "|O:" #NAME , &data_obj)) { \ + if (!PyArg_ParseTupleAndKeywords(args, kw, "|O$p:" #NAME, kwnames, &data_obj, &usedforsecurity)) { \ return NULL; \ } \ + if (!usedforsecurity) { \ + return _EVP_new_impl( \ + CONST_ ## NAME ## _name_obj, SSL_NAME, \ + data_obj, usedforsecurity); \ + } \ \ if (CONST_new_ ## NAME ## _ctx_p == NULL) { \ EVP_MD_CTX *ctx_p = EVP_MD_CTX_new(); \ @@ -938,7 +978,8 @@ generate_hash_name_list(void) NULL, \ CONST_new_ ## NAME ## _ctx_p, \ (unsigned char*)view.buf, \ - view.len); \ + view.len, \ + usedforsecurity); \ \ if (data_obj) \ PyBuffer_Release(&view); \ @@ -947,7 +988,8 @@ generate_hash_name_list(void) /* a PyMethodDef structure for the constructor */ #define CONSTRUCTOR_METH_DEF(NAME) \ - {"openssl_" #NAME, (PyCFunction)EVP_new_ ## NAME, METH_VARARGS, \ + {"openssl_" #NAME, (PyCFunction)EVP_new_ ## NAME, \ + METH_VARARGS|METH_KEYWORDS, \ PyDoc_STR("Returns a " #NAME \ " hash object; optionally initialized with a string") \ }, -- 2.25.4 From 90cdc1df8818f62425ed70841a977e3b4ae617db Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2019 18:59:15 +0200 Subject: [PATCH 32/47] Add no-op usedforsecurity argument to internal hash implementations --- Modules/_blake2/blake2b_impl.c | 6 ++++-- Modules/_blake2/blake2s_impl.c | 6 ++++-- Modules/_blake2/clinic/blake2b_impl.c.h | 16 +++++++++------- Modules/_blake2/clinic/blake2s_impl.c.h | 16 +++++++++------- Modules/_sha3/sha3module.c | 5 +++++ 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c index f6bfce8..ae9d244 100644 --- a/Modules/_blake2/blake2b_impl.c +++ b/Modules/_blake2/blake2b_impl.c @@ -87,6 +87,8 @@ _blake2.blake2b.__new__ as py_blake2b_new inner_size: int = 0 last_node: bool = False + usedforsecurity: bool = True + Return a new BLAKE2b hash object. [clinic start generated code]*/ @@ -95,8 +97,8 @@ py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, PyObject *leaf_size_obj, PyObject *node_offset_obj, int node_depth, - int inner_size, int last_node) -/*[clinic end generated code: output=7506d8d890e5f13b input=aca35b33c5612b4b]*/ + int inner_size, int last_node, int usedforsecurity) +/*[clinic end generated code: output=02dcc52ee784622b input=c5dfcb847f9065ac]*/ { BLAKE2bObject *self = NULL; Py_buffer buf; diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c index 28ae5b6..bf80d6b 100644 --- a/Modules/_blake2/blake2s_impl.c +++ b/Modules/_blake2/blake2s_impl.c @@ -87,6 +87,8 @@ _blake2.blake2s.__new__ as py_blake2s_new inner_size: int = 0 last_node: bool = False + usedforsecurity: bool = True + Return a new BLAKE2s hash object. [clinic start generated code]*/ @@ -95,8 +97,8 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, PyObject *leaf_size_obj, PyObject *node_offset_obj, int node_depth, - int inner_size, int last_node) -/*[clinic end generated code: output=fe060b258a8cbfc6 input=3abfaabe7f5f62cc]*/ + int inner_size, int last_node, int usedforsecurity) +/*[clinic end generated code: output=e32ea5e22d54db91 input=af5344c57efd870d]*/ { BLAKE2sObject *self = NULL; Py_buffer buf; diff --git a/Modules/_blake2/clinic/blake2b_impl.c.h b/Modules/_blake2/clinic/blake2b_impl.c.h index 9b2965e..9688c04 100644 --- a/Modules/_blake2/clinic/blake2b_impl.c.h +++ b/Modules/_blake2/clinic/blake2b_impl.c.h @@ -5,7 +5,8 @@ preserve PyDoc_STRVAR(py_blake2b_new__doc__, "blake2b(data=b\'\', /, *, digest_size=_blake2.blake2b.MAX_DIGEST_SIZE,\n" " key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" -" node_offset=0, node_depth=0, inner_size=0, last_node=False)\n" +" node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" +" usedforsecurity=True)\n" "--\n" "\n" "Return a new BLAKE2b hash object."); @@ -15,14 +16,14 @@ py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, PyObject *leaf_size_obj, PyObject *node_offset_obj, int node_depth, - int inner_size, int last_node); + int inner_size, int last_node, int usedforsecurity); static PyObject * py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", NULL}; - static _PyArg_Parser _parser = {"|O$iy*y*y*iiOOiip:blake2b", _keywords, 0}; + static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = {"|O$iy*y*y*iiOOiipp:blake2b", _keywords, 0}; PyObject *data = NULL; int digest_size = BLAKE2B_OUTBYTES; Py_buffer key = {NULL, NULL}; @@ -35,12 +36,13 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int node_depth = 0; int inner_size = 0; int last_node = 0; + int usedforsecurity = 1; if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, - &data, &digest_size, &key, &salt, &person, &fanout, &depth, &leaf_size_obj, &node_offset_obj, &node_depth, &inner_size, &last_node)) { + &data, &digest_size, &key, &salt, &person, &fanout, &depth, &leaf_size_obj, &node_offset_obj, &node_depth, &inner_size, &last_node, &usedforsecurity)) { goto exit; } - return_value = py_blake2b_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size_obj, node_offset_obj, node_depth, inner_size, last_node); + return_value = py_blake2b_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size_obj, node_offset_obj, node_depth, inner_size, last_node, usedforsecurity); exit: /* Cleanup for key */ @@ -121,4 +123,4 @@ _blake2_blake2b_hexdigest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2b_hexdigest_impl(self); } -/*[clinic end generated code: output=0eb559f418fc0a21 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d5f214b052c75135 input=a9049054013a1b77]*/ diff --git a/Modules/_blake2/clinic/blake2s_impl.c.h b/Modules/_blake2/clinic/blake2s_impl.c.h index 42b87b7..5653e93 100644 --- a/Modules/_blake2/clinic/blake2s_impl.c.h +++ b/Modules/_blake2/clinic/blake2s_impl.c.h @@ -5,7 +5,8 @@ preserve PyDoc_STRVAR(py_blake2s_new__doc__, "blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" " key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" -" node_offset=0, node_depth=0, inner_size=0, last_node=False)\n" +" node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" +" usedforsecurity=True)\n" "--\n" "\n" "Return a new BLAKE2s hash object."); @@ -15,14 +16,14 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, PyObject *leaf_size_obj, PyObject *node_offset_obj, int node_depth, - int inner_size, int last_node); + int inner_size, int last_node, int usedforsecurity); static PyObject * py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", NULL}; - static _PyArg_Parser _parser = {"|O$iy*y*y*iiOOiip:blake2s", _keywords, 0}; + static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = {"|O$iy*y*y*iiOOiipp:blake2s", _keywords, 0}; PyObject *data = NULL; int digest_size = BLAKE2S_OUTBYTES; Py_buffer key = {NULL, NULL}; @@ -35,12 +36,13 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int node_depth = 0; int inner_size = 0; int last_node = 0; + int usedforsecurity = 1; if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, - &data, &digest_size, &key, &salt, &person, &fanout, &depth, &leaf_size_obj, &node_offset_obj, &node_depth, &inner_size, &last_node)) { + &data, &digest_size, &key, &salt, &person, &fanout, &depth, &leaf_size_obj, &node_offset_obj, &node_depth, &inner_size, &last_node, &usedforsecurity)) { goto exit; } - return_value = py_blake2s_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size_obj, node_offset_obj, node_depth, inner_size, last_node); + return_value = py_blake2s_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size_obj, node_offset_obj, node_depth, inner_size, last_node, usedforsecurity); exit: /* Cleanup for key */ @@ -121,4 +123,4 @@ _blake2_blake2s_hexdigest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2s_hexdigest_impl(self); } -/*[clinic end generated code: output=13d4b08ea9ee2d62 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2373a3b3fa542e89 input=a9049054013a1b77]*/ diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c index 2783a75..62db9cb 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/_sha3/sha3module.c @@ -185,6 +185,11 @@ py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) HashReturn res; PyObject *data = NULL; + // Ignore "usedforsecurity" + if (PyMapping_DelItemString(kwargs, "usedforsecurity")) { + PyErr_Clear(); + } + if (!_PyArg_NoKeywords(type->tp_name, kwargs)) { return NULL; } -- 2.25.4 From bd0b941369ff30164065db71777e21cf6813dc40 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2019 19:09:39 +0200 Subject: [PATCH 33/47] Test the usedforsecurity flag --- Lib/test/test_hashlib.py | 82 ++++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 28 deletions(-) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 9745bfd..19a2fbd 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -24,6 +24,7 @@ from test import support from test.support import _4G, bigmemtest, import_fresh_module from test.support import requires_hashdigest from http.client import HTTPException +from functools import partial # Were we compiled --with-pydebug or with #define Py_DEBUG? COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount') @@ -32,8 +33,10 @@ c_hashlib = import_fresh_module('hashlib', fresh=['_hashlib']) py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) if hashlib.get_fips_mode(): - FIPS_DISABLED = {'md5', 'MD5', 'blake2b', 'blake2s'} + FIPS_UNAVAILABLE = {'blake2b', 'blake2s'} + FIPS_DISABLED = {'md5', 'MD5', *FIPS_UNAVAILABLE} else: + FIPS_UNAVAILABLE = set() FIPS_DISABLED = set() try: @@ -78,6 +81,15 @@ def read_vectors(hash_name): yield parts +def _is_openssl_constructor(constructor): + if getattr(constructor, '__name__', '').startswith('openssl_'): + return True + if isinstance(constructor, partial): + if constructor.func.__name__.startswith('openssl_'): + return True + return False + + class HashLibTestCase(unittest.TestCase): supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1', 'sha224', 'SHA224', 'sha256', 'SHA256', @@ -93,8 +105,6 @@ class HashLibTestCase(unittest.TestCase): if hashlib.get_fips_mode(): shakes = set() - supported_hash_names = tuple( - n for n in supported_hash_names if n not in FIPS_DISABLED) def _conditional_import_module(self, module_name): """Import a module and return a reference to it or None on failure.""" @@ -120,7 +130,8 @@ class HashLibTestCase(unittest.TestCase): def __init__(self, *args, **kwargs): algorithms = set() for algorithm in self.supported_hash_names: - algorithms.add(algorithm.lower()) + if algorithm not in FIPS_UNAVAILABLE: + algorithms.add(algorithm.lower()) _blake2 = self._conditional_import_module('_blake2') if _blake2: @@ -130,15 +141,21 @@ class HashLibTestCase(unittest.TestCase): for algorithm in algorithms: self.constructors_to_test[algorithm] = set() + def _add_constructor(algorithm, constructor): + constructors.add(partial(constructor, usedforsecurity=False)) + if algorithm not in FIPS_DISABLED: + constructors.add(constructor) + constructors.add(partial(constructor, usedforsecurity=True)) + # For each algorithm, test the direct constructor and the use # of hashlib.new given the algorithm name. for algorithm, constructors in self.constructors_to_test.items(): - constructors.add(getattr(hashlib, algorithm)) + _add_constructor(algorithm, getattr(hashlib, algorithm)) def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs): if data is None: return hashlib.new(_alg, **kwargs) return hashlib.new(_alg, data, **kwargs) - constructors.add(_test_algorithm_via_hashlib_new) + _add_constructor(algorithm, _test_algorithm_via_hashlib_new) _hashlib = self._conditional_import_module('_hashlib') self._hashlib = _hashlib @@ -150,7 +167,7 @@ class HashLibTestCase(unittest.TestCase): for algorithm, constructors in self.constructors_to_test.items(): constructor = getattr(_hashlib, 'openssl_'+algorithm, None) if constructor: - constructors.add(constructor) + _add_constructor(algorithm, constructor) def add_builtin_constructor(name): constructor = getattr(hashlib, "__get_builtin_constructor")(name) @@ -210,7 +227,7 @@ class HashLibTestCase(unittest.TestCase): c.hexdigest() def test_algorithms_guaranteed(self): - self.assertEqual(hashlib.algorithms_guaranteed - FIPS_DISABLED, + self.assertEqual(hashlib.algorithms_guaranteed, set(_algo for _algo in self.supported_hash_names if _algo.islower())) @@ -286,7 +303,9 @@ class HashLibTestCase(unittest.TestCase): self.assertIn(h.name, self.supported_hash_names) else: self.assertNotIn(h.name, self.supported_hash_names) - self.assertEqual(h.name, hashlib.new(h.name).name) + if h.name not in FIPS_DISABLED: + self.assertEqual(h.name, hashlib.new(h.name).name) + self.assertEqual(h.name, hashlib.new(h.name, usedforsecurity=False).name) def test_large_update(self): aas = b'a' * 128 @@ -328,13 +347,15 @@ class HashLibTestCase(unittest.TestCase): self.assertGreaterEqual(len(constructors), 2) for hash_object_constructor in constructors: if ( - kwargs - and hash_object_constructor.__name__.startswith('openssl_') + (kwargs.keys() - {'usedforsecurity'}) + and _is_openssl_constructor(hash_object_constructor) ): + # Don't check openssl constructors with + # any extra keys (except usedforsecurity) return m = hash_object_constructor(data, **kwargs) if shake: - if hash_object_constructor.__name__.startswith('openssl_'): + if _is_openssl_constructor(hash_object_constructor): if length > m.digest_size: # OpenSSL doesn't give long digests return @@ -351,7 +372,7 @@ class HashLibTestCase(unittest.TestCase): % (name, hash_object_constructor, computed, len(data), hexdigest)) if shake: - if hash_object_constructor.__name__.startswith('openssl_'): + if _is_openssl_constructor(hash_object_constructor): computed = m.digest()[:length] else: computed = m.digest(length) @@ -369,8 +390,7 @@ class HashLibTestCase(unittest.TestCase): self.assertRaises(TypeError, hash_object_constructor, 'spam') def test_no_unicode(self): - if not hashlib.get_fips_mode(): - self.check_no_unicode('md5') + self.check_no_unicode('md5') self.check_no_unicode('sha1') self.check_no_unicode('sha224') self.check_no_unicode('sha256') @@ -397,10 +417,10 @@ class HashLibTestCase(unittest.TestCase): for hash_object_constructor in constructors: m = hash_object_constructor() self.assertEqual(m.block_size, block_size) - if not hash_object_constructor.__name__.startswith('openssl_'): + if not _is_openssl_constructor(hash_object_constructor): self.assertEqual(m.digest_size, digest_size) if digest_length: - if not hash_object_constructor.__name__.startswith('openssl_'): + if not _is_openssl_constructor(hash_object_constructor): self.assertEqual(len(m.digest(digest_length)), digest_length) self.assertEqual(len(m.hexdigest(digest_length)), @@ -435,7 +455,7 @@ class HashLibTestCase(unittest.TestCase): for hash_object_constructor in constructors: m = hash_object_constructor() self.assertEqual(capacity + rate, 1600) - if not hash_object_constructor.__name__.startswith('openssl_'): + if not _is_openssl_constructor(hash_object_constructor): self.assertEqual(m._capacity_bits, capacity) self.assertEqual(m._rate_bits, rate) self.assertEqual(m._suffix, suffix) @@ -454,31 +474,27 @@ class HashLibTestCase(unittest.TestCase): self.check_blocksize_name('blake2b', 128, 64) self.check_blocksize_name('blake2s', 64, 32) - @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") def test_case_md5_0(self): - self.check('md5', b'', 'd41d8cd98f00b204e9800998ecf8427e') + self.check('md5', b'', 'd41d8cd98f00b204e9800998ecf8427e', usedforsecurity=False) - @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") def test_case_md5_1(self): - self.check('md5', b'abc', '900150983cd24fb0d6963f7d28e17f72') + self.check('md5', b'abc', '900150983cd24fb0d6963f7d28e17f72', usedforsecurity=False) - @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") def test_case_md5_2(self): self.check('md5', b'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', - 'd174ab98d277d9f5a5611c2c9f419d9f') + 'd174ab98d277d9f5a5611c2c9f419d9f', + usedforsecurity=False) - @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") @unittest.skipIf(sys.maxsize < _4G + 5, 'test cannot run on 32-bit systems') @bigmemtest(size=_4G + 5, memuse=1, dry_run=False) def test_case_md5_huge(self, size): - self.check('md5', b'A'*size, 'c9af2dff37468ce5dfee8f2cfc0a9c6d') + self.check('md5', b'A'*size, 'c9af2dff37468ce5dfee8f2cfc0a9c6d', usedforsecurity=False) - @unittest.skipIf(hashlib.get_fips_mode(), "md5 unacceptable in FIPS mode") @unittest.skipIf(sys.maxsize < _4G - 1, 'test cannot run on 32-bit systems') @bigmemtest(size=_4G - 1, memuse=1, dry_run=False) def test_case_md5_uintmax(self, size): - self.check('md5', b'A'*size, '28138d306ff1b8281f1a9067e1a1a2b3') + self.check('md5', b'A'*size, '28138d306ff1b8281f1a9067e1a1a2b3', usedforsecurity=False) # use the three examples from Federal Information Processing Standards # Publication 180-1, Secure Hash Standard, 1995 April 17 @@ -904,6 +920,16 @@ class HashLibTestCase(unittest.TestCase): self.assertEqual(expected_hash, hasher.hexdigest()) + @unittest.skipUnless(hashlib.get_fips_mode(), 'Needs FIPS mode.') + def test_usedforsecurity_repeat(self): + """Make sure usedforsecurity flag isn't copied to other contexts""" + for i in range(3): + for cons in hashlib.md5, partial(hashlib.new, 'md5'): + self.assertRaises(ValueError, cons) + self.assertRaises(ValueError, partial(cons, usedforsecurity=True)) + self.assertEqual(cons(usedforsecurity=False).hexdigest(), + 'd41d8cd98f00b204e9800998ecf8427e') + class KDFTests(unittest.TestCase): -- 2.25.4 From e59140ddc48ba822792a1726221e626c96f31ec7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 29 Aug 2019 10:25:28 +0200 Subject: [PATCH 34/47] Skip error checking in _hashlib.get_fips_mode Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1745499 --- Modules/_hashopenssl.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index b621c33..6bfb12c 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -1021,20 +1021,22 @@ static PyObject * _hashlib_get_fips_mode_impl(PyObject *module) /*[clinic end generated code: output=ad8a7793310d3f98 input=f42a2135df2a5e11]*/ { - int result = FIPS_mode(); - if (result == 0) { - // "If the library was built without support of the FIPS Object Module, - // then the function will return 0 with an error code of - // CRYPTO_R_FIPS_MODE_NOT_SUPPORTED (0x0f06d065)." - // But 0 is also a valid result value. - - unsigned long errcode = ERR_peek_last_error(); - if (errcode) { - _setException(PyExc_ValueError); - return NULL; - } - } - return PyLong_FromLong(result); + // XXX: This function skips error checking. + // This is only appropriate for RHEL. + + // From the OpenSSL docs: + // "If the library was built without support of the FIPS Object Module, + // then the function will return 0 with an error code of + // CRYPTO_R_FIPS_MODE_NOT_SUPPORTED (0x0f06d065)." + // In RHEL: + // * we do build with FIPS, so the function always succeeds + // * even if it didn't, people seem used to errors being left on the + // OpenSSL error stack. + + // For more info, see: + // https://bugzilla.redhat.com/show_bug.cgi?id=1745499 + + return PyLong_FromLong(FIPS_mode()); } -- 2.25.4 From d97fb77472776eec19828c1b158c5bf6beb302e6 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 10 Oct 2019 13:04:50 +0200 Subject: [PATCH 35/47] Skip error checking in _Py_hashlib_fips_error https://bugzilla.redhat.com/show_bug.cgi?id=1760106 --- Include/_hashopenssl.h | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/Include/_hashopenssl.h b/Include/_hashopenssl.h index 47ed003..d4cbdef 100644 --- a/Include/_hashopenssl.h +++ b/Include/_hashopenssl.h @@ -42,16 +42,10 @@ static int _Py_hashlib_fips_error(PyObject *exc, char *name) { int result = FIPS_mode(); if (result == 0) { - // "If the library was built without support of the FIPS Object Module, - // then the function will return 0 with an error code of - // CRYPTO_R_FIPS_MODE_NOT_SUPPORTED (0x0f06d065)." - // But 0 is also a valid result value. + // XXX: This function skips error checking. + // This is only appropriate for RHEL. + // See _hashlib.get_fips_mode for details. - unsigned long errcode = ERR_peek_last_error(); - if (errcode) { - _setException(exc); - return 1; - } return 0; } PyErr_Format(exc, "%s is not available in FIPS mode", name); -- 2.25.4 From b0b51e9ffff8b60da14bffaa0819a364f84d9f6c Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Thu, 21 Nov 2019 00:20:09 +0100 Subject: [PATCH 36/47] Use usedforsecurity=False in uuid Partially backport commit 7cad53e6b084435a220e6604010f1fa5778bd0b1 only for uuid --- Lib/uuid.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index db8b2ef..7e680a2 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -615,8 +615,11 @@ def uuid1(node=None, clock_seq=None): def uuid3(namespace, name): """Generate a UUID from the MD5 hash of a namespace UUID and a name.""" from hashlib import md5 - hash = md5(namespace.bytes + bytes(name, "utf-8")).digest() - return UUID(bytes=hash[:16], version=3) + digest = md5( + namespace.bytes + bytes(name, "utf-8"), + usedforsecurity=False + ).digest() + return UUID(bytes=digest[:16], version=3) def uuid4(): """Generate a random UUID.""" -- 2.25.4 From 47478e3b19d3a884ecd42bd3eee5b7dbb5395feb Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 5 Aug 2019 19:12:38 +0200 Subject: [PATCH 37/47] Fixups - Adjust error message of the original hmac.HMAC class - Don't duplicate a test name --- Lib/hmac.py | 2 +- Lib/test/test_hmac.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/hmac.py b/Lib/hmac.py index 7af94c3..33b5be6 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -41,7 +41,7 @@ class HMAC: """ if _hashlib.get_fips_mode(): raise ValueError( - 'hmac.HMAC is not available in FIPS mode. ' + 'This class is not available in FIPS mode. ' + 'Use hmac.new().' ) diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 5b09072..449b136 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -401,7 +401,7 @@ class CopyTestCase(unittest.TestCase): self.assertTrue(id(h1.outer) != id(h2.outer), "No real copy of the attribute 'outer'.") - def test_realcopy(self): + def test_realcopy_by_digest(self): # Testing if the copy method created a real copy. h1 = hmac.HMAC(b"key", digestmod="sha1") h2 = h1.copy() -- 2.25.4 From 50a0b40901940e6e0c8e4ef89d41955c4158a2d7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2019 19:39:48 +0200 Subject: [PATCH 38/47] Don't re-export get_fips_mode from hashlib Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1745685 --- Lib/distutils/command/upload.py | 3 ++- Lib/distutils/tests/test_upload.py | 5 +++-- Lib/hashlib.py | 23 ++++++++++++----------- Lib/hmac.py | 6 +++--- Lib/test/test_fips.py | 6 +++--- Lib/test/test_hashlib.py | 16 +++++++++------- Lib/test/test_hmac.py | 8 +++++--- Lib/test/test_logging.py | 1 + Lib/test/test_smtplib.py | 4 +++- Lib/test/test_tools/test_md5sum.py | 4 ++-- Lib/test/test_urllib2_localnet.py | 1 + 11 files changed, 44 insertions(+), 33 deletions(-) diff --git a/Lib/distutils/command/upload.py b/Lib/distutils/command/upload.py index 170acb3..d0a4aee 100644 --- a/Lib/distutils/command/upload.py +++ b/Lib/distutils/command/upload.py @@ -126,7 +126,8 @@ class upload(PyPIRCCommand): except ValueError as e: msg = 'calculating md5 checksum failed: %s' % e self.announce(msg, log.INFO) - if not hashlib.get_fips_mode(): + from _hashlib import get_fips_mode + if not get_fips_mode(): # this really shouldn't fail raise else: diff --git a/Lib/distutils/tests/test_upload.py b/Lib/distutils/tests/test_upload.py index f720a79..a198b21 100644 --- a/Lib/distutils/tests/test_upload.py +++ b/Lib/distutils/tests/test_upload.py @@ -4,6 +4,7 @@ import unittest import unittest.mock as mock from urllib.request import HTTPError import hashlib +from _hashlib import get_fips_mode from test.support import run_unittest @@ -131,7 +132,7 @@ class uploadTestCase(BasePyPIRCCommandTestCase): # what did we send ? headers = dict(self.last_open.req.headers) - if hashlib.get_fips_mode(): + if get_fips_mode(): # only sha256 hash is used self.assertEqual(headers['Content-length'], '2197') else: @@ -172,7 +173,7 @@ class uploadTestCase(BasePyPIRCCommandTestCase): cmd.run() headers = dict(self.last_open.req.headers) - if hashlib.get_fips_mode(): + if get_fips_mode(): # only sha256 hash is used self.assertEqual(headers['Content-length'], '2207') else: diff --git a/Lib/hashlib.py b/Lib/hashlib.py index cd3b035..3e9a4aa 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -68,13 +68,13 @@ __all__ = __always_supported + ('new', 'algorithms_guaranteed', 'algorithms_available', 'pbkdf2_hmac') try: - from _hashlib import get_fips_mode + from _hashlib import get_fips_mode as _hashlib_get_fips_mode except ImportError: - def get_fips_mode(): + def _hashlib_get_fips_mode(): return 0 -if not get_fips_mode(): +if not _hashlib_get_fips_mode(): __builtin_constructor_cache = {} def __get_builtin_constructor(name): @@ -121,7 +121,7 @@ if not get_fips_mode(): def __get_openssl_constructor(name): - if not get_fips_mode(): + if not _hashlib.get_fips_mode(): if name in { 'blake2b', 'blake2s', 'shake_256', 'shake_128', #'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', @@ -132,7 +132,7 @@ def __get_openssl_constructor(name): f = getattr(_hashlib, 'openssl_' + name) # Allow the C module to raise ValueError. The function will be # defined but the hash not actually available thanks to OpenSSL. - if not get_fips_mode(): + if not _hashlib.get_fips_mode(): # N.B. In "FIPS mode", there is no fallback. # If this test fails, we want to export the broken hash # constructor anyway. @@ -142,7 +142,7 @@ def __get_openssl_constructor(name): except (AttributeError, ValueError): return __get_builtin_constructor(name) -if not get_fips_mode(): +if not _hashlib_get_fips_mode(): def __py_new(name, data=b'', **kwargs): """new(name, data=b'', **kwargs) - Return a new hashing object using the named algorithm; optionally initialized with data (which must be @@ -155,7 +155,7 @@ def __hash_new(name, data=b'', **kwargs): """new(name, data=b'') - Return a new hashing object using the named algorithm; optionally initialized with data (which must be a bytes-like object). """ - if get_fips_mode(): + if _hashlib.get_fips_mode(): # Use OpenSSL names for Python built-in hashes orig_name = name name = { @@ -177,7 +177,7 @@ def __hash_new(name, data=b'', **kwargs): usedforsecurity = kwargs.pop('usedforsecurity', True) retval = _hashlib.new( name, data, usedforsecurity=usedforsecurity) - if get_fips_mode() and name != orig_name: + if _hashlib.get_fips_mode() and name != orig_name: retval._set_name(orig_name) return retval except ValueError: @@ -185,7 +185,7 @@ def __hash_new(name, data=b'', **kwargs): # hash, try using our builtin implementations. # This allows for SHA224/256 and SHA384/512 support even though # the OpenSSL library prior to 0.9.8 doesn't provide them. - if get_fips_mode(): + if _hashlib.get_fips_mode(): raise return __get_builtin_constructor(name)(data) @@ -197,7 +197,7 @@ try: algorithms_available = algorithms_available.union( _hashlib.openssl_md_meth_names) except ImportError: - if get_fips_mode(): + if _hashlib_get_fips_mode(): raise new = __py_new __get_hash = __get_builtin_constructor @@ -225,5 +225,6 @@ for __func_name in __always_supported: # Cleanup locals() del __always_supported, __func_name, __get_hash del __hash_new, __get_openssl_constructor -if not get_fips_mode(): +if not _hashlib.get_fips_mode(): del __py_new +del _hashlib_get_fips_mode diff --git a/Lib/hmac.py b/Lib/hmac.py index 33b5be6..ca83d9d 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -39,7 +39,7 @@ class HMAC: Note: key and msg must be a bytes or bytearray objects. """ - if _hashlib.get_fips_mode(): + if _hashlibopenssl.get_fips_mode(): raise ValueError( 'This class is not available in FIPS mode. ' + 'Use hmac.new().' @@ -97,7 +97,7 @@ class HMAC: def update(self, msg): """Update this hashing object with the string msg. """ - if _hashlib.get_fips_mode(): + if _hashlibopenssl.get_fips_mode(): raise ValueError('hmac.HMAC is not available in FIPS mode') self.inner.update(msg) @@ -167,7 +167,7 @@ class HMAC_openssl(_hmacopenssl.HMAC): return result -if _hashlib.get_fips_mode(): +if _hashlibopenssl.get_fips_mode(): HMAC = HMAC_openssl diff --git a/Lib/test/test_fips.py b/Lib/test/test_fips.py index 34812e6..86e61e2 100644 --- a/Lib/test/test_fips.py +++ b/Lib/test/test_fips.py @@ -6,7 +6,7 @@ import hashlib, _hashlib class HashlibFipsTests(unittest.TestCase): - @unittest.skipUnless(hashlib.get_fips_mode(), "Test only when FIPS is enabled") + @unittest.skipUnless(_hashlib.get_fips_mode(), "Test only when FIPS is enabled") def test_fips_imports(self): """blake2s and blake2b should fail to import in FIPS mode """ @@ -30,7 +30,7 @@ class HashlibFipsTests(unittest.TestCase): self.assertEqual(m, h) - @unittest.skipIf(hashlib.get_fips_mode(), "blake2 hashes are not available under FIPS") + @unittest.skipIf(_hashlib.get_fips_mode(), "blake2 hashes are not available under FIPS") def test_blake2_hashes(self): self.compare_hashes(hashlib.blake2b(b'abc'), _hashlib.openssl_blake2b(b'abc')) self.compare_hashes(hashlib.blake2s(b'abc'), _hashlib.openssl_blake2s(b'abc')) @@ -41,7 +41,7 @@ class HashlibFipsTests(unittest.TestCase): self.compare_hashes(hashlib.sha3_384(b'abc'), _hashlib.openssl_sha3_384(b'abc')) self.compare_hashes(hashlib.sha3_512(b'abc'), _hashlib.openssl_sha3_512(b'abc')) - @unittest.skipIf(hashlib.get_fips_mode(), "shake hashes are not available under FIPS") + @unittest.skipIf(_hashlib.get_fips_mode(), "shake hashes are not available under FIPS") def test_shake_hashes(self): self.compare_hashes(hashlib.shake_128(b'abc'), _hashlib.openssl_shake_128(b'abc')) self.compare_hashes(hashlib.shake_256(b'abc'), _hashlib.openssl_shake_256(b'abc')) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 19a2fbd..a6f9a35 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -32,7 +32,9 @@ COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount') c_hashlib = import_fresh_module('hashlib', fresh=['_hashlib']) py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) -if hashlib.get_fips_mode(): +from _hashlib import get_fips_mode as _get_fips_mode + +if _get_fips_mode(): FIPS_UNAVAILABLE = {'blake2b', 'blake2s'} FIPS_DISABLED = {'md5', 'MD5', *FIPS_UNAVAILABLE} else: @@ -103,7 +105,7 @@ class HashLibTestCase(unittest.TestCase): # Issue #14693: fallback modules are always compiled under POSIX _warn_on_extension_import = os.name == 'posix' or COMPILED_WITH_PYDEBUG - if hashlib.get_fips_mode(): + if _get_fips_mode(): shakes = set() def _conditional_import_module(self, module_name): @@ -114,7 +116,7 @@ class HashLibTestCase(unittest.TestCase): if self._warn_on_extension_import: warnings.warn('Did a C extension fail to compile? %s' % error) except ImportError as error: - if not hashlib.get_fips_mode(): + if not _get_fips_mode(): raise return None @@ -239,12 +241,12 @@ class HashLibTestCase(unittest.TestCase): self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam') self.assertRaises(TypeError, hashlib.new, 1) - @unittest.skipUnless(hashlib.get_fips_mode(), "Builtin constructor only unavailable in FIPS mode") + @unittest.skipUnless(_get_fips_mode(), "Builtin constructor only unavailable in FIPS mode") def test_get_builtin_constructor_fips(self): with self.assertRaises(AttributeError): hashlib.__get_builtin_constructor - @unittest.skipIf(hashlib.get_fips_mode(), "No builtin constructors in FIPS mode") + @unittest.skipIf(_get_fips_mode(), "No builtin constructors in FIPS mode") def test_get_builtin_constructor(self): get_builtin_constructor = getattr(hashlib, '__get_builtin_constructor') @@ -433,7 +435,7 @@ class HashLibTestCase(unittest.TestCase): self.assertIn(name.split("_")[0], repr(m)) def test_blocksize_name(self): - if not hashlib.get_fips_mode(): + if not _get_fips_mode(): self.check_blocksize_name('md5', 64, 16) self.check_blocksize_name('sha1', 64, 20) self.check_blocksize_name('sha224', 64, 28) @@ -920,7 +922,7 @@ class HashLibTestCase(unittest.TestCase): self.assertEqual(expected_hash, hasher.hexdigest()) - @unittest.skipUnless(hashlib.get_fips_mode(), 'Needs FIPS mode.') + @unittest.skipUnless(_get_fips_mode(), 'Needs FIPS mode.') def test_usedforsecurity_repeat(self): """Make sure usedforsecurity flag isn't copied to other contexts""" for i in range(3): diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 449b136..35b8935 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -3,6 +3,7 @@ import hmac import hashlib import unittest import warnings +from _hashlib import get_fips_mode from test.support import requires_hashdigest @@ -250,7 +251,7 @@ class TestVectorsTestCase(unittest.TestCase): def test_sha512_rfc4231(self): self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128) - @unittest.skipIf(hashlib.get_fips_mode(), 'MockCrazyHash unacceptable in FIPS mode.') + @unittest.skipIf(get_fips_mode(), 'MockCrazyHash unacceptable in FIPS mode.') @requires_hashdigest('sha256') def test_legacy_block_size_warnings(self): class MockCrazyHash(object): @@ -274,6 +275,7 @@ class TestVectorsTestCase(unittest.TestCase): hmac.HMAC(b'a', b'b', digestmod=MockCrazyHash) self.fail('Expected warning about small block_size') + @unittest.skipIf(get_fips_mode(), 'md5 is not default in FIPS mode.') def test_with_digestmod_warning(self): with self.assertWarns(PendingDeprecationWarning): key = b"\x0b" * 16 @@ -375,7 +377,7 @@ class SanityTestCase(unittest.TestCase): class CopyTestCase(unittest.TestCase): - @unittest.skipIf(hashlib.get_fips_mode(), "Internal attributes unavailable in FIPS mode") + @unittest.skipIf(get_fips_mode(), "Internal attributes unavailable in FIPS mode") @requires_hashdigest('sha256') def test_attributes(self): # Testing if attributes are of same type. @@ -388,7 +390,7 @@ class CopyTestCase(unittest.TestCase): self.assertEqual(type(h1.outer), type(h2.outer), "Types of outer don't match.") - @unittest.skipIf(hashlib.get_fips_mode(), "Internal attributes unavailable in FIPS mode") + @unittest.skipIf(get_fips_mode(), "Internal attributes unavailable in FIPS mode") @requires_hashdigest('sha256') def test_realcopy(self): # Testing if the copy method created a real copy. diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index d5c63b4..45b72e3 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -46,6 +46,7 @@ import time import unittest import warnings import weakref + try: import threading # The following imports are needed only for tests which diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 64b3201..704b2bc 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -16,6 +16,8 @@ import time import select import errno import textwrap +import hashlib +from _hashlib import get_fips_mode import unittest from test import support, mock_socket @@ -980,7 +982,7 @@ class SMTPSimTests(unittest.TestCase): def testAUTH_multiple(self): # Test that multiple authentication methods are tried. - self.serv.add_feature("AUTH BOGUS PLAIN LOGIN CRAM-MD5") + self.serv.add_feature("AUTH BOGUS PLAIN LOGIN") smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) resp = smtp.login(sim_auth[0], sim_auth[1]) self.assertEqual(resp, (235, b'Authentication Succeeded')) diff --git a/Lib/test/test_tools/test_md5sum.py b/Lib/test/test_tools/test_md5sum.py index 7028a4d..3ba1ca0 100644 --- a/Lib/test/test_tools/test_md5sum.py +++ b/Lib/test/test_tools/test_md5sum.py @@ -4,13 +4,13 @@ import os import unittest from test import support from test.support.script_helper import assert_python_ok, assert_python_failure -import hashlib +import _hashlib from test.test_tools import scriptsdir, skip_if_missing skip_if_missing() -if hashlib.get_fips_mode(): +if _hashlib.get_fips_mode(): raise unittest.SkipTest("md5sum won't work at all in FIPS mode") class MD5SumTests(unittest.TestCase): diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index 895f97c..b4fb13a 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -6,6 +6,7 @@ import urllib.request import http.server import unittest import hashlib +from _hashlib import get_fips_mode from test import support -- 2.25.4 From 0e3b2641d41349fb2847d2d41bdad42b8fd39178 Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Wed, 6 May 2020 06:27:04 +0200 Subject: [PATCH 39/47] Pass kwargs (like usedforsecurity) through __hash_new --- Lib/hashlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 3e9a4aa..9bc90a4 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -187,7 +187,7 @@ def __hash_new(name, data=b'', **kwargs): # the OpenSSL library prior to 0.9.8 doesn't provide them. if _hashlib.get_fips_mode(): raise - return __get_builtin_constructor(name)(data) + return __get_builtin_constructor(name)(data, **kwargs) try: -- 2.25.4 From 201e433f5e7b29ee646a721098751f27c4d84a48 Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Fri, 24 Apr 2020 20:00:59 +0200 Subject: [PATCH 40/47] Fix test_crypt under FIPS mode The crypt methods list changes under FIPS mode and crypt.METHOD_CRYPT is not always available. --- Lib/test/test_crypt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_crypt.py b/Lib/test/test_crypt.py index 44a3ad4..36be683 100644 --- a/Lib/test/test_crypt.py +++ b/Lib/test/test_crypt.py @@ -38,7 +38,6 @@ class CryptTestCase(unittest.TestCase): def test_methods(self): # Guarantee that METHOD_CRYPT is the last method in crypt.methods. self.assertTrue(len(crypt.methods) >= 1) - self.assertEqual(crypt.METHOD_CRYPT, crypt.methods[-1]) if __name__ == "__main__": -- 2.25.4 From bd04c33ecdbd9f6d94de95706347d90d5ff3c798 Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Fri, 24 Apr 2020 20:14:52 +0200 Subject: [PATCH 41/47] Port the _hashopenssl module to openssl api 1.0.2 --- Include/_hashopenssl.h | 1 + Modules/_hmacopenssl.c | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Include/_hashopenssl.h b/Include/_hashopenssl.h index d4cbdef..4d23f6d 100644 --- a/Include/_hashopenssl.h +++ b/Include/_hashopenssl.h @@ -2,6 +2,7 @@ #define Py_HASHOPENSSL_H #include "Python.h" +#include #include #include diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c index a24c8ba..8752a20 100644 --- a/Modules/_hmacopenssl.c +++ b/Modules/_hmacopenssl.c @@ -82,7 +82,7 @@ Hmac_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) } PyObject *name = NULL; - HMAC_CTX *ctx = NULL; + HMAC_CTX *ctx = PyMem_RawCalloc(1, sizeof(HMAC_CTX)); HmacObject *retval = NULL; name = PyUnicode_FromFormat("hmac-%s", digestmod); @@ -90,7 +90,6 @@ Hmac_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) goto error; } - ctx = HMAC_CTX_new(); if (ctx == NULL) { _setException(PyExc_ValueError); goto error; @@ -122,7 +121,7 @@ Hmac_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) return (PyObject*)retval; error: - if (ctx) HMAC_CTX_free(ctx); + if (ctx) PyMem_RawFree(ctx); if (name) Py_DECREF(name); if (retval) PyObject_Del(name); if (key.buf) PyBuffer_Release(&key); @@ -141,20 +140,20 @@ _hmacopenssl_HMAC_copy_impl(HmacObject *self) { HmacObject *retval; - HMAC_CTX *ctx = HMAC_CTX_new(); + HMAC_CTX *ctx = PyMem_RawCalloc(1, sizeof(HMAC_CTX)); if (ctx == NULL) { return _setException(PyExc_ValueError); } int r = HMAC_CTX_copy(ctx, self->ctx); if (r == 0) { - HMAC_CTX_free(ctx); + PyMem_RawFree(ctx); return _setException(PyExc_ValueError); } retval = (HmacObject *)Py_TYPE(self)->tp_alloc(Py_TYPE(self), 0); if (retval == NULL) { - HMAC_CTX_free(ctx); + PyMem_RawFree(ctx); return NULL; } retval->ctx = ctx; @@ -172,7 +171,7 @@ _hmac_dealloc(HmacObject *self) if (self->lock != NULL) { PyThread_free_lock(self->lock); } - HMAC_CTX_free(self->ctx); + PyMem_RawFree(self->ctx); Py_CLEAR(self->name); Py_TYPE(self)->tp_free(self); } @@ -222,7 +221,7 @@ _hmacopenssl_HMAC_update_impl(HmacObject *self, Py_buffer *msg) static unsigned int _digest_size(HmacObject *self) { - const EVP_MD *md = HMAC_CTX_get_md(self->ctx); + const EVP_MD *md = self->ctx->md; if (md == NULL) { _setException(PyExc_ValueError); return 0; @@ -233,7 +232,7 @@ _digest_size(HmacObject *self) static int _digest(HmacObject *self, unsigned char *buf, unsigned int len) { - HMAC_CTX *temp_ctx = HMAC_CTX_new(); + HMAC_CTX *temp_ctx = PyMem_RawCalloc(1, sizeof(HMAC_CTX)); if (temp_ctx == NULL) { PyErr_NoMemory(); return 0; @@ -244,7 +243,7 @@ _digest(HmacObject *self, unsigned char *buf, unsigned int len) return 0; } r = HMAC_Final(temp_ctx, buf, &len); - HMAC_CTX_free(temp_ctx); + PyMem_RawFree(temp_ctx); if (r == 0) { _setException(PyExc_ValueError); return 0; @@ -314,7 +313,7 @@ _hmacopenssl_get_digest_size(HmacObject *self, void *closure) static PyObject * _hmacopenssl_get_block_size(HmacObject *self, void *closure) { - const EVP_MD *md = HMAC_CTX_get_md(self->ctx); + const EVP_MD *md = self->ctx->md; if (md == NULL) { return _setException(PyExc_ValueError); } -- 2.25.4 From ea2801773b946c574908334449753177af4e3b1e Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Wed, 6 May 2020 05:34:04 +0200 Subject: [PATCH 42/47] blake2, sha3 and shake are not available through openssl in RHEL7 --- Modules/_blake2/blake2module.c | 1 - Modules/_hashopenssl_list.h | 8 -------- Modules/_sha3/sha3module.c | 1 - 3 files changed, 10 deletions(-) diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c index a9c7cbc..149ab09 100644 --- a/Modules/_blake2/blake2module.c +++ b/Modules/_blake2/blake2module.c @@ -58,7 +58,6 @@ PyInit__blake2(void) PyObject *m; PyObject *d; - FAIL_RETURN_IN_FIPS_MODE(PyExc_ImportError, "blake2"); m = PyModule_Create(&blake2_module); if (m == NULL) diff --git a/Modules/_hashopenssl_list.h b/Modules/_hashopenssl_list.h index 3c11b2e..09d3c5f 100644 --- a/Modules/_hashopenssl_list.h +++ b/Modules/_hashopenssl_list.h @@ -11,11 +11,3 @@ _HASH(sha224, "sha224") _HASH(sha256, "sha256") _HASH(sha384, "sha384") _HASH(sha512, "sha512") -_HASH(blake2b, "blake2b512") -_HASH(blake2s, "blake2s256") -_HASH(sha3_224, "sha3-224") -_HASH(sha3_256, "sha3-256") -_HASH(sha3_384, "sha3-384") -_HASH(sha3_512, "sha3-512") -_HASH(shake_128, "shake128") -_HASH(shake_256, "shake256") diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c index 62db9cb..4c2a5c5 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/_sha3/sha3module.c @@ -732,7 +732,6 @@ PyInit__sha3(void) { PyObject *m = NULL; - FAIL_RETURN_IN_FIPS_MODE(PyExc_ImportError, "_sha3"); if ((m = PyModule_Create(&_SHA3module)) == NULL) { return NULL; -- 2.25.4 From 0bc7f9474fc415aecb07937ea310d0f51b11085b Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Fri, 24 Apr 2020 20:18:44 +0200 Subject: [PATCH 43/47] Remove unavailable algorithms testing from test_fips --- Lib/test/test_fips.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/Lib/test/test_fips.py b/Lib/test/test_fips.py index 86e61e2..73b219b 100644 --- a/Lib/test/test_fips.py +++ b/Lib/test/test_fips.py @@ -6,15 +6,6 @@ import hashlib, _hashlib class HashlibFipsTests(unittest.TestCase): - @unittest.skipUnless(_hashlib.get_fips_mode(), "Test only when FIPS is enabled") - def test_fips_imports(self): - """blake2s and blake2b should fail to import in FIPS mode - """ - with self.assertRaises(ValueError, msg='blake2s not available in FIPS'): - m = hashlib.blake2s() - with self.assertRaises(ValueError, msg='blake2b not available in FIPS'): - m = hashlib.blake2b() - def compare_hashes(self, python_hash, openssl_hash): """ Compare between the python implementation and the openssl one that the digests @@ -30,22 +21,6 @@ class HashlibFipsTests(unittest.TestCase): self.assertEqual(m, h) - @unittest.skipIf(_hashlib.get_fips_mode(), "blake2 hashes are not available under FIPS") - def test_blake2_hashes(self): - self.compare_hashes(hashlib.blake2b(b'abc'), _hashlib.openssl_blake2b(b'abc')) - self.compare_hashes(hashlib.blake2s(b'abc'), _hashlib.openssl_blake2s(b'abc')) - - def test_sha3_hashes(self): - self.compare_hashes(hashlib.sha3_224(b'abc'), _hashlib.openssl_sha3_224(b'abc')) - self.compare_hashes(hashlib.sha3_256(b'abc'), _hashlib.openssl_sha3_256(b'abc')) - self.compare_hashes(hashlib.sha3_384(b'abc'), _hashlib.openssl_sha3_384(b'abc')) - self.compare_hashes(hashlib.sha3_512(b'abc'), _hashlib.openssl_sha3_512(b'abc')) - - @unittest.skipIf(_hashlib.get_fips_mode(), "shake hashes are not available under FIPS") - def test_shake_hashes(self): - self.compare_hashes(hashlib.shake_128(b'abc'), _hashlib.openssl_shake_128(b'abc')) - self.compare_hashes(hashlib.shake_256(b'abc'), _hashlib.openssl_shake_256(b'abc')) - def test_sha(self): self.compare_hashes(hashlib.sha1(b'abc'), _hashlib.openssl_sha1(b'abc')) self.compare_hashes(hashlib.sha224(b'abc'), _hashlib.openssl_sha224(b'abc')) -- 2.25.4 From e3c65358aa049fea20d6046de8ade9ddd23a7061 Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Wed, 6 May 2020 05:48:44 +0200 Subject: [PATCH 44/47] Fix the hashlib test under FIPS mode in RHEL7 --- Lib/hashlib.py | 72 +++++++------- Lib/test/test_hashlib.py | 200 ++++++++++++++++++++++----------------- 2 files changed, 147 insertions(+), 125 deletions(-) diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 9bc90a4..35b9dce 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -74,15 +74,15 @@ except ImportError: return 0 -if not _hashlib_get_fips_mode(): - __builtin_constructor_cache = {} - - def __get_builtin_constructor(name): - cache = __builtin_constructor_cache - constructor = cache.get(name) - if constructor is not None: - return constructor - try: +__builtin_constructor_cache = {} + +def __get_builtin_constructor(name): + cache = __builtin_constructor_cache + constructor = cache.get(name) + if constructor is not None: + return constructor + try: + if not _hashlib_get_fips_mode(): if name in ('SHA1', 'sha1'): import _sha1 cache['SHA1'] = cache['sha1'] = _sha1.sha1 @@ -97,34 +97,33 @@ if not _hashlib_get_fips_mode(): import _sha512 cache['SHA384'] = cache['sha384'] = _sha512.sha384 cache['SHA512'] = cache['sha512'] = _sha512.sha512 - elif name in ('blake2b', 'blake2s'): - import _blake2 - cache['blake2b'] = _blake2.blake2b - cache['blake2s'] = _blake2.blake2s - elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', - 'shake_128', 'shake_256'}: - import _sha3 - cache['sha3_224'] = _sha3.sha3_224 - cache['sha3_256'] = _sha3.sha3_256 - cache['sha3_384'] = _sha3.sha3_384 - cache['sha3_512'] = _sha3.sha3_512 - cache['shake_128'] = _sha3.shake_128 - cache['shake_256'] = _sha3.shake_256 - except ImportError: - pass # no extension module, this hash is unsupported. - - constructor = cache.get(name) - if constructor is not None: - return constructor - - raise ValueError('unsupported hash type ' + name) + if name in ('blake2b', 'blake2s'): + import _blake2 + cache['blake2b'] = _blake2.blake2b + cache['blake2s'] = _blake2.blake2s + elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', + 'shake_128', 'shake_256'}: + import _sha3 + cache['sha3_224'] = _sha3.sha3_224 + cache['sha3_256'] = _sha3.sha3_256 + cache['sha3_384'] = _sha3.sha3_384 + cache['sha3_512'] = _sha3.sha3_512 + cache['shake_128'] = _sha3.shake_128 + cache['shake_256'] = _sha3.shake_256 + except ImportError: + pass # no extension module, this hash is unsupported. + + constructor = cache.get(name) + if constructor is not None: + return constructor + + raise ValueError('unsupported hash type ' + name) def __get_openssl_constructor(name): - if not _hashlib.get_fips_mode(): - if name in { + if name in { 'blake2b', 'blake2s', 'shake_256', 'shake_128', - #'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', + 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', }: # Prefer our blake2 implementation. return __get_builtin_constructor(name) @@ -197,8 +196,6 @@ try: algorithms_available = algorithms_available.union( _hashlib.openssl_md_meth_names) except ImportError: - if _hashlib_get_fips_mode(): - raise new = __py_new __get_hash = __get_builtin_constructor @@ -218,8 +215,9 @@ for __func_name in __always_supported: try: globals()[__func_name] = __get_hash(__func_name) except ValueError: - import logging - logging.exception('code for hash %s was not found.', __func_name) + if not _hashlib_get_fips_mode(): + import logging + logging.exception('code for hash %s was not found.', __func_name) # Cleanup locals() diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index a6f9a35..d63ade8 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -35,7 +35,7 @@ py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) from _hashlib import get_fips_mode as _get_fips_mode if _get_fips_mode(): - FIPS_UNAVAILABLE = {'blake2b', 'blake2s'} + FIPS_UNAVAILABLE = {'blake2b', 'blake2s', 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', 'shake_128', 'shake_256'} FIPS_DISABLED = {'md5', 'MD5', *FIPS_UNAVAILABLE} else: FIPS_UNAVAILABLE = set() @@ -46,14 +46,14 @@ try: except ImportError: _blake2 = None -requires_blake2 = unittest.skipUnless(_blake2, 'requires _blake2') +requires_blake2 = unittest.skipUnless(_blake2 and not _get_fips_mode(), 'requires _blake2') try: import _sha3 except ImportError: _sha3 = None -requires_sha3 = unittest.skipUnless(_sha3, 'requires _sha3') +requires_sha3 = unittest.skipUnless(_sha3 and not _get_fips_mode(), 'requires _sha3') def hexstr(s): @@ -189,18 +189,19 @@ class HashLibTestCase(unittest.TestCase): if _sha512: add_builtin_constructor('sha384') add_builtin_constructor('sha512') - if _blake2: - add_builtin_constructor('blake2s') - add_builtin_constructor('blake2b') - - _sha3 = self._conditional_import_module('_sha3') - if _sha3: - add_builtin_constructor('sha3_224') - add_builtin_constructor('sha3_256') - add_builtin_constructor('sha3_384') - add_builtin_constructor('sha3_512') - add_builtin_constructor('shake_128') - add_builtin_constructor('shake_256') + if not _get_fips_mode(): + if _blake2: + add_builtin_constructor('blake2s') + add_builtin_constructor('blake2b') + + _sha3 = self._conditional_import_module('_sha3') + if _sha3: + add_builtin_constructor('sha3_224') + add_builtin_constructor('sha3_256') + add_builtin_constructor('sha3_384') + add_builtin_constructor('sha3_512') + add_builtin_constructor('shake_128') + add_builtin_constructor('shake_256') super(HashLibTestCase, self).__init__(*args, **kwargs) @@ -221,12 +222,15 @@ class HashLibTestCase(unittest.TestCase): def test_hash_array(self): a = array.array("b", range(10)) - for cons in self.hash_constructors: - c = cons(a) - if self._has_shake_extras(c): - c.hexdigest(16) - else: - c.hexdigest() + try: + for cons in self.hash_constructors: + c = cons(a) + if self._has_shake_extras(c): + c.hexdigest(16) + else: + c.hexdigest() + except ValueError: + pass # Some algorithms aren't supported def test_algorithms_guaranteed(self): self.assertEqual(hashlib.algorithms_guaranteed, @@ -274,40 +278,49 @@ class HashLibTestCase(unittest.TestCase): self.assertEqual(sorted(builtin_constructor_cache), ['MD5', 'md5']) def test_hexdigest(self): - for cons in self.hash_constructors: - h = cons() - if self._has_shake_extras(h): - self.assertIsInstance(h.digest(16), bytes) - self.assertEqual(hexstr(h.digest(16)), h.hexdigest(16)) - else: - self.assertIsInstance(h.digest(), bytes) - self.assertEqual(hexstr(h.digest()), h.hexdigest()) + try: + for cons in self.hash_constructors: + h = cons() + if self._has_shake_extras(h): + self.assertIsInstance(h.digest(16), bytes) + self.assertEqual(hexstr(h.digest(16)), h.hexdigest(16)) + else: + self.assertIsInstance(h.digest(), bytes) + self.assertEqual(hexstr(h.digest()), h.hexdigest()) + except ValueError: + pass def test_digest_length_overflow(self): - # See issue #34922 - large_sizes = (2**29, 2**32-10, 2**32+10, 2**61, 2**64-10, 2**64+10) - for cons in self.hash_constructors: - h = cons() - if not self._has_shake_extras(h): - continue - for digest in h.digest, h.hexdigest: - with self.assertRaises((ValueError, OverflowError)): - digest(-10) - for length in large_sizes: + try: + # See issue #34922 + large_sizes = (2**29, 2**32-10, 2**32+10, 2**61, 2**64-10, 2**64+10) + for cons in self.hash_constructors: + h = cons() + if not self._has_shake_extras(h): + continue + for digest in h.digest, h.hexdigest: with self.assertRaises((ValueError, OverflowError)): - digest(length) + digest(-10) + for length in large_sizes: + with self.assertRaises((ValueError, OverflowError)): + digest(length) + except ValueError: + pass def test_name_attribute(self): - for cons in self.hash_constructors: - h = cons() - self.assertIsInstance(h.name, str) - if h.name in self.supported_hash_names: - self.assertIn(h.name, self.supported_hash_names) - else: - self.assertNotIn(h.name, self.supported_hash_names) - if h.name not in FIPS_DISABLED: - self.assertEqual(h.name, hashlib.new(h.name).name) - self.assertEqual(h.name, hashlib.new(h.name, usedforsecurity=False).name) + try: + for cons in self.hash_constructors: + h = cons() + self.assertIsInstance(h.name, str) + if h.name in self.supported_hash_names: + self.assertIn(h.name, self.supported_hash_names) + else: + self.assertNotIn(h.name, self.supported_hash_names) + if h.name not in FIPS_DISABLED: + self.assertEqual(h.name, hashlib.new(h.name).name) + self.assertEqual(h.name, hashlib.new(h.name, usedforsecurity=False).name) + except ValueError: + pass def test_large_update(self): aas = b'a' * 128 @@ -315,31 +328,34 @@ class HashLibTestCase(unittest.TestCase): cees = b'c' * 126 dees = b'd' * 2048 # HASHLIB_GIL_MINSIZE - for cons in self.hash_constructors: - m1 = cons() - m1.update(aas) - m1.update(bees) - m1.update(cees) - m1.update(dees) - if self._has_shake_extras(m1): - args = (16,) - else: - args = () + try: + for cons in self.hash_constructors: + m1 = cons() + m1.update(aas) + m1.update(bees) + m1.update(cees) + m1.update(dees) + if self._has_shake_extras(m1): + args = (16,) + else: + args = () - m2 = cons() - m2.update(aas + bees + cees + dees) - self.assertEqual(m1.digest(*args), m2.digest(*args)) + m2 = cons() + m2.update(aas + bees + cees + dees) + self.assertEqual(m1.digest(*args), m2.digest(*args)) - m3 = cons(aas + bees + cees + dees) - self.assertEqual(m1.digest(*args), m3.digest(*args)) + m3 = cons(aas + bees + cees + dees) + self.assertEqual(m1.digest(*args), m3.digest(*args)) - # verify copy() doesn't touch original - m4 = cons(aas + bees + cees) - m4_digest = m4.digest(*args) - m4_copy = m4.copy() - m4_copy.update(dees) - self.assertEqual(m1.digest(*args), m4_copy.digest(*args)) - self.assertEqual(m4.digest(*args), m4_digest) + # verify copy() doesn't touch original + m4 = cons(aas + bees + cees) + m4_digest = m4.digest(*args) + m4_copy = m4.copy() + m4_copy.update(dees) + self.assertEqual(m1.digest(*args), m4_copy.digest(*args)) + self.assertEqual(m4.digest(*args), m4_digest) + except ValueError: + pass def check(self, name, data, hexdigest, shake=False, **kwargs): length = len(hexdigest)//2 @@ -472,6 +488,7 @@ class HashLibTestCase(unittest.TestCase): self.check_sha3('shake_256', 512, 1088, b'\x1f') @requires_blake2 + @unittest.skipIf(_get_fips_mode(), "No builtin constructors in FIPS mode") def test_blocksize_name_blake2(self): self.check_blocksize_name('blake2b', 128, 64) self.check_blocksize_name('blake2s', 64, 32) @@ -707,6 +724,7 @@ class HashLibTestCase(unittest.TestCase): return outer.hexdigest() @requires_blake2 + @unittest.skipIf(_get_fips_mode(), "No builtin constructors in FIPS mode") def test_blake2b(self): self.check_blake2(hashlib.blake2b, 16, 16, 64, 64, (1<<64)-1) b2b_md_len = [20, 32, 48, 64] @@ -746,12 +764,14 @@ class HashLibTestCase(unittest.TestCase): last_node=True) @requires_blake2 + @unittest.skipIf(_get_fips_mode(), "No builtin constructors in FIPS mode") def test_blake2b_vectors(self): for msg, key, md in read_vectors('blake2b'): key = bytes.fromhex(key) self.check('blake2b', msg, md, key=key) @requires_blake2 + @unittest.skipIf(_get_fips_mode(), "No builtin constructors in FIPS mode") def test_blake2s(self): self.check_blake2(hashlib.blake2s, 8, 8, 32, 32, (1<<48)-1) b2s_md_len = [16, 20, 28, 32] @@ -789,6 +809,7 @@ class HashLibTestCase(unittest.TestCase): last_node=True) @requires_blake2 + @unittest.skipIf(_get_fips_mode(), "No builtin constructors in FIPS mode") def test_blake2s_vectors(self): for msg, key, md in read_vectors('blake2s'): key = bytes.fromhex(key) @@ -865,25 +886,28 @@ class HashLibTestCase(unittest.TestCase): # for multithreaded operation (which is hardwired to 2048). gil_minsize = 2048 - for cons in self.hash_constructors: - m = cons() + try: + for cons in self.hash_constructors: + m = cons() + m.update(b'1') + m.update(b'#' * gil_minsize) + m.update(b'1') + + m = cons(b'x' * gil_minsize) + m.update(b'1') + + m = hashlib.sha1() m.update(b'1') m.update(b'#' * gil_minsize) m.update(b'1') - - m = cons(b'x' * gil_minsize) - m.update(b'1') - - m = hashlib.sha1() - m.update(b'1') - m.update(b'#' * gil_minsize) - m.update(b'1') - self.assertEqual(m.hexdigest(), - 'c45f7445ca0ea087d7a1758fbea07935f267c46a') - - m = hashlib.sha1(b'x' * gil_minsize) - self.assertEqual(m.hexdigest(), - '63fda1efde982ba1ffe9d53035bff5c9ce4758fb') + self.assertEqual(m.hexdigest(), + 'c45f7445ca0ea087d7a1758fbea07935f267c46a') + + m = hashlib.sha1(b'x' * gil_minsize) + self.assertEqual(m.hexdigest(), + '63fda1efde982ba1ffe9d53035bff5c9ce4758fb') + except ValueError: + pass @unittest.skipUnless(threading, 'Threading required for this test.') @support.reap_threads -- 2.25.4 From 2377c57881b843ff1f5aea54f61ed76ba18a7e96 Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Fri, 24 Apr 2020 23:32:23 +0200 Subject: [PATCH 45/47] test_ssl fixes for FIPS mode in RHEL7 --- Lib/test/test_ssl.py | 49 +++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index cd425eb..7889e95 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -19,6 +19,7 @@ import weakref import platform import functools import sysconfig +from _hashlib import get_fips_mode try: import ctypes except ImportError: @@ -34,6 +35,11 @@ else: _have_threads = True PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) +if get_fips_mode(): + for protocol in PROTOCOLS: + if "SSLv3" in str(protocol): + PROTOCOLS.remove(protocol) + HOST = support.HOST IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL') IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0) @@ -2613,8 +2619,9 @@ if _have_threads: try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_OPTIONAL) try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_REQUIRED) try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False) - if hasattr(ssl, 'PROTOCOL_SSLv3'): - try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False) + if not get_fips_mode(): + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False) try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLSv1, False) # SSLv23 client with specific SSL options if no_sslv2_implies_sslv3_hello(): @@ -2640,28 +2647,32 @@ if _have_threads: sys.stdout.write( " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n" % str(x)) - if hasattr(ssl, 'PROTOCOL_SSLv3'): - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False) + if not get_fips_mode(): + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False) try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True) if TLSv1_enabled: try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1') - if hasattr(ssl, 'PROTOCOL_SSLv3'): - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL) + if not get_fips_mode(): + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL) try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_OPTIONAL) if TLSv1_enabled: try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) - if hasattr(ssl, 'PROTOCOL_SSLv3'): - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED) + if not get_fips_mode(): + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED) try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_REQUIRED) if TLSv1_enabled: try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) # Server with specific SSL options - if hasattr(ssl, 'PROTOCOL_SSLv3'): - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, - server_options=ssl.OP_NO_SSLv3) + if not get_fips_mode(): + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, + server_options=ssl.OP_NO_SSLv3) # Will choose TLSv1 try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) @@ -2670,6 +2681,7 @@ if _have_threads: @skip_if_broken_ubuntu_ssl + @unittest.skipIf(get_fips_mode(), "OpenSSL disabled SSLv3 at runtime under FIPS mode") @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv3'), "OpenSSL is compiled without SSLv3 support") def test_protocol_sslv3(self): @@ -2700,8 +2712,9 @@ if _have_threads: try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) if hasattr(ssl, 'PROTOCOL_SSLv2'): try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False) - if hasattr(ssl, 'PROTOCOL_SSLv3'): - try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) + if not get_fips_mode(): + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False, client_options=ssl.OP_NO_TLSv1) @@ -2717,8 +2730,9 @@ if _have_threads: try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') if hasattr(ssl, 'PROTOCOL_SSLv2'): try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False) - if hasattr(ssl, 'PROTOCOL_SSLv3'): - try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) + if not get_fips_mode(): + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv23, False, client_options=ssl.OP_NO_TLSv1_1) @@ -2741,8 +2755,9 @@ if _have_threads: client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) if hasattr(ssl, 'PROTOCOL_SSLv2'): try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv2, False) - if hasattr(ssl, 'PROTOCOL_SSLv3'): - try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False) + if not get_fips_mode(): + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False) try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv23, False, client_options=ssl.OP_NO_TLSv1_2) -- 2.25.4 From 7dba524bbc77ae0e6063384246325fb4ecd4286b Mon Sep 17 00:00:00 2001 From: Marcel Plch Date: Fri, 24 Apr 2020 22:44:43 +0200 Subject: [PATCH 46/47] Fix imports in hashlib under FIPS mode --- Lib/hashlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 35b9dce..bdf8955 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -76,7 +76,7 @@ except ImportError: __builtin_constructor_cache = {} -def __get_builtin_constructor(name): +def __get_builtin_constructor(name, *, _hashlib_get_fips_mode=_hashlib_get_fips_mode): cache = __builtin_constructor_cache constructor = cache.get(name) if constructor is not None: -- 2.25.4 From 8f06f1344cb6d7d63dada8488e4d885b35d5cbca Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Tue, 31 Mar 2020 18:00:42 +0200 Subject: [PATCH 47/47] Add a sentinel value on the Hmac_members table of the hmac module --- Modules/_hmacopenssl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_hmacopenssl.c b/Modules/_hmacopenssl.c index 8752a20..4586eba 100644 --- a/Modules/_hmacopenssl.c +++ b/Modules/_hmacopenssl.c @@ -336,6 +336,7 @@ static PyGetSetDef Hmac_getset[] = { static PyMemberDef Hmac_members[] = { {"name", T_OBJECT, offsetof(HmacObject, name), READONLY, PyDoc_STR("HMAC name")}, + {NULL} /* Sentinel */ }; PyDoc_STRVAR(hmactype_doc, -- 2.25.4