commit d41618f34f1d2f5416ec3c035f33dcb15cf5ab99

Author: Alexey Bakhtin <abakhtin@openjdk.org>

Date:   Tue Apr 4 10:29:11 2023 +0000

    8271199: Mutual TLS handshake fails signing client certificate with custom sensitive PKCS11 key

    Reviewed-by: andrew, mbalao

    Backport-of: f6232982b91cb2314e96ddbde3984836a810a556

diff --git a/jdk/src/share/classes/sun/security/rsa/RSAPSSSignature.java b/jdk/src/share/classes/sun/security/rsa/RSAPSSSignature.java

index a79e97d7c74..5378446b97b 100644

--- a/jdk/src/share/classes/sun/security/rsa/RSAPSSSignature.java

+++ b/jdk/src/share/classes/sun/security/rsa/RSAPSSSignature.java

@@ -127,12 +127,15 @@ public class RSAPSSSignature extends SignatureSpi {

     @Override

     protected void engineInitVerify(PublicKey publicKey)

             throws InvalidKeyException {

-        if (!(publicKey instanceof RSAPublicKey)) {

+        if (publicKey instanceof RSAPublicKey) {

+            RSAPublicKey rsaPubKey = (RSAPublicKey)publicKey;

+            isPublicKeyValid(rsaPubKey);

+            this.pubKey = rsaPubKey;

+            this.privKey = null;

+            resetDigest();

+        } else {

             throw new InvalidKeyException("key must be RSAPublicKey");

         }

-        this.pubKey = (RSAPublicKey) isValid((RSAKey)publicKey);

-        this.privKey = null;

-        resetDigest();

     }

     // initialize for signing. See JCA doc

@@ -146,14 +149,17 @@ public class RSAPSSSignature extends SignatureSpi {

     @Override

     protected void engineInitSign(PrivateKey privateKey, SecureRandom random)

             throws InvalidKeyException {

-        if (!(privateKey instanceof RSAPrivateKey)) {

+        if (privateKey instanceof RSAPrivateKey) {

+            RSAPrivateKey rsaPrivateKey = (RSAPrivateKey)privateKey;

+            isPrivateKeyValid(rsaPrivateKey);

+            this.privKey = rsaPrivateKey;

+            this.pubKey = null;

+            this.random =

+                    (random == null ? JCAUtil.getSecureRandom() : random);

+            resetDigest();

+        } else {

             throw new InvalidKeyException("key must be RSAPrivateKey");

         }

-        this.privKey = (RSAPrivateKey) isValid((RSAKey)privateKey);

-        this.pubKey = null;

-        this.random =

-            (random == null? JCAUtil.getSecureRandom() : random);

-        resetDigest();

     }

     /**

@@ -205,11 +211,57 @@ public class RSAPSSSignature extends SignatureSpi {

         }

     }

+    /**

+     * Validate the specified RSAPrivateKey

+     */

+    private void isPrivateKeyValid(RSAPrivateKey prKey)  throws InvalidKeyException {

+        try {

+            if (prKey instanceof RSAPrivateCrtKey) {

+                RSAPrivateCrtKey crtKey = (RSAPrivateCrtKey)prKey;

+                if (RSAPrivateCrtKeyImpl.checkComponents(crtKey)) {

+                    RSAKeyFactory.checkRSAProviderKeyLengths(

+                            crtKey.getModulus().bitLength(),

+                            crtKey.getPublicExponent());

+                } else {

+                    throw new InvalidKeyException(

+                            "Some of the CRT-specific components are not available");

+                }

+            } else {

+                RSAKeyFactory.checkRSAProviderKeyLengths(

+                        prKey.getModulus().bitLength(),

+                        null);

+            }

+        } catch (InvalidKeyException ikEx) {

+            throw ikEx;

+        } catch (Exception e) {

+            throw new InvalidKeyException(

+                    "Can not access private key components", e);

+        }

+        isValid(prKey);

+    }

+

+    /**

+     * Validate the specified RSAPublicKey

+     */

+    private void isPublicKeyValid(RSAPublicKey pKey)  throws InvalidKeyException {

+        try {

+            RSAKeyFactory.checkRSAProviderKeyLengths(

+                    pKey.getModulus().bitLength(),

+                    pKey.getPublicExponent());

+        } catch (InvalidKeyException ikEx) {

+            throw ikEx;

+        } catch (Exception e) {

+            throw new InvalidKeyException(

+                    "Can not access public key components", e);

+        }

+        isValid(pKey);

+    }

+

     /**

      * Validate the specified RSAKey and its associated parameters against

      * internal signature parameters.

      */

-    private RSAKey isValid(RSAKey rsaKey) throws InvalidKeyException {

+    private void isValid(RSAKey rsaKey) throws InvalidKeyException {

         try {

             AlgorithmParameterSpec keyParams = rsaKey.getParams();

             // validate key parameters

@@ -227,7 +279,6 @@ public class RSAPSSSignature extends SignatureSpi {

                 }

                 checkKeyLength(rsaKey, hLen, this.sigParams.getSaltLength());

             }

-            return rsaKey;

         } catch (SignatureException e) {

             throw new InvalidKeyException(e);

         }

diff --git a/jdk/src/share/classes/sun/security/rsa/RSAPrivateCrtKeyImpl.java b/jdk/src/share/classes/sun/security/rsa/RSAPrivateCrtKeyImpl.java

index 6b219937981..b3c1fae9672 100644

--- a/jdk/src/share/classes/sun/security/rsa/RSAPrivateCrtKeyImpl.java

+++ b/jdk/src/share/classes/sun/security/rsa/RSAPrivateCrtKeyImpl.java

@@ -80,22 +80,28 @@ public final class RSAPrivateCrtKeyImpl

         RSAPrivateCrtKeyImpl key = new RSAPrivateCrtKeyImpl(encoded);

         // check all CRT-specific components are available, if any one

         // missing, return a non-CRT key instead

-        if ((key.getPublicExponent().signum() == 0) ||

-            (key.getPrimeExponentP().signum() == 0) ||

-            (key.getPrimeExponentQ().signum() == 0) ||

-            (key.getPrimeP().signum() == 0) ||

-            (key.getPrimeQ().signum() == 0) ||

-            (key.getCrtCoefficient().signum() == 0)) {

+        if (checkComponents(key)) {

+            return key;

+        } else {

             return new RSAPrivateKeyImpl(

                 key.algid,

                 key.getModulus(),

-                key.getPrivateExponent()

-            );

-        } else {

-            return key;

+                key.getPrivateExponent());

         }

     }

+    /**

+     * Validate if all CRT-specific components are available.

+     */

+    static boolean checkComponents(RSAPrivateCrtKey key) {

+        return !((key.getPublicExponent().signum() == 0) ||

+            (key.getPrimeExponentP().signum() == 0) ||

+            (key.getPrimeExponentQ().signum() == 0) ||

+            (key.getPrimeP().signum() == 0) ||

+            (key.getPrimeQ().signum() == 0) ||

+            (key.getCrtCoefficient().signum() == 0));

+    }

+

     /**

      * Generate a new key from the specified type and components.

      * Returns a CRT key if possible and a non-CRT key otherwise.