Added "RSASSA-PSS" algorithm in allowed algorithm list. (#11626)

Motivation:
While OpenSSK is provided support for the "RSASSA-PSS" algorithm this was still not valid from netty. Which was causing issue in validating certificates which was signed using this algorithm.

Modification:
Added "RSASSA-PSS" in LOCAL_SUPPORTED_SIGNATURE_ALGORITHMS.
validation:
Validated and tested with CA and User cert singed with RSASSA-PSS algorithm.

Result:
Fixes #11360

Co-authored-by: Norman Maurer <norman_maurer@apple.com>

handler/src/main/java/io/netty/handler/ssl/ExtendedOpenSslSession.java

@@ -40,6 +40,7 @@ abstract class ExtendedOpenSslSession extends ExtendedSSLSession implements Open
     private static final String[] LOCAL_SUPPORTED_SIGNATURE_ALGORITHMS = {
             "SHA512withRSA", "SHA512withECDSA", "SHA384withRSA", "SHA384withECDSA", "SHA256withRSA",
             "SHA256withECDSA", "SHA224withRSA", "SHA224withECDSA", "SHA1withRSA", "SHA1withECDSA",
+            "RSASSA-PSS",
     };
 
     private final OpenSslSession wrapped;

handler/src/test/java/io/netty/handler/ssl/ConscryptOpenSslEngineInteropTest.java

@@ -18,6 +18,8 @@ package io.netty.handler.ssl;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.condition.DisabledIf;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLSessionContext;

handler/src/test/java/io/netty/handler/ssl/ConscryptSslEngineTest.java

@@ -69,8 +69,15 @@ public class ConscryptSslEngineTest extends SSLEngineTest {
     }
 
     @Disabled("Possible Conscrypt bug")
+    @Override
     public void testSessionCacheTimeout(SSLEngineTestParam param) throws Exception {
         // Skip
         // https://github.com/google/conscrypt/issues/851
     }
+
+    @Disabled("Not supported")
+    @Override
+    public void testRSASSAPSS(SSLEngineTestParam param) {
+        // skip
+    }
 }

handler/src/test/java/io/netty/handler/ssl/JdkConscryptSslEngineInteropTest.java

@@ -81,8 +81,15 @@ public class JdkConscryptSslEngineInteropTest extends SSLEngineTest {
     }
 
     @Disabled("Possible Conscrypt bug")
+    @Override
     public void testSessionCacheTimeout(SSLEngineTestParam param) {
         // Skip
         // https://github.com/google/conscrypt/issues/851
     }
+
+    @Disabled("Not supported")
+    @Override
+    public void testRSASSAPSS(SSLEngineTestParam param) {
+        // skip
+    }
 }

handler/src/test/java/io/netty/handler/ssl/JdkOpenSslEngineInteroptTest.java

@@ -18,6 +18,8 @@ package io.netty.handler.ssl;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.condition.DisabledOnOs;
 import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import javax.net.ssl.SSLEngine;
 import java.util.ArrayList;
@@ -192,6 +194,14 @@ public class JdkOpenSslEngineInteroptTest extends SSLEngineTest {
         super.testSessionCacheSize(param);
     }
 
+    @MethodSource("newTestParams")
+    @ParameterizedTest
+    @Override
+    public void testRSASSAPSS(SSLEngineTestParam param) throws Exception {
+        checkShouldUseKeyManagerFactory();
+        super.testRSASSAPSS(param);
+    }
+
     @Override
     protected SSLEngine wrapEngine(SSLEngine engine) {
         return Java8SslTestUtils.wrapSSLEngineForTesting(engine);

handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java

@@ -24,7 +24,6 @@ import io.netty.handler.ssl.util.SelfSignedCertificate;
 import io.netty.internal.tcnative.SSL;
 import io.netty.util.CharsetUtil;
 import io.netty.util.internal.EmptyArrays;
-
 import io.netty.util.internal.PlatformDependent;
 import org.junit.AssumptionViolatedException;
 import org.junit.jupiter.api.AfterEach;
@@ -1558,4 +1557,12 @@ public class OpenSslEngineTest extends SSLEngineTest {
     protected boolean isSessionMaybeReused(SSLEngine engine) {
         return unwrapEngine(engine).isSessionReused();
     }
+
+    @MethodSource("newTestParams")
+    @ParameterizedTest
+    @Override
+    public void testRSASSAPSS(SSLEngineTestParam param) throws Exception {
+        checkShouldUseKeyManagerFactory();
+        super.testRSASSAPSS(param);
+    }
 }

handler/src/test/java/io/netty/handler/ssl/OpenSslJdkSslEngineInteroptTest.java

@@ -17,6 +17,8 @@ package io.netty.handler.ssl;
 
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import javax.net.ssl.SSLEngine;
 
@@ -148,6 +150,14 @@ public class OpenSslJdkSslEngineInteroptTest extends SSLEngineTest {
         super.testSessionCacheSize(param);
     }
 
+    @MethodSource("newTestParams")
+    @ParameterizedTest
+    @Override
+    public void testRSASSAPSS(SSLEngineTestParam param) throws Exception {
+        checkShouldUseKeyManagerFactory();
+        super.testRSASSAPSS(param);
+    }
+
     @Override
     protected SSLEngine wrapEngine(SSLEngine engine) {
         return Java8SslTestUtils.wrapSSLEngineForTesting(engine);

handler/src/test/java/io/netty/handler/ssl/SSLEngineTest.java

@@ -782,11 +782,11 @@ public abstract class SSLEngineTest {
     protected void mySetupMutualAuthServerInitSslHandler(SslHandler handler) {
     }
 
-    private void mySetupMutualAuth(final SSLEngineTestParam param, KeyManagerFactory serverKMF,
+    protected void mySetupMutualAuth(final SSLEngineTestParam param, KeyManagerFactory serverKMF,
-                                   final File serverTrustManager,
+                                     final File serverTrustManager,
-                                   KeyManagerFactory clientKMF, File clientTrustManager,
+                                     KeyManagerFactory clientKMF, File clientTrustManager,
-                                   ClientAuth clientAuth, final boolean failureExpected,
+                                     ClientAuth clientAuth, final boolean failureExpected,
-                                   final boolean serverInitEngine)
+                                     final boolean serverInitEngine)
             throws Exception {
         serverSslCtx =
                 wrapContext(param, SslContextBuilder.forServer(serverKMF)
@@ -911,7 +911,7 @@ public abstract class SSLEngineTest {
         clientChannel = ccf.get();
     }
 
-    private static void rethrowIfNotNull(Throwable error) {
+    protected static void rethrowIfNotNull(Throwable error) {
         if (error != null) {
             throw new AssertionFailedError("Expected no error", error);
         }
@@ -4164,6 +4164,36 @@ public abstract class SSLEngineTest {
                 SslUtils.arrayContains(serverProtocols, SslProtocols.TLS_v1_3));
     }
 
+    @MethodSource("newTestParams")
+    @ParameterizedTest
+    public void testRSASSAPSS(SSLEngineTestParam param) throws Exception {
+        char[] password = "password".toCharArray();
+
+        final KeyStore serverKeyStore = KeyStore.getInstance("PKCS12");
+        serverKeyStore.load(getClass().getResourceAsStream("rsaValidations-server-keystore.p12"), password);
+
+        final KeyStore clientKeyStore = KeyStore.getInstance("PKCS12");
+        clientKeyStore.load(getClass().getResourceAsStream("rsaValidation-user-certs.p12"), password);
+
+        final KeyManagerFactory serverKeyManagerFactory =
+                KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+        serverKeyManagerFactory.init(serverKeyStore, password);
+        final KeyManagerFactory clientKeyManagerFactory =
+                KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+        clientKeyManagerFactory.init(clientKeyStore, password);
+
+        File commonChain = ResourcesUtil.getFile(getClass(), "rsapss-ca-cert.cert");
+        ClientAuth auth = ClientAuth.REQUIRE;
+
+        mySetupMutualAuth(param, serverKeyManagerFactory, commonChain, clientKeyManagerFactory, commonChain,
+                auth, false, true);
+
+        assertTrue(clientLatch.await(10, TimeUnit.SECONDS));
+        rethrowIfNotNull(clientException);
+        assertTrue(serverLatch.await(5, TimeUnit.SECONDS));
+        rethrowIfNotNull(serverException);
+    }
+
     protected SSLEngine wrapEngine(SSLEngine engine) {
         return engine;
     }

handler/src/test/resources/io/netty/handler/ssl/generate-certificate.sh

@@ -0,0 +1,18 @@
+# Generate CA key and certificate.
+openssl req -x509 -newkey rsa:2048 -days 3650 -keyout rsapss-ca-key.pem -out rsapss-ca-cert.cert -subj "/C=GB/O=Netty/OU=netty-parent/CN=west.int" -sigopt rsa_padding_mode:pss -sha256 -sigopt rsa_pss_saltlen:20
+
+# Generate user key nand.
+openssl req -newkey rsa:2048 -keyout rsapss-user-key.pem -out rsaValidation-req.pem -subj "/C=GB/O=Netty/OU=netty-parent/CN=c1" -sigopt rsa_padding_mode:pss -sha256 -sigopt rsa_pss_saltlen:20
+
+# Sign user cert request using CA certificate.
+openssl x509 -req -in rsaValidation-req.pem -days 365 -extensions ext -extfile rsapss-signing-ext.txt -CA rsapss-ca-cert.cert -CAkey rsapss-ca-key.pem -CAcreateserial -out rsapss-user-singed.cert -sigopt rsa_padding_mode:pss -sha256 -sigopt rsa_pss_saltlen:20
+
+# Create user certificate keystore.
+openssl pkcs12 -export -out rsaValidation-user-certs.p12 -inkey rsapss-user-key.pem -in rsapss-user-singed.cert
+
+# create keystore for the 
+openssl pkcs12 -in rsapss-ca-cert.cert -inkey rsapss-ca-key.pem -passin pass:password -certfile rsapss-ca-cert.cert -export -out rsaValidations-server-keystore.p12 -passout pass:password -name localhost
+
+
+# Create Trustore to verify the EndEntity certificate we have created.
+keytool -importcert -storetype PKCS12 -keystore rsaValidations-truststore.p12 -storepass password -alias ca -file rsapss-ca-cert.cert -noprompt

handler/src/test/resources/io/netty/handler/ssl/rsapss-ca-cert.cert

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIIDxTCCAoKgAwIBAgIUJ2aZ084kIATHBPDJFXVu7SJ4uVcwOAYJKoZIhvcNAQEK
+MCugDTALBglghkgBZQMEAgGhGjAYBgkqhkiG9w0BAQgwCwYJYIZIAWUDBAIBMEcx
+CzAJBgNVBAYTAkdCMQ4wDAYDVQQKDAVOZXR0eTEVMBMGA1UECwwMbmV0dHktcGFy
+ZW50MREwDwYDVQQDDAh3ZXN0LmludDAeFw0yMTA4MjkwNjAxMTNaFw0zMTA4Mjcw
+NjAxMTNaMEcxCzAJBgNVBAYTAkdCMQ4wDAYDVQQKDAVOZXR0eTEVMBMGA1UECwwM
+bmV0dHktcGFyZW50MREwDwYDVQQDDAh3ZXN0LmludDCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAL+xcxKjWgbYHIRqnP3Sw91SNTwY85ocb+3D4xh7/F6w
+cYgXwxgaHRKlk97HUzYZGFEb34BG89EOdDa1DvwxAMaN8sirefjrMLpvmfUD3Yti
+kGKj+CM3gh5wFSb9mPPoY/S61+KoRSAeMKeYyFQh5IIJyVqN5mrziu0+t04X4YEw
+9nATkmoS1V27Ucmo3OTkNNamqlXqVeiLKhvHtMViRGua8HwfEmjvFOTfyFHudcAz
+NFFH9JR9C2g9wuokcWFD3sdFfOZ4DJVN35NrXCO4FhxxcjHOXKRdbtsucFHqCPaE
+fVL0qrlkAm3pd9jKnBujC5sQbritg0uvmVuoxzy1jIUCAwEAAaNTMFEwHQYDVR0O
+BBYEFAzguQlpxd/3TPhYZqEryBQ6lUdJMB8GA1UdIwQYMBaAFAzguQlpxd/3TPhY
+ZqEryBQ6lUdJMA8GA1UdEwEB/wQFMAMBAf8wOAYJKoZIhvcNAQEKMCugDTALBglg
+hkgBZQMEAgGhGjAYBgkqhkiG9w0BAQgwCwYJYIZIAWUDBAIBA4IBAQB3jsUwdyFO
+9u/abLBGuETWbyuLX7NA9yvQL7cei40fJdsZZpZkHDJvNnrblpdaeFjuAI4vmAqz
+odiHzZodSaFCwODFX8oYyBcMTHW99UYiGywskF1NnJKq13r4kP7+7w7ZaE/5YukW
+VSeCXTHp1c0umuieluG87MZH4dCZgrvzpZwBeGoLLNyMyo4qHwYfkZiG2rTRpVX3
++VsWnMOaRVMYrzTB2tPZyAZyRMEfTd0fNi7ufSu6ywrOdziTu6Y1qVh18qDKpPsG
+eaSCNQoO5D9vUbiFjxKPJe8hZ0bDWTbVKRpeIrQMeHXnXGPEV5rPOcJUzwnDsGqI
+gqr6XlcEs+lp
+-----END CERTIFICATE-----

handler/src/test/resources/io/netty/handler/ssl/rsapss-signing-ext.txt

@@ -0,0 +1,21 @@
+[ ext ]
+extendedKeyUsage = clientAuth
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+#subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer
+
+
+[ exts ]
+extendedKeyUsage = serverAuth
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+#subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer
+#subjectAltName = @alt_names
+
+[ extca ]
+authorityKeyIdentifier = keyid,issuer
+basicConstraints=CA:TRUE
+subjectKeyIdentifier = hash
+
+[alt_names]
+DNS.1 = aws-dev-node.skylo.local