At the moment the cache provided by OpenSslCachingKeyMaterialProvider… (#9759)

Motivation:

At the moment te cache is not bound and so lead to huge memory consumpation. We should ensure its bound by default.

Modifications:

Ensure cache is bound

Result:

Fixes https://github.com/netty/netty/issues/9747.

handler/src/main/java/io/netty/handler/ssl/OpenSslCachingKeyMaterialProvider.java

@@ -28,10 +28,13 @@ import java.util.concurrent.ConcurrentMap;
  */
 final class OpenSslCachingKeyMaterialProvider extends OpenSslKeyMaterialProvider {
 
+    private final int maxCachedEntries;
+    private volatile boolean full;
     private final ConcurrentMap<String, OpenSslKeyMaterial> cache = new ConcurrentHashMap<>();
 
-    OpenSslCachingKeyMaterialProvider(X509KeyManager keyManager, String password) {
+    OpenSslCachingKeyMaterialProvider(X509KeyManager keyManager, String password, int maxCachedEntries) {
         super(keyManager, password);
+        this.maxCachedEntries = maxCachedEntries;
     }
 
     @Override
@@ -44,6 +47,14 @@ final class OpenSslCachingKeyMaterialProvider extends OpenSslKeyMaterialProvider
                 return null;
             }
 
+            if (full) {
+                return material;
+            }
+            if (cache.size() > maxCachedEntries) {
+                full = true;
+                // Do not cache...
+                return material;
+            }
             OpenSslKeyMaterial old = cache.putIfAbsent(alias, material);
             if (old != null) {
                 material.release();

handler/src/main/java/io/netty/handler/ssl/OpenSslCachingX509KeyManagerFactory.java

@@ -15,6 +15,8 @@
  */
 package io.netty.handler.ssl;
 
+import io.netty.util.internal.ObjectUtil;
+
 import javax.net.ssl.KeyManager;
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.KeyManagerFactorySpi;
@@ -37,7 +39,13 @@ import java.security.cert.X509Certificate;
  */
 public final class OpenSslCachingX509KeyManagerFactory extends KeyManagerFactory {
 
+    private final int maxCachedEntries;
+
     public OpenSslCachingX509KeyManagerFactory(final KeyManagerFactory factory) {
+        this(factory, 1024);
+    }
+
+    public OpenSslCachingX509KeyManagerFactory(final KeyManagerFactory factory, int maxCachedEntries) {
         super(new KeyManagerFactorySpi() {
             @Override
             protected void engineInit(KeyStore keyStore, char[] chars)
@@ -56,5 +64,11 @@ public final class OpenSslCachingX509KeyManagerFactory extends KeyManagerFactory
                 return factory.getKeyManagers();
             }
         }, factory.getProvider(), factory.getAlgorithm());
+        this.maxCachedEntries = ObjectUtil.checkPositive(maxCachedEntries, "maxCachedEntries");
+    }
+
+    OpenSslCachingKeyMaterialProvider newProvider(String password) {
+        return new OpenSslCachingKeyMaterialProvider(
+                ReferenceCountedOpenSslContext.chooseX509KeyManager(getKeyManagers()), password, maxCachedEntries);
     }
 }

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslContext.java

@@ -898,13 +898,12 @@ public abstract class ReferenceCountedOpenSslContext extends SslContext implemen
             return ((OpenSslX509KeyManagerFactory) factory).newProvider();
         }
 
-        X509KeyManager keyManager = chooseX509KeyManager(factory.getKeyManagers());
         if (factory instanceof OpenSslCachingX509KeyManagerFactory) {
             // The user explicit used OpenSslCachingX509KeyManagerFactory which signals us that its fine to cache.
-            return new OpenSslCachingKeyMaterialProvider(keyManager, password);
+            return ((OpenSslCachingX509KeyManagerFactory) factory).newProvider(password);
         }
         // We can not be sure if the material may change at runtime so we will not cache it.
-        return new OpenSslKeyMaterialProvider(keyManager, password);
+        return new OpenSslKeyMaterialProvider(chooseX509KeyManager(factory.getKeyManagers()), password);
     }
 
     private static final class PrivateKeyMethod implements SSLPrivateKeyMethod {

handler/src/test/java/io/netty/handler/ssl/OpenSslCachingKeyMaterialProviderTest.java

@@ -33,7 +33,7 @@ public class OpenSslCachingKeyMaterialProviderTest extends OpenSslKeyMaterialPro
     @Override
     protected OpenSslKeyMaterialProvider newMaterialProvider(KeyManagerFactory factory, String password) {
         return new OpenSslCachingKeyMaterialProvider(ReferenceCountedOpenSslContext.chooseX509KeyManager(
-                factory.getKeyManagers()), password);
+                factory.getKeyManagers()), password, Integer.MAX_VALUE);
     }
 
     @Override