Preload classes before calling native OnLoad function to prevent clas… (#11215)

Motivation:

It turns out it is quite easy to cause a classloader deadlock in more recent java updates if you cause classloading while you are in native code. Because of this we should just workaround this issue by pre-load all the classes that needs to be accessed in the OnLoad function.

Modifications:

- Preload all classes that would otherwise be loaded by native OnLoad functions.

Result:

Workaround for https://github.com/netty/netty/issues/11209 and https://bugs.openjdk.java.net/browse/JDK-8266310

common/src/main/java/io/netty/util/internal/ClassInitializerUtil.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2021 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.util.internal;
+
+/**
+ * Utility which ensures that classes are loaded by the {@link ClassLoader}.
+ */
+public final class ClassInitializerUtil {
+
+    private ClassInitializerUtil() { }
+
+    /**
+     * Preload the given classes and so ensure the {@link ClassLoader} has these loaded after this method call.
+     *
+     * @param loadingClass      the {@link Class} that wants to load the classes.
+     * @param classes           the classes to load.
+     */
+    public static void tryLoadClasses(Class<?> loadingClass, Class<?>... classes) {
+        ClassLoader loader = PlatformDependent.getClassLoader(loadingClass);
+        for (Class<?> clazz: classes) {
+            tryLoadClass(loader, clazz.getName());
+        }
+    }
+
+    private static void tryLoadClass(ClassLoader classLoader, String className) {
+        try {
+            // Load the class and also ensure we init it which means its linked etc.
+            Class.forName(className, true, classLoader);
+        } catch (ClassNotFoundException ignore) {
+            // Ignore
+        } catch (SecurityException ignore) {
+            // Ignore
+        }
+    }
+}

resolver-dns-native-macos/src/main/c/netty_resolver_dns_macos.c

@@ -162,6 +162,8 @@ static void netty_resolver_dns_native_macos_JNI_OnUnLoad(JNIEnv* env) {
     }
 }
 
+// IMPORTANT: If you add any NETTY_JNI_UTIL_LOAD_CLASS or NETTY_JNI_UTIL_FIND_CLASS calls you also need to update
+//            MacOSDnsServerAddressStreamProvider to reflect that.
 static jint netty_resolver_dns_native_macos_JNI_OnLoad(JNIEnv* env, const char* packagePrefix) {
     int ret = JNI_ERR;
     int providerRegistered = 0;

resolver-dns-native-macos/src/main/java/io/netty/resolver/dns/macos/MacOSDnsServerAddressStreamProvider.java

@@ -19,6 +19,7 @@ import io.netty.resolver.dns.DnsServerAddressStream;
 import io.netty.resolver.dns.DnsServerAddressStreamProvider;
 import io.netty.resolver.dns.DnsServerAddressStreamProviders;
 import io.netty.resolver.dns.DnsServerAddresses;
+import io.netty.util.internal.ClassInitializerUtil;
 import io.netty.util.internal.NativeLibraryLoader;
 import io.netty.util.internal.PlatformDependent;
 import io.netty.util.internal.StringUtil;
@@ -51,6 +52,16 @@ public final class MacOSDnsServerAddressStreamProvider implements DnsServerAddre
     private static final long REFRESH_INTERVAL = TimeUnit.SECONDS.toNanos(10);
 
     static {
+        // Preload all classes that will be used in the OnLoad(...) function of JNI to eliminate the possiblity of a
+        // class-loader deadlock. This is a workaround for https://github.com/netty/netty/issues/11209.
+
+        // This needs to match all the classes that are loaded via NETTY_JNI_UTIL_LOAD_CLASS or looked up via
+        // NETTY_JNI_UTIL_FIND_CLASS.
+        ClassInitializerUtil.tryLoadClasses(MacOSDnsServerAddressStreamProvider.class,
+                // netty_resolver_dns_macos
+                byte[].class, String.class
+        );
+
         Throwable cause = null;
         try {
             loadNativeLibrary();

transport-native-epoll/src/main/c/netty_epoll_linuxsocket.c

@@ -746,6 +746,8 @@ error:
 
 // JNI Method Registration Table End
 
+// IMPORTANT: If you add any NETTY_JNI_UTIL_LOAD_CLASS or NETTY_JNI_UTIL_FIND_CLASS calls you also need to update
+//            Native to reflect that.
 jint netty_epoll_linuxsocket_JNI_OnLoad(JNIEnv* env, const char* packagePrefix) {
     int ret = JNI_ERR;
     char* nettyClassName = NULL;

transport-native-epoll/src/main/c/netty_epoll_native.c

@@ -727,6 +727,8 @@ error:
 
 // JNI Method Registration Table End
 
+// IMPORTANT: If you add any NETTY_JNI_UTIL_LOAD_CLASS or NETTY_JNI_UTIL_FIND_CLASS calls you also need to update
+//            Native to reflect that.
 static jint netty_epoll_native_JNI_OnLoad(JNIEnv* env, const char* packagePrefix) {
     int ret = JNI_ERR;
     int staticallyRegistered = 0;

transport-native-epoll/src/main/java/io/netty/channel/epoll/Native.java

@@ -15,9 +15,12 @@
  */
 package io.netty.channel.epoll;
 
+import io.netty.channel.DefaultFileRegion;
 import io.netty.channel.unix.FileDescriptor;
+import io.netty.channel.unix.PeerCredentials;
 import io.netty.channel.unix.Socket;
 import io.netty.channel.unix.Unix;
+import io.netty.util.internal.ClassInitializerUtil;
 import io.netty.util.internal.NativeLibraryLoader;
 import io.netty.util.internal.PlatformDependent;
 import io.netty.util.internal.SystemPropertyUtil;
@@ -26,6 +29,7 @@ import io.netty.util.internal.logging.InternalLogger;
 import io.netty.util.internal.logging.InternalLoggerFactory;
 
 import java.io.IOException;
+import java.nio.channels.FileChannel;
 import java.nio.channels.Selector;
 import java.util.Locale;
 
@@ -61,6 +65,19 @@ public final class Native {
         } catch (IOException ignore) {
             // Just ignore
         }
+
+        // Preload all classes that will be used in the OnLoad(...) function of JNI to eliminate the possiblity of a
+        // class-loader deadlock. This is a workaround for https://github.com/netty/netty/issues/11209.
+
+        // This needs to match all the classes that are loaded via NETTY_JNI_UTIL_LOAD_CLASS or looked up via
+        // NETTY_JNI_UTIL_FIND_CLASS.
+        ClassInitializerUtil.tryLoadClasses(Native.class,
+                // netty_epoll_linuxsocket
+                PeerCredentials.class, DefaultFileRegion.class, FileChannel.class, java.io.FileDescriptor.class,
+                // netty_epoll_native
+                NativeDatagramPacketArray.NativeDatagramPacket.class
+        );
+
         try {
             // First, try calling a side-effect free JNI method to see if the library was already
             // loaded by the application.

transport-native-kqueue/src/main/c/netty_kqueue_bsdsocket.c

@@ -238,6 +238,8 @@ error:
 
 // JNI Method Registration Table End
 
+// IMPORTANT: If you add any NETTY_JNI_UTIL_LOAD_CLASS or NETTY_JNI_UTIL_FIND_CLASS calls you also need to update
+//            Native to reflect that.
 jint netty_kqueue_bsdsocket_JNI_OnLoad(JNIEnv* env, const char* packagePrefix) {
     int ret = JNI_ERR;
     char* nettyClassName = NULL;

transport-native-kqueue/src/main/c/netty_kqueue_eventarray.c

@@ -38,6 +38,8 @@ static const jint fixed_method_table_size = sizeof(fixed_method_table) / sizeof(
 
 // JNI Method Registration Table End
 
+// IMPORTANT: If you add any NETTY_JNI_UTIL_LOAD_CLASS or NETTY_JNI_UTIL_FIND_CLASS calls you also need to update
+//            Native to reflect that.
 jint netty_kqueue_eventarray_JNI_OnLoad(JNIEnv* env, const char* packagePrefix) {
     if (netty_jni_util_register_natives(env,
             packagePrefix,

transport-native-kqueue/src/main/c/netty_kqueue_native.c

@@ -281,6 +281,8 @@ static const JNINativeMethod fixed_method_table[] = {
 static const jint fixed_method_table_size = sizeof(fixed_method_table) / sizeof(fixed_method_table[0]);
 // JNI Method Registration Table End
 
+// IMPORTANT: If you add any NETTY_JNI_UTIL_LOAD_CLASS or NETTY_JNI_UTIL_FIND_CLASS calls you also need to update
+//            Native to reflect that.
 static jint netty_kqueue_native_JNI_OnLoad(JNIEnv* env, const char* packagePrefix) {
     int staticallyRegistered = 0;
     int nativeRegistered = 0;

transport-native-kqueue/src/main/java/io/netty/channel/kqueue/Native.java

@@ -15,8 +15,11 @@
  */
 package io.netty.channel.kqueue;
 
+import io.netty.channel.DefaultFileRegion;
 import io.netty.channel.unix.FileDescriptor;
+import io.netty.channel.unix.PeerCredentials;
 import io.netty.channel.unix.Unix;
+import io.netty.util.internal.ClassInitializerUtil;
 import io.netty.util.internal.NativeLibraryLoader;
 import io.netty.util.internal.PlatformDependent;
 import io.netty.util.internal.SystemPropertyUtil;
@@ -25,6 +28,7 @@ import io.netty.util.internal.logging.InternalLogger;
 import io.netty.util.internal.logging.InternalLoggerFactory;
 
 import java.io.IOException;
+import java.nio.channels.FileChannel;
 import java.util.Locale;
 
 import static io.netty.channel.kqueue.KQueueStaticallyReferencedJniMethods.evAdd;
@@ -51,6 +55,16 @@ final class Native {
     private static final InternalLogger logger = InternalLoggerFactory.getInstance(Native.class);
 
     static {
+        // Preload all classes that will be used in the OnLoad(...) function of JNI to eliminate the possiblity of a
+        // class-loader deadlock. This is a workaround for https://github.com/netty/netty/issues/11209.
+
+        // This needs to match all the classes that are loaded via NETTY_JNI_UTIL_LOAD_CLASS or looked up via
+        // NETTY_JNI_UTIL_FIND_CLASS.
+        ClassInitializerUtil.tryLoadClasses(Native.class,
+                // netty_kqueue_bsdsocket
+                PeerCredentials.class, DefaultFileRegion.class, FileChannel.class, java.io.FileDescriptor.class
+        );
+
         try {
             // First, try calling a side-effect free JNI method to see if the library was already
             // loaded by the application.

transport-native-unix-common/src/main/c/netty_unix.c

@@ -22,6 +22,8 @@
 #include "netty_unix_socket.h"
 #include "netty_unix_util.h"
 
+// IMPORTANT: If you add any NETTY_JNI_UTIL_LOAD_CLASS or NETTY_JNI_UTIL_FIND_CLASS calls you also need to update
+//            Unix to reflect that.
 jint netty_unix_register(JNIEnv* env, const char* packagePrefix) {
     int limitsOnLoadCalled = 0;
     int errorsOnLoadCalled = 0;

transport-native-unix-common/src/main/c/netty_unix_buffer.c

@@ -39,6 +39,8 @@ static const JNINativeMethod statically_referenced_fixed_method_table[] = {
 static const jint statically_referenced_fixed_method_table_size = sizeof(statically_referenced_fixed_method_table) / sizeof(statically_referenced_fixed_method_table[0]);
 // JNI Method Registration Table End
 
+// IMPORTANT: If you add any NETTY_JNI_UTIL_LOAD_CLASS or NETTY_JNI_UTIL_FIND_CLASS calls you also need to update
+//            Unix to reflect that.
 jint netty_unix_buffer_JNI_OnLoad(JNIEnv* env, const char* packagePrefix) {
     // We must register the statically referenced methods first!
     if (netty_jni_util_register_natives(env,

transport-native-unix-common/src/main/c/netty_unix_errors.c

@@ -212,6 +212,8 @@ static const JNINativeMethod statically_referenced_fixed_method_table[] = {
 static const jint statically_referenced_fixed_method_table_size = sizeof(statically_referenced_fixed_method_table) / sizeof(statically_referenced_fixed_method_table[0]);
 // JNI Method Registration Table End
 
+// IMPORTANT: If you add any NETTY_JNI_UTIL_LOAD_CLASS or NETTY_JNI_UTIL_FIND_CLASS calls you also need to update
+//            Unix to reflect that.
 jint netty_unix_errors_JNI_OnLoad(JNIEnv* env, const char* packagePrefix) {
     char* nettyClassName = NULL;
     // We must register the statically referenced methods first!

transport-native-unix-common/src/main/c/netty_unix_filedescriptor.c

@@ -278,6 +278,8 @@ static const JNINativeMethod method_table[] = {
 static const jint method_table_size = sizeof(method_table) / sizeof(method_table[0]);
 // JNI Method Registration Table End
 
+// IMPORTANT: If you add any NETTY_JNI_UTIL_LOAD_CLASS or NETTY_JNI_UTIL_FIND_CLASS calls you also need to update
+//            Unix to reflect that.
 jint netty_unix_filedescriptor_JNI_OnLoad(JNIEnv* env, const char* packagePrefix) {
     int ret = JNI_ERR;
     void* mem = NULL;

transport-native-unix-common/src/main/c/netty_unix_limits.c

@@ -69,6 +69,8 @@ static const JNINativeMethod statically_referenced_fixed_method_table[] = {
 static const jint statically_referenced_fixed_method_table_size = sizeof(statically_referenced_fixed_method_table) / sizeof(statically_referenced_fixed_method_table[0]);
 // JNI Method Registration Table End
 
+// IMPORTANT: If you add any NETTY_JNI_UTIL_LOAD_CLASS or NETTY_JNI_UTIL_FIND_CLASS calls you also need to update
+//            Unix to reflect that.
 jint netty_unix_limits_JNI_OnLoad(JNIEnv* env, const char* packagePrefix) {
     // We must register the statically referenced methods first!
     if (netty_jni_util_register_natives(env,

transport-native-unix-common/src/main/c/netty_unix_socket.c

@@ -1011,6 +1011,8 @@ error:
 
 // JNI Method Registration Table End
 
+// IMPORTANT: If you add any NETTY_JNI_UTIL_LOAD_CLASS or NETTY_JNI_UTIL_FIND_CLASS calls you also need to update
+//            Unix to reflect that.
 jint netty_unix_socket_JNI_OnLoad(JNIEnv* env, const char* packagePrefix) {
     int ret = JNI_ERR;
     char* nettyClassName = NULL;

transport-native-unix-common/src/main/java/io/netty/channel/unix/Unix.java

@@ -15,8 +15,13 @@
  */
 package io.netty.channel.unix;
 
+import io.netty.util.internal.ClassInitializerUtil;
 import io.netty.util.internal.UnstableApi;
 
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.PortUnreachableException;
+import java.nio.channels.ClosedChannelException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -26,6 +31,22 @@ import java.util.concurrent.atomic.AtomicBoolean;
 public final class Unix {
     private static final AtomicBoolean registered = new AtomicBoolean();
 
+    static {
+        // Preload all classes that will be used in the OnLoad(...) function of JNI to eliminate the possiblity of a
+        // class-loader deadlock. This is a workaround for https://github.com/netty/netty/issues/11209.
+
+        // This needs to match all the classes that are loaded via NETTY_JNI_UTIL_LOAD_CLASS or looked up via
+        // NETTY_JNI_UTIL_FIND_CLASS.
+        ClassInitializerUtil.tryLoadClasses(Unix.class,
+                // netty_unix_errors
+                OutOfMemoryError.class, RuntimeException.class, ClosedChannelException.class,
+                IOException.class, PortUnreachableException.class,
+
+                // netty_unix_socket
+                DatagramSocketAddress.class, InetSocketAddress.class
+        );
+    }
+
     /**
      * Internal method... Should never be called from the user.
      *