Mark initialization of unsafe as privileged

Motiviation:

Preparing platform dependent code for using unsafe requires executing
privileged code. The privileged code for initializing unsafe is executed
in a manner that would require all code leading up to the initialization
to have the requisite permissions. Yet, in a restrictive environment
(e.g., under a security policy that only grants the requisite
permissions the Netty common jar but not to application code triggering
the Netty initialization), then initializing unsafe will not succeed
even if the security policy would otherwise permit it.

Modifications:

This commit marks the necessary blocks as privileged. This enables
access to the necessary resources for initialization unsafe. The idea is
that we are saying the Netty code is trusted, and as long as the Netty
code has been granted the necessary permissions, then we will allow the
caller access to these resources even though the caller itself might not
have the requisite permissions.

Result:

Unsafe can be initialized in a restrictive security environment.

common/src/main/java/io/netty/util/internal/PlatformDependent.java

@@ -27,6 +27,7 @@ import org.jctools.queues.atomic.MpscAtomicArrayQueue;
 import org.jctools.queues.atomic.MpscLinkedAtomicQueue;
 import org.jctools.queues.atomic.SpscLinkedAtomicQueue;
 import org.jctools.util.Pow2;
+import org.jctools.util.UnsafeAccess;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -833,6 +834,50 @@ public final class PlatformDependent {
         return null;
     }
 
+    private static final class Mpsc {
+        private static final boolean USE_MPSC_CHUNKED_ARRAY_QUEUE;
+
+        private Mpsc() {
+        }
+
+        static {
+            Object unsafe = null;
+            if (hasUnsafe()) {
+                // jctools goes through its own process of initializing unsafe; of
+                // course, this requires permissions which might not be granted to calling code, so we
+                // must mark this block as privileged too
+                unsafe = AccessController.doPrivileged(new PrivilegedAction<Object>() {
+                    @Override
+                    public Object run() {
+                        // force JCTools to initialize unsafe
+                        return UnsafeAccess.UNSAFE;
+                    }
+                });
+            }
+
+            if (unsafe == null) {
+                logger.debug("org.jctools-core.MpscChunkedArrayQueue: unavailable");
+                USE_MPSC_CHUNKED_ARRAY_QUEUE = false;
+            } else {
+                logger.debug("org.jctools-core.MpscChunkedArrayQueue: available");
+                USE_MPSC_CHUNKED_ARRAY_QUEUE = true;
+            }
+        }
+
+        static <T> Queue<T> newMpscQueue(final int maxCapacity) {
+            if (USE_MPSC_CHUNKED_ARRAY_QUEUE) {
+                // Calculate the max capacity which can not be bigger then MAX_ALLOWED_MPSC_CAPACITY.
+                // This is forced by the MpscChunkedArrayQueue implementation as will try to round it
+                // up to the next power of two and so will overflow otherwise.
+                final int capacity =
+                        Math.max(Math.min(maxCapacity, MAX_ALLOWED_MPSC_CAPACITY), MIN_MAX_MPSC_CAPACITY);
+                return new MpscChunkedArrayQueue<T>(MPSC_CHUNK_SIZE, capacity, true);
+            } else {
+                return new MpscLinkedAtomicQueue<T>();
+            }
+        }
+    }
+
     /**
      * Create a new {@link Queue} which is safe to use for multiple producers (different threads) and a single
      * consumer (one thread!).
@@ -845,14 +890,8 @@ public final class PlatformDependent {
      * Create a new {@link Queue} which is safe to use for multiple producers (different threads) and a single
      * consumer (one thread!).
      */
-    public static <T> Queue<T> newMpscQueue(int maxCapacity) {
+    public static <T> Queue<T> newMpscQueue(final int maxCapacity) {
-        return hasUnsafe() ?
+        return Mpsc.newMpscQueue(maxCapacity);
-                new MpscChunkedArrayQueue<T>(MPSC_CHUNK_SIZE,
-                        // Calculate the max capacity which can not be bigger then MAX_ALLOWED_MPSC_CAPACITY.
-                        // This is forced by the MpscChunkedArrayQueue implementation as will try to round it
-                        // up to the next power of two and so will overflow otherwise.
-                        Math.max(Math.min(maxCapacity, MAX_ALLOWED_MPSC_CAPACITY), MIN_MAX_MPSC_CAPACITY), true)
-                : new MpscLinkedAtomicQueue<T>();
     }
 
     /**

common/src/main/java/io/netty/util/internal/PlatformDependent0.java

@@ -21,6 +21,7 @@ import sun.misc.Unsafe;
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.nio.Buffer;
 import java.nio.ByteBuffer;
@@ -57,49 +58,98 @@ final class PlatformDependent0 {
     private static final boolean UNALIGNED;
 
     static {
-        ByteBuffer direct = ByteBuffer.allocateDirect(1);
+        final ByteBuffer direct = ByteBuffer.allocateDirect(1);
-        Field addressField;
+        final Field addressField;
+        // attempt to access field Buffer#address
+        final Object maybeAddressField = AccessController.doPrivileged(new PrivilegedAction<Object>() {
+            @Override
+            public Object run() {
                 try {
-            addressField = Buffer.class.getDeclaredField("address");
+                    final Field field = Buffer.class.getDeclaredField("address");
-            addressField.setAccessible(true);
+                    field.setAccessible(true);
-            if (addressField.getLong(direct) == 0) {
+                    // if direct really is a direct buffer, address will be non-zero
-                // A direct buffer must have non-zero address.
+                    if (field.getLong(direct) == 0) {
-                addressField = null;
+                        return null;
                     }
-        } catch (Throwable t) {
+                    return field;
-            // Failed to access the address field.
+                } catch (IllegalAccessException e) {
-            addressField = null;
+                    return e;
+                } catch (NoSuchFieldException e) {
+                    return e;
+                } catch (SecurityException e) {
+                    return e;
                 }
+            }
+        });
 
-        logger.debug("java.nio.Buffer.address: {}", addressField != null? "available" : "unavailable");
+        if (maybeAddressField instanceof Field) {
+            addressField = (Field) maybeAddressField;
+            logger.debug("java.nio.Buffer.address: available");
+        } else {
+            logger.debug("java.nio.Buffer.address: unavailable", (Exception) maybeAddressField);
+            addressField = null;
+        }
 
         Unsafe unsafe;
         if (addressField != null) {
+            // attempt to access field Unsafe#theUnsafe
+            final Object maybeUnsafe = AccessController.doPrivileged(new PrivilegedAction<Object>() {
+                @Override
+                public Object run() {
                     try {
-                Field unsafeField = Unsafe.class.getDeclaredField("theUnsafe");
+                        final Field unsafeField = Unsafe.class.getDeclaredField("theUnsafe");
                         unsafeField.setAccessible(true);
-                unsafe = (Unsafe) unsafeField.get(null);
+                        // the unsafe instance
-                logger.debug("sun.misc.Unsafe.theUnsafe: {}", unsafe != null ? "available" : "unavailable");
+                        return unsafeField.get(null);
+                    } catch (NoSuchFieldException e) {
+                        return e;
+                    } catch (SecurityException e) {
+                        return e;
+                    } catch (IllegalAccessException e) {
+                        return e;
+                    }
+                }
+            });
 
-                // Ensure the unsafe supports all necessary methods to work around the mistake in the latest OpenJDK.
+            // the conditional check here can not be replaced with checking that maybeUnsafe
+            // is an instanceof Unsafe and reversing the if and else blocks; this is because an
+            // instanceof check against Unsafe will trigger a class load and we might not have
+            // the runtime permission accessClassInPackage.sun.misc
+            if (maybeUnsafe instanceof Exception) {
+                unsafe = null;
+                logger.debug("sun.misc.Unsafe.theUnsafe: unavailable", (Exception) maybeUnsafe);
+            } else {
+                unsafe = (Unsafe) maybeUnsafe;
+                logger.debug("sun.misc.Unsafe.theUnsafe: available");
+            }
+
+            // ensure the unsafe supports all necessary methods to work around the mistake in the latest OpenJDK
             // https://github.com/netty/netty/issues/1061
             // http://www.mail-archive.com/jdk6-dev@openjdk.java.net/msg00698.html
-                try {
             if (unsafe != null) {
-                        unsafe.getClass().getDeclaredMethod(
+                final Unsafe finalUnsafe = unsafe;
+                final Object maybeException = AccessController.doPrivileged(new PrivilegedAction<Object>() {
+                    @Override
+                    public Object run() {
+                        try {
+                            finalUnsafe.getClass().getDeclaredMethod(
                                     "copyMemory", Object.class, long.class, Object.class, long.class, long.class);
-                        logger.debug("sun.misc.Unsafe.copyMemory: available");
+                            return null;
-                    }
-                } catch (NoSuchMethodError t) {
-                    logger.debug("sun.misc.Unsafe.copyMemory: unavailable");
-                    throw t;
                         } catch (NoSuchMethodException e) {
-                    logger.debug("sun.misc.Unsafe.copyMemory: unavailable");
+                            return e;
-                    throw e;
+                        } catch (SecurityException e) {
+                            return e;
                         }
-            } catch (Throwable cause) {
+                    }
+                });
+
+                if (maybeException == null) {
+                    logger.debug("sun.misc.Unsafe.copyMemory: available");
+                } else {
                     // Unsafe.copyMemory(Object, long, Object, long, long) unavailable.
                     unsafe = null;
+                    logger.debug("sun.misc.Unsafe.copyMemory: unavailable", (Exception) maybeException);
+                }
             }
         } else {
             // If we cannot access the address of a direct buffer, there's no point of using unsafe.
@@ -118,14 +168,43 @@ final class PlatformDependent0 {
             Constructor<?> directBufferConstructor;
             long address = -1;
             try {
-                directBufferConstructor = direct.getClass().getDeclaredConstructor(long.class, int.class);
+                final Object maybeDirectBufferConstructor =
-                directBufferConstructor.setAccessible(true);
+                        AccessController.doPrivileged(new PrivilegedAction<Object>() {
-                address = UNSAFE.allocateMemory(1);
+                            @Override
+                            public Object run() {
+                                try {
+                                    final Constructor constructor =
+                                            direct.getClass().getDeclaredConstructor(long.class, int.class);
+                                    constructor.setAccessible(true);
+                                    return constructor;
+                                } catch (NoSuchMethodException e) {
+                                    return e;
+                                } catch (SecurityException e) {
+                                    return e;
+                                }
+                            }
+                        });
 
-                // Try to use the constructor now
+                if (maybeDirectBufferConstructor instanceof Constructor<?>) {
-                directBufferConstructor.newInstance(address, 1);
+                    address = UNSAFE.allocateMemory(1);
-            } catch (Throwable t) {
+                    // try to use the constructor now
+                    try {
+                        ((Constructor) maybeDirectBufferConstructor).newInstance(address, 1);
+                        directBufferConstructor = (Constructor<?>) maybeDirectBufferConstructor;
+                        logger.debug("direct buffer constructor: available");
+                    } catch (InstantiationException e) {
                         directBufferConstructor = null;
+                    } catch (IllegalAccessException e) {
+                        directBufferConstructor = null;
+                    } catch (InvocationTargetException e) {
+                        directBufferConstructor = null;
+                    }
+                } else {
+                    logger.debug(
+                            "direct buffer constructor: unavailable",
+                            (Exception) maybeDirectBufferConstructor);
+                    directBufferConstructor = null;
+                }
             } finally {
                 if (address != -1) {
                     UNSAFE.freeMemory(address);
@@ -136,24 +215,45 @@ final class PlatformDependent0 {
             ADDRESS_FIELD_OFFSET = objectFieldOffset(addressField);
             BYTE_ARRAY_BASE_OFFSET = UNSAFE.arrayBaseOffset(byte[].class);
             boolean unaligned;
+            Object maybeUnaligned = AccessController.doPrivileged(new PrivilegedAction<Object>() {
+                @Override
+                public Object run() {
                     try {
-                Class<?> bitsClass = Class.forName("java.nio.Bits", false, ClassLoader.getSystemClassLoader());
+                        Class<?> bitsClass =
+                                Class.forName("java.nio.Bits", false, PlatformDependent.getSystemClassLoader());
                         Method unalignedMethod = bitsClass.getDeclaredMethod("unaligned");
                         unalignedMethod.setAccessible(true);
-                unaligned = Boolean.TRUE.equals(unalignedMethod.invoke(null));
+                        return unalignedMethod.invoke(null);
-            } catch (Throwable t) {
+                    } catch (ClassNotFoundException e) {
-                // We at least know x86 and x64 support unaligned access.
+                        return e;
+                    } catch (NoSuchMethodException e) {
+                        return e;
+                    } catch (InvocationTargetException e) {
+                        return e;
+                    } catch (IllegalAccessException e) {
+                        return e;
+                    } catch (SecurityException e) {
+                        return e;
+                    }
+                }
+            });
+
+            if (maybeUnaligned instanceof Boolean) {
+                unaligned = (Boolean) maybeUnaligned;
+                logger.debug("java.nio.Bits.unaligned: available, {}", unaligned);
+            } else {
                 String arch = SystemPropertyUtil.get("os.arch", "");
                 //noinspection DynamicRegexReplaceableByCompiledPattern
                 unaligned = arch.matches("^(i[3-6]86|x86(_64)?|x64|amd64)$");
+                Exception e = (Exception) maybeUnaligned;
+                logger.debug("java.nio.Bits.unaligned: unavailable, " + unaligned, e);
             }
 
             UNALIGNED = unaligned;
-            logger.debug("java.nio.Bits.unaligned: {}", UNALIGNED);
         }
 
         logger.debug("java.nio.DirectByteBuffer.<init>(long, int): {}",
-                DIRECT_BUFFER_CONSTRUCTOR != null? "available" : "unavailable");
+                DIRECT_BUFFER_CONSTRUCTOR != null ? "available" : "unavailable");
 
         freeDirectBuffer(direct);
     }