Not try to write more then Integer.MAX_VALUE / SSIZE_MAX via writev

Motivation:

When trying to write more then Integer.MAX_VALUE / SSIZE_MAX via writev(...) the OS may return EINVAL depending on the kernel or the actual OS (bsd / osx always return EINVAL). This will trigger an IOException.

Modifications:

Never try to write more then Integer.MAX_VALUE / SSIZE_MAX when using writev.

Result:

No more IOException when write more data then Integer.MAX_VALUE / SSIZE_MAX via writev.

transport-native-epoll/src/main/c/io_netty_channel_epoll_Native.c

@@ -31,6 +31,7 @@
 #include <fcntl.h>
 #include <sys/utsname.h>
 #include <stddef.h>
+#include <limits.h>
 #include "io_netty_channel_epoll_Native.h"
 
 /**
@@ -1629,3 +1630,7 @@ JNIEXPORT jint JNICALL Java_io_netty_channel_epoll_Native_splice0(JNIEnv* env, j
     }
     return (jint) res;
 }
+
+JNIEXPORT jlong JNICALL Java_io_netty_channel_epoll_Native_ssizeMax(JNIEnv* env, jclass clazz) {
+    return SSIZE_MAX;
+}

transport-native-epoll/src/main/c/io_netty_channel_epoll_Native.h

@@ -104,6 +104,7 @@ jint Java_io_netty_channel_epoll_Native_getSoError(JNIEnv* env, jclass clazz, ji
 jstring Java_io_netty_channel_epoll_Native_kernelVersion(JNIEnv* env, jclass clazz);
 jint Java_io_netty_channel_epoll_Native_iovMax(JNIEnv* env, jclass clazz);
 jint Java_io_netty_channel_epoll_Native_uioMaxIov(JNIEnv* env, jclass clazz);
+jlong Java_io_netty_channel_epoll_Native_ssizeMax(JNIEnv* env, jclass clazz);
 jboolean Java_io_netty_channel_epoll_Native_isSupportingSendmmsg(JNIEnv* env, jclass clazz);
 
 jint Java_io_netty_channel_epoll_Native_errnoEBADF(JNIEnv* env, jclass clazz);

transport-native-epoll/src/main/java/io/netty/channel/epoll/IovArray.java

@@ -89,19 +89,29 @@ final class IovArray implements MessageProcessor {
 
         final long addr = buf.memoryAddress();
         final int offset = buf.readerIndex();
-        add(addr, offset, len);
+        return add(addr, offset, len);
-        return true;
     }
 
-    private void add(long addr, int offset, int len) {
+    private boolean add(long addr, int offset, int len) {
         if (len == 0) {
             // No need to add an empty buffer.
-            return;
+            return true;
         }
 
         final long baseOffset = memoryAddress(count++);
         final long lengthOffset = baseOffset + ADDRESS_SIZE;
 
+        if (Native.SSIZE_MAX - len < size) {
+            // If the size + len will overflow an SSIZE_MAX we stop populate the IovArray. This is done as linux
+            //  not allow to write more bytes then SSIZE_MAX with one writev(...) call and so will
+            // return 'EINVAL', which will raise an IOException.
+            //
+            // See also:
+            // - http://linux.die.net/man/2/writev
+            return false;
+        }
+        size += len;
+
         if (ADDRESS_SIZE == 8) {
             // 64bit
             PlatformDependent.putLong(baseOffset, addr + offset);
@@ -111,8 +121,7 @@ final class IovArray implements MessageProcessor {
             PlatformDependent.putInt(baseOffset, (int) addr + offset);
             PlatformDependent.putInt(lengthOffset, len);
         }
-
+        return true;
-        size += len;
     }
 
     /**
@@ -135,7 +144,9 @@ final class IovArray implements MessageProcessor {
             }
             long addr = PlatformDependent.directBufferAddress(nioBuffer);
 
-            add(addr, offset, len);
+            if (!add(addr, offset, len)) {
+                return false;
+            }
         }
         return true;
     }

transport-native-epoll/src/main/java/io/netty/channel/epoll/Native.java

@@ -58,7 +58,7 @@ public final class Native {
     public static final int IOV_MAX = iovMax();
     public static final int UIO_MAX_IOV = uioMaxIov();
     public static final boolean IS_SUPPORTING_SENDMMSG = isSupportingSendmmsg();
-
+    public static final long SSIZE_MAX = ssizeMax();
     private static final byte[] IPV4_MAPPED_IPV6_PREFIX = {
             0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, (byte) 0xff, (byte) 0xff };
 
@@ -695,6 +695,7 @@ public final class Native {
     private static native int epollrdhup();
     private static native int epollet();
 
+    private static native long ssizeMax();
     private Native() {
         // utility
     }

transport/src/main/java/io/netty/channel/ChannelOutboundBuffer.java

@@ -370,6 +370,19 @@ public final class ChannelOutboundBuffer {
                 final int readableBytes = buf.writerIndex() - readerIndex;
 
                 if (readableBytes > 0) {
+                    if (Integer.MAX_VALUE - readableBytes < nioBufferSize) {
+                        // If the nioBufferSize + readableBytes will overflow an Integer we stop populate the
+                        // ByteBuffer array. This is done as bsd/osx don't allow to write more bytes then
+                        // Integer.MAX_VALUE with one writev(...) call and so will return 'EINVAL', which will
+                        // raise an IOException. On Linux it may work depending on the
+                        // architecture and kernel but to be safe we also enforce the limit here.
+                        // This said writing more the Integer.MAX_VALUE is not a good idea anyway.
+                        //
+                        // See also:
+                        // - https://www.freebsd.org/cgi/man.cgi?query=write&sektion=2
+                        // - http://linux.die.net/man/2/writev
+                        break;
+                    }
                     nioBufferSize += readableBytes;
                     int count = entry.count;
                     if (count == -1) {