Avoid integer overflow in ByteBuf.ensureWritable (#10648)

Motivation:

- To make ensureWritable throw IOOBE when maxCapacity is exceeded, even if
the requested new capacity would overflow Integer.MAX_VALUE

Modification:

- AbstractByteBuf.ensureWritable0 is modified to detect when
targetCapacity has wrapped around
- Test added for correct behaviour in AbstractByteBufTest

Result:

- Calls to ensureWritable will always throw IOOBE when maxCapacity is
exceeded (and bounds checking is enabled)

buffer/src/main/java/io/netty/buffer/AbstractByteBuf.java

@@ -235,11 +235,12 @@ public abstract class AbstractByteBuf extends ByteBuf {
     final void ensureWritable0(int minWritableBytes) {
         final int writerIndex = writerIndex();
         final int targetCapacity = writerIndex + minWritableBytes;
-        if (targetCapacity <= capacity()) {
+        // using non-short-circuit & to reduce branching - this is a hot path and targetCapacity should rarely overflow
+        if (targetCapacity >= 0 & targetCapacity <= capacity()) {
             ensureAccessible();
             return;
         }
-        if (checkBounds && targetCapacity > maxCapacity) {
+        if (checkBounds && (targetCapacity < 0 || targetCapacity > maxCapacity)) {
             ensureAccessible();
             throw new IndexOutOfBoundsException(String.format(
                     "writerIndex(%d) + minWritableBytes(%d) exceeds maxCapacity(%d): %s",

buffer/src/test/java/io/netty/buffer/AbstractByteBufTest.java

@@ -4892,4 +4892,19 @@ public abstract class AbstractByteBufTest {
         assertEquals(50, buffer.maxFastWritableBytes());
         buffer.release();
     }
+
+    @Test
+    public void testEnsureWritableIntegerOverflow() {
+        ByteBuf buffer = newBuffer(CAPACITY);
+        buffer.writerIndex(buffer.readerIndex());
+        buffer.writeByte(1);
+        try {
+            buffer.ensureWritable(Integer.MAX_VALUE);
+            fail();
+        } catch (IndexOutOfBoundsException e) {
+            // expected
+        } finally {
+            buffer.release();
+        }
+    }
 }