Centralize internal reference counting logic (#8614)

Motivation

AbstractReferenceCounted and AbstractReferenceCountedByteBuf contain
duplicate logic for managing the volatile refcount in an optimized and
consistent manner, which increased in complexity in #8583. It's possible
to extract this into a common helper class now that all access is via an
AtomicIntegerFieldUpdater.

Modifications

- Move duplicate logic into a shared ReferenceCountUpdater class
- Incorporate some additional simplification for the most common single
increment/decrement cases (fewer checks/operations)

Result

Less code duplication, better encapsulation of the "non-trivial"
internal volatile refcount manipulation

buffer/src/main/java/io/netty/buffer/AbstractPooledDerivedByteBuf.java

@@ -63,7 +63,7 @@ abstract class AbstractPooledDerivedByteBuf extends AbstractReferenceCountedByte
         try {
             maxCapacity(maxCapacity);
             setIndex0(readerIndex, writerIndex); // It is assumed the bounds checking is done by the caller.
-            setRefCnt(1);
+            resetRefCnt();
 
             @SuppressWarnings("unchecked")
             final U castThis = (U) this;

buffer/src/main/java/io/netty/buffer/AbstractReferenceCountedByteBuf.java

@@ -16,106 +16,73 @@
 
 package io.netty.buffer;
 
-import io.netty.util.IllegalReferenceCountException;
-import io.netty.util.internal.PlatformDependent;
-
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 
-import static io.netty.util.internal.ObjectUtil.checkPositive;
+import io.netty.util.internal.ReferenceCountUpdater;
 
 /**
  * Abstract base class for {@link ByteBuf} implementations that count references.
  */
 public abstract class AbstractReferenceCountedByteBuf extends AbstractByteBuf {
-    private static final long REFCNT_FIELD_OFFSET;
+    private static final long REFCNT_FIELD_OFFSET =
-    private static final AtomicIntegerFieldUpdater<AbstractReferenceCountedByteBuf> refCntUpdater =
+            ReferenceCountUpdater.getUnsafeOffset(AbstractReferenceCountedByteBuf.class, "refCnt");
+    private static final AtomicIntegerFieldUpdater<AbstractReferenceCountedByteBuf> AIF_UPDATER =
             AtomicIntegerFieldUpdater.newUpdater(AbstractReferenceCountedByteBuf.class, "refCnt");
 
-    // even => "real" refcount is (refCnt >>> 1); odd => "real" refcount is 0
+    private static final ReferenceCountUpdater<AbstractReferenceCountedByteBuf> updater =
+            new ReferenceCountUpdater<AbstractReferenceCountedByteBuf>() {
+        @Override
+        protected AtomicIntegerFieldUpdater<AbstractReferenceCountedByteBuf> updater() {
+            return AIF_UPDATER;
+        }
+        @Override
+        protected long unsafeOffset() {
+            return REFCNT_FIELD_OFFSET;
+        }
+    };
+
+    // Value might not equal "real" reference count, all access should be via the updater
     @SuppressWarnings("unused")
-    private volatile int refCnt = 2;
+    private volatile int refCnt = updater.initialValue();
-
-    static {
-        long refCntFieldOffset = -1;
-        try {
-            if (PlatformDependent.hasUnsafe()) {
-                refCntFieldOffset = PlatformDependent.objectFieldOffset(
-                        AbstractReferenceCountedByteBuf.class.getDeclaredField("refCnt"));
-            }
-        } catch (Throwable ignore) {
-            refCntFieldOffset = -1;
-        }
-
-        REFCNT_FIELD_OFFSET = refCntFieldOffset;
-    }
-
-    private static int realRefCnt(int rawCnt) {
-        return (rawCnt & 1) != 0 ? 0 : rawCnt >>> 1;
-    }
 
     protected AbstractReferenceCountedByteBuf(int maxCapacity) {
         super(maxCapacity);
     }
 
-    private int nonVolatileRawCnt() {
-        // TODO: Once we compile against later versions of Java we can replace the Unsafe usage here by varhandles.
-        return REFCNT_FIELD_OFFSET != -1 ? PlatformDependent.getInt(this, REFCNT_FIELD_OFFSET)
-                : refCntUpdater.get(this);
-    }
-
     @Override
     boolean isAccessible() {
         // Try to do non-volatile read for performance as the ensureAccessible() is racy anyway and only provide
         // a best-effort guard.
-
+        return updater.isLiveNonVolatile(this);
-        // This is copied explicitly from the nonVolatileRawCnt() method above to reduce call stack depth,
-        // to avoid hitting the default limit for inlining (9)
-        final int rawCnt = REFCNT_FIELD_OFFSET != -1 ? PlatformDependent.getInt(this, REFCNT_FIELD_OFFSET)
-                : refCntUpdater.get(this);
-
-        // The "real" ref count is > 0 if the rawCnt is even.
-        // (x & y) appears to be surprisingly expensive relative to (x == y). Thus the expression below provides
-        // a fast path for most common cases where the ref count is 1, 2, 3 or 4.
-        return rawCnt == 2 || rawCnt == 4 || rawCnt == 6 || rawCnt == 8 || (rawCnt & 1) == 0;
     }
 
     @Override
     public int refCnt() {
-        return realRefCnt(refCntUpdater.get(this));
+        return updater.refCnt(this);
     }
 
     /**
      * An unsafe operation intended for use by a subclass that sets the reference count of the buffer directly
      */
-    protected final void setRefCnt(int newRefCnt) {
+    protected final void setRefCnt(int refCnt) {
-        refCntUpdater.set(this, newRefCnt << 1); // overflow OK here
+        updater.setRefCnt(this, refCnt);
+    }
+
+    /**
+     * An unsafe operation intended for use by a subclass that resets the reference count of the buffer to 1
+     */
+    protected final void resetRefCnt() {
+        updater.resetRefCnt(this);
     }
 
     @Override
     public ByteBuf retain() {
-        return retain0(1);
+        return updater.retain(this);
     }
 
     @Override
     public ByteBuf retain(int increment) {
-        return retain0(checkPositive(increment, "increment"));
+        return updater.retain(this, increment);
-    }
-
-    private ByteBuf retain0(final int increment) {
-        // all changes to the raw count are 2x the "real" change
-        int adjustedIncrement = increment << 1; // overflow OK here
-        int oldRef = refCntUpdater.getAndAdd(this, adjustedIncrement);
-        if ((oldRef & 1) != 0) {
-            throw new IllegalReferenceCountException(0, increment);
-        }
-        // don't pass 0!
-        if ((oldRef <= 0 && oldRef + adjustedIncrement >= 0)
-                || (oldRef >= 0 && oldRef + adjustedIncrement < oldRef)) {
-            // overflow case
-            refCntUpdater.getAndAdd(this, -adjustedIncrement);
-            throw new IllegalReferenceCountException(realRefCnt(oldRef), increment);
-        }
-        return this;
     }
 
     @Override
@@ -130,64 +97,19 @@ public abstract class AbstractReferenceCountedByteBuf extends AbstractByteBuf {
 
     @Override
     public boolean release() {
-        return release0(1);
+        return handleRelease(updater.release(this));
     }
 
     @Override
     public boolean release(int decrement) {
-        return release0(checkPositive(decrement, "decrement"));
+        return handleRelease(updater.release(this, decrement));
     }
 
-    private boolean release0(int decrement) {
+    private boolean handleRelease(boolean result) {
-        int rawCnt = nonVolatileRawCnt(), realCnt = toLiveRealCnt(rawCnt, decrement);
+        if (result) {
-        if (decrement == realCnt) {
-            if (refCntUpdater.compareAndSet(this, rawCnt, 1)) {
             deallocate();
-                return true;
         }
-            return retryRelease0(decrement);
+        return result;
-        }
-        return releaseNonFinal0(decrement, rawCnt, realCnt);
-    }
-
-    private boolean releaseNonFinal0(int decrement, int rawCnt, int realCnt) {
-        if (decrement < realCnt
-                // all changes to the raw count are 2x the "real" change
-                && refCntUpdater.compareAndSet(this, rawCnt, rawCnt - (decrement << 1))) {
-            return false;
-        }
-        return retryRelease0(decrement);
-    }
-
-    private boolean retryRelease0(int decrement) {
-        for (;;) {
-            int rawCnt = refCntUpdater.get(this), realCnt = toLiveRealCnt(rawCnt, decrement);
-            if (decrement == realCnt) {
-                if (refCntUpdater.compareAndSet(this, rawCnt, 1)) {
-                    deallocate();
-                    return true;
-                }
-            } else if (decrement < realCnt) {
-                // all changes to the raw count are 2x the "real" change
-                if (refCntUpdater.compareAndSet(this, rawCnt, rawCnt - (decrement << 1))) {
-                    return false;
-                }
-            } else {
-                throw new IllegalReferenceCountException(realCnt, -decrement);
-            }
-            Thread.yield(); // this benefits throughput under high contention
-        }
-    }
-
-    /**
-     * Like {@link #realRefCnt(int)} but throws if refCnt == 0
-     */
-    private static int toLiveRealCnt(int rawCnt, int decrement) {
-        if ((rawCnt & 1) == 0) {
-            return rawCnt >>> 1;
-        }
-        // odd rawCnt => already deallocated
-        throw new IllegalReferenceCountException(0, -decrement);
     }
 
     /**

buffer/src/main/java/io/netty/buffer/ByteBufUtil.java

@@ -1139,7 +1139,7 @@ public final class ByteBufUtil {
 
         static ThreadLocalUnsafeDirectByteBuf newInstance() {
             ThreadLocalUnsafeDirectByteBuf buf = RECYCLER.get();
-            buf.setRefCnt(1);
+            buf.resetRefCnt();
             return buf;
         }
 
@@ -1172,7 +1172,7 @@ public final class ByteBufUtil {
 
         static ThreadLocalDirectByteBuf newInstance() {
             ThreadLocalDirectByteBuf buf = RECYCLER.get();
-            buf.setRefCnt(1);
+            buf.resetRefCnt();
             return buf;
         }
 

buffer/src/main/java/io/netty/buffer/PooledByteBuf.java

@@ -72,7 +72,7 @@ abstract class PooledByteBuf<T> extends AbstractReferenceCountedByteBuf {
      */
     final void reuse(int maxCapacity) {
         maxCapacity(maxCapacity);
-        setRefCnt(1);
+        resetRefCnt();
         setIndex0(0, 0);
         discardMarks();
     }

common/src/main/java/io/netty/util/AbstractReferenceCounted.java

@@ -15,85 +15,55 @@
  */
 package io.netty.util;
 
-import static io.netty.util.internal.ObjectUtil.checkPositive;
-
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 
-import io.netty.util.internal.PlatformDependent;
+import io.netty.util.internal.ReferenceCountUpdater;
 
 /**
  * Abstract base class for classes wants to implement {@link ReferenceCounted}.
  */
 public abstract class AbstractReferenceCounted implements ReferenceCounted {
-    private static final long REFCNT_FIELD_OFFSET;
+    private static final long REFCNT_FIELD_OFFSET =
-    private static final AtomicIntegerFieldUpdater<AbstractReferenceCounted> refCntUpdater =
+            ReferenceCountUpdater.getUnsafeOffset(AbstractReferenceCounted.class, "refCnt");
+    private static final AtomicIntegerFieldUpdater<AbstractReferenceCounted> AIF_UPDATER =
             AtomicIntegerFieldUpdater.newUpdater(AbstractReferenceCounted.class, "refCnt");
 
-    // even => "real" refcount is (refCnt >>> 1); odd => "real" refcount is 0
+    private static final ReferenceCountUpdater<AbstractReferenceCounted> updater =
+            new ReferenceCountUpdater<AbstractReferenceCounted>() {
+        @Override
+        protected AtomicIntegerFieldUpdater<AbstractReferenceCounted> updater() {
+            return AIF_UPDATER;
+        }
+        @Override
+        protected long unsafeOffset() {
+            return REFCNT_FIELD_OFFSET;
+        }
+    };
+
+    // Value might not equal "real" reference count, all access should be via the updater
     @SuppressWarnings("unused")
-    private volatile int refCnt = 2;
+    private volatile int refCnt = updater.initialValue();
-
-    static {
-        long refCntFieldOffset = -1;
-        try {
-            if (PlatformDependent.hasUnsafe()) {
-                refCntFieldOffset = PlatformDependent.objectFieldOffset(
-                        AbstractReferenceCounted.class.getDeclaredField("refCnt"));
-            }
-        } catch (Throwable ignore) {
-            refCntFieldOffset = -1;
-        }
-
-        REFCNT_FIELD_OFFSET = refCntFieldOffset;
-    }
-
-    private static int realRefCnt(int rawCnt) {
-        return (rawCnt & 1) != 0 ? 0 : rawCnt >>> 1;
-    }
-
-    private int nonVolatileRawCnt() {
-        // TODO: Once we compile against later versions of Java we can replace the Unsafe usage here by varhandles.
-        return REFCNT_FIELD_OFFSET != -1 ? PlatformDependent.getInt(this, REFCNT_FIELD_OFFSET)
-                : refCntUpdater.get(this);
-    }
 
     @Override
     public int refCnt() {
-        return realRefCnt(refCntUpdater.get(this));
+        return updater.refCnt(this);
     }
 
     /**
      * An unsafe operation intended for use by a subclass that sets the reference count of the buffer directly
      */
-    protected final void setRefCnt(int newRefCnt) {
+    protected final void setRefCnt(int refCnt) {
-        refCntUpdater.set(this, newRefCnt << 1); // overflow OK here
+        updater.setRefCnt(this, refCnt);
     }
 
     @Override
     public ReferenceCounted retain() {
-        return retain0(1);
+        return updater.retain(this);
     }
 
     @Override
     public ReferenceCounted retain(int increment) {
-        return retain0(checkPositive(increment, "increment"));
+        return updater.retain(this, increment);
-    }
-
-    private ReferenceCounted retain0(final int increment) {
-        // all changes to the raw count are 2x the "real" change
-        int adjustedIncrement = increment << 1; // overflow OK here
-        int oldRef = refCntUpdater.getAndAdd(this, adjustedIncrement);
-        if ((oldRef & 1) != 0) {
-            throw new IllegalReferenceCountException(0, increment);
-        }
-        // don't pass 0!
-        if ((oldRef <= 0 && oldRef + adjustedIncrement >= 0)
-                || (oldRef >= 0 && oldRef + adjustedIncrement < oldRef)) {
-            // overflow case
-            refCntUpdater.getAndAdd(this, -adjustedIncrement);
-            throw new IllegalReferenceCountException(realRefCnt(oldRef), increment);
-        }
-        return this;
     }
 
     @Override
@@ -103,64 +73,19 @@ public abstract class AbstractReferenceCounted implements ReferenceCounted {
 
     @Override
     public boolean release() {
-        return release0(1);
+        return handleRelease(updater.release(this));
     }
 
     @Override
     public boolean release(int decrement) {
-        return release0(checkPositive(decrement, "decrement"));
+        return handleRelease(updater.release(this, decrement));
     }
 
-    private boolean release0(int decrement) {
+    private boolean handleRelease(boolean result) {
-        int rawCnt = nonVolatileRawCnt(), realCnt = toLiveRealCnt(rawCnt, decrement);
+        if (result) {
-        if (decrement == realCnt) {
-            if (refCntUpdater.compareAndSet(this, rawCnt, 1)) {
             deallocate();
-                return true;
         }
-            return retryRelease0(decrement);
+        return result;
-        }
-        return releaseNonFinal0(decrement, rawCnt, realCnt);
-    }
-
-    private boolean releaseNonFinal0(int decrement, int rawCnt, int realCnt) {
-        if (decrement < realCnt
-                // all changes to the raw count are 2x the "real" change
-                && refCntUpdater.compareAndSet(this, rawCnt, rawCnt - (decrement << 1))) {
-            return false;
-        }
-        return retryRelease0(decrement);
-    }
-
-    private boolean retryRelease0(int decrement) {
-        for (;;) {
-            int rawCnt = refCntUpdater.get(this), realCnt = toLiveRealCnt(rawCnt, decrement);
-            if (decrement == realCnt) {
-                if (refCntUpdater.compareAndSet(this, rawCnt, 1)) {
-                    deallocate();
-                    return true;
-                }
-            } else if (decrement < realCnt) {
-                // all changes to the raw count are 2x the "real" change
-                if (refCntUpdater.compareAndSet(this, rawCnt, rawCnt - (decrement << 1))) {
-                    return false;
-                }
-            } else {
-                throw new IllegalReferenceCountException(realCnt, -decrement);
-            }
-            Thread.yield(); // this benefits throughput under high contention
-        }
-    }
-
-    /**
-     * Like {@link #realRefCnt(int)} but throws if refCnt == 0
-     */
-    private static int toLiveRealCnt(int rawCnt, int decrement) {
-        if ((rawCnt & 1) == 0) {
-            return rawCnt >>> 1;
-        }
-        // odd rawCnt => already deallocated
-        throw new IllegalReferenceCountException(0, -decrement);
     }
 
     /**

common/src/main/java/io/netty/util/internal/ReferenceCountUpdater.java

@@ -0,0 +1,179 @@
+/*
+ * Copyright 2019 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.util.internal;
+
+import static io.netty.util.internal.ObjectUtil.checkPositive;
+
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+
+import io.netty.util.IllegalReferenceCountException;
+import io.netty.util.ReferenceCounted;
+
+/**
+ * Common logic for {@link ReferenceCounted} implementations
+ */
+public abstract class ReferenceCountUpdater<T extends ReferenceCounted> {
+    /*
+     * Implementation notes:
+     *
+     * For the updated int field:
+     *   Even => "real" refcount is (refCnt >>> 1)
+     *   Odd  => "real" refcount is 0
+     *
+     * (x & y) appears to be surprisingly expensive relative to (x == y). Thus this class uses
+     * a fast-path in some places for most common low values when checking for live (even) refcounts,
+     * for example: if (rawCnt == 2 || rawCnt == 4 || (rawCnt & 1) == 0) { ...
+     */
+
+    protected ReferenceCountUpdater() { }
+
+    public static long getUnsafeOffset(Class<? extends ReferenceCounted> clz, String fieldName) {
+        try {
+            if (PlatformDependent.hasUnsafe()) {
+                return PlatformDependent.objectFieldOffset(clz.getDeclaredField(fieldName));
+            }
+        } catch (Throwable ignore) {
+            // fall-back
+        }
+        return -1;
+    }
+
+    protected abstract AtomicIntegerFieldUpdater<T> updater();
+
+    protected abstract long unsafeOffset();
+
+    public final int initialValue() {
+        return 2;
+    }
+
+    private static int realRefCnt(int rawCnt) {
+        return rawCnt != 2 && rawCnt != 4 && (rawCnt & 1) != 0 ? 0 : rawCnt >>> 1;
+    }
+
+    /**
+     * Like {@link #realRefCnt(int)} but throws if refCnt == 0
+     */
+    private static int toLiveRealRefCnt(int rawCnt, int decrement) {
+        if (rawCnt == 2 || rawCnt == 4 || (rawCnt & 1) == 0) {
+            return rawCnt >>> 1;
+        }
+        // odd rawCnt => already deallocated
+        throw new IllegalReferenceCountException(0, -decrement);
+    }
+
+    private int nonVolatileRawCnt(T instance) {
+        // TODO: Once we compile against later versions of Java we can replace the Unsafe usage here by varhandles.
+        final long offset = unsafeOffset();
+        return offset != -1 ? PlatformDependent.getInt(instance, offset) : updater().get(instance);
+    }
+
+    public final int refCnt(T instance) {
+        return realRefCnt(updater().get(instance));
+    }
+
+    public final boolean isLiveNonVolatile(T instance) {
+        final long offset = unsafeOffset();
+        final int rawCnt = offset != -1 ? PlatformDependent.getInt(instance, offset) : updater().get(instance);
+
+        // The "real" ref count is > 0 if the rawCnt is even.
+        return rawCnt == 2 || rawCnt == 4 || rawCnt == 6 || rawCnt == 8 || (rawCnt & 1) == 0;
+    }
+
+    /**
+     * An unsafe operation that sets the reference count directly
+     */
+    public final void setRefCnt(T instance, int refCnt) {
+        updater().set(instance, refCnt > 0 ? refCnt << 1 : 1); // overflow OK here
+    }
+
+    /**
+     * Resets the reference count to 1
+     */
+    public final void resetRefCnt(T instance) {
+        updater().set(instance, initialValue());
+    }
+
+    public final T retain(T instance) {
+        return retain0(instance, 1, 2);
+    }
+
+    public final T retain(T instance, int increment) {
+        // all changes to the raw count are 2x the "real" change - overflow is OK
+        int rawIncrement = checkPositive(increment, "increment") << 1;
+        return retain0(instance, increment, rawIncrement);
+    }
+
+    // rawIncrement == increment << 1
+    private T retain0(T instance, final int increment, final int rawIncrement) {
+        int oldRef = updater().getAndAdd(instance, rawIncrement);
+        if (oldRef != 2 && oldRef != 4 && (oldRef & 1) != 0) {
+            throw new IllegalReferenceCountException(0, increment);
+        }
+        // don't pass 0!
+        if ((oldRef <= 0 && oldRef + rawIncrement >= 0)
+                || (oldRef >= 0 && oldRef + rawIncrement < oldRef)) {
+            // overflow case
+            updater().getAndAdd(instance, -rawIncrement);
+            throw new IllegalReferenceCountException(realRefCnt(oldRef), increment);
+        }
+        return instance;
+    }
+
+    public final boolean release(T instance) {
+        int rawCnt = nonVolatileRawCnt(instance);
+        return rawCnt == 2 ? tryFinalRelease0(instance, 2) || retryRelease0(instance, 1)
+                : nonFinalRelease0(instance, 1, rawCnt, toLiveRealRefCnt(rawCnt, 1));
+    }
+
+    public final boolean release(T instance, int decrement) {
+        int rawCnt = nonVolatileRawCnt(instance);
+        int realCnt = toLiveRealRefCnt(rawCnt, checkPositive(decrement, "decrement"));
+        return decrement == realCnt ? tryFinalRelease0(instance, rawCnt) || retryRelease0(instance, decrement)
+                : nonFinalRelease0(instance, decrement, rawCnt, realCnt);
+    }
+
+    private boolean tryFinalRelease0(T instance, int expectRawCnt) {
+        return updater().compareAndSet(instance, expectRawCnt, 1); // any odd number will work
+    }
+
+    private boolean nonFinalRelease0(T instance, int decrement, int rawCnt, int realCnt) {
+        if (decrement < realCnt
+                // all changes to the raw count are 2x the "real" change - overflow is OK
+                && updater().compareAndSet(instance, rawCnt, rawCnt - (decrement << 1))) {
+            return false;
+        }
+        return retryRelease0(instance, decrement);
+    }
+
+    private boolean retryRelease0(T instance, int decrement) {
+        for (;;) {
+            int rawCnt = updater().get(instance), realCnt = toLiveRealRefCnt(rawCnt, decrement);
+            if (decrement == realCnt) {
+                if (tryFinalRelease0(instance, rawCnt)) {
+                    return true;
+                }
+            } else if (decrement < realCnt) {
+                // all changes to the raw count are 2x the "real" change
+                if (updater().compareAndSet(instance, rawCnt, rawCnt - (decrement << 1))) {
+                    return false;
+                }
+            } else {
+                throw new IllegalReferenceCountException(realCnt, -decrement);
+            }
+            Thread.yield(); // this benefits throughput under high contention
+        }
+    }
+}