From b26a61acd1bde5b6fca21026cd9123a38f277ad6 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Tue, 9 Apr 2019 07:22:32 -0700 Subject: [PATCH] Centralize internal reference counting logic (#8614) Motivation AbstractReferenceCounted and AbstractReferenceCountedByteBuf contain duplicate logic for managing the volatile refcount in an optimized and consistent manner, which increased in complexity in #8583. It's possible to extract this into a common helper class now that all access is via an AtomicIntegerFieldUpdater. Modifications - Move duplicate logic into a shared ReferenceCountUpdater class - Incorporate some additional simplification for the most common single increment/decrement cases (fewer checks/operations) Result Less code duplication, better encapsulation of the "non-trivial" internal volatile refcount manipulation --- .../buffer/AbstractPooledDerivedByteBuf.java | 2 +- .../AbstractReferenceCountedByteBuf.java | 150 ++++----------- .../java/io/netty/buffer/ByteBufUtil.java | 4 +- .../java/io/netty/buffer/PooledByteBuf.java | 2 +- .../netty/util/AbstractReferenceCounted.java | 131 +++---------- .../util/internal/ReferenceCountUpdater.java | 179 ++++++++++++++++++ 6 files changed, 247 insertions(+), 221 deletions(-) create mode 100644 common/src/main/java/io/netty/util/internal/ReferenceCountUpdater.java diff --git a/buffer/src/main/java/io/netty/buffer/AbstractPooledDerivedByteBuf.java b/buffer/src/main/java/io/netty/buffer/AbstractPooledDerivedByteBuf.java index 3388b14190..bc1b9c93d2 100644 --- a/buffer/src/main/java/io/netty/buffer/AbstractPooledDerivedByteBuf.java +++ b/buffer/src/main/java/io/netty/buffer/AbstractPooledDerivedByteBuf.java @@ -63,7 +63,7 @@ abstract class AbstractPooledDerivedByteBuf extends AbstractReferenceCountedByte try { maxCapacity(maxCapacity); setIndex0(readerIndex, writerIndex); // It is assumed the bounds checking is done by the caller. - setRefCnt(1); + resetRefCnt(); @SuppressWarnings("unchecked") final U castThis = (U) this; diff --git a/buffer/src/main/java/io/netty/buffer/AbstractReferenceCountedByteBuf.java b/buffer/src/main/java/io/netty/buffer/AbstractReferenceCountedByteBuf.java index 548bdba5e4..22eea0eeaa 100644 --- a/buffer/src/main/java/io/netty/buffer/AbstractReferenceCountedByteBuf.java +++ b/buffer/src/main/java/io/netty/buffer/AbstractReferenceCountedByteBuf.java @@ -16,106 +16,73 @@ package io.netty.buffer; -import io.netty.util.IllegalReferenceCountException; -import io.netty.util.internal.PlatformDependent; - import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import static io.netty.util.internal.ObjectUtil.checkPositive; +import io.netty.util.internal.ReferenceCountUpdater; /** * Abstract base class for {@link ByteBuf} implementations that count references. */ public abstract class AbstractReferenceCountedByteBuf extends AbstractByteBuf { - private static final long REFCNT_FIELD_OFFSET; - private static final AtomicIntegerFieldUpdater refCntUpdater = + private static final long REFCNT_FIELD_OFFSET = + ReferenceCountUpdater.getUnsafeOffset(AbstractReferenceCountedByteBuf.class, "refCnt"); + private static final AtomicIntegerFieldUpdater AIF_UPDATER = AtomicIntegerFieldUpdater.newUpdater(AbstractReferenceCountedByteBuf.class, "refCnt"); - // even => "real" refcount is (refCnt >>> 1); odd => "real" refcount is 0 - @SuppressWarnings("unused") - private volatile int refCnt = 2; - - static { - long refCntFieldOffset = -1; - try { - if (PlatformDependent.hasUnsafe()) { - refCntFieldOffset = PlatformDependent.objectFieldOffset( - AbstractReferenceCountedByteBuf.class.getDeclaredField("refCnt")); - } - } catch (Throwable ignore) { - refCntFieldOffset = -1; + private static final ReferenceCountUpdater updater = + new ReferenceCountUpdater() { + @Override + protected AtomicIntegerFieldUpdater updater() { + return AIF_UPDATER; } + @Override + protected long unsafeOffset() { + return REFCNT_FIELD_OFFSET; + } + }; - REFCNT_FIELD_OFFSET = refCntFieldOffset; - } - - private static int realRefCnt(int rawCnt) { - return (rawCnt & 1) != 0 ? 0 : rawCnt >>> 1; - } + // Value might not equal "real" reference count, all access should be via the updater + @SuppressWarnings("unused") + private volatile int refCnt = updater.initialValue(); protected AbstractReferenceCountedByteBuf(int maxCapacity) { super(maxCapacity); } - private int nonVolatileRawCnt() { - // TODO: Once we compile against later versions of Java we can replace the Unsafe usage here by varhandles. - return REFCNT_FIELD_OFFSET != -1 ? PlatformDependent.getInt(this, REFCNT_FIELD_OFFSET) - : refCntUpdater.get(this); - } - @Override boolean isAccessible() { // Try to do non-volatile read for performance as the ensureAccessible() is racy anyway and only provide // a best-effort guard. - - // This is copied explicitly from the nonVolatileRawCnt() method above to reduce call stack depth, - // to avoid hitting the default limit for inlining (9) - final int rawCnt = REFCNT_FIELD_OFFSET != -1 ? PlatformDependent.getInt(this, REFCNT_FIELD_OFFSET) - : refCntUpdater.get(this); - - // The "real" ref count is > 0 if the rawCnt is even. - // (x & y) appears to be surprisingly expensive relative to (x == y). Thus the expression below provides - // a fast path for most common cases where the ref count is 1, 2, 3 or 4. - return rawCnt == 2 || rawCnt == 4 || rawCnt == 6 || rawCnt == 8 || (rawCnt & 1) == 0; + return updater.isLiveNonVolatile(this); } @Override public int refCnt() { - return realRefCnt(refCntUpdater.get(this)); + return updater.refCnt(this); } /** * An unsafe operation intended for use by a subclass that sets the reference count of the buffer directly */ - protected final void setRefCnt(int newRefCnt) { - refCntUpdater.set(this, newRefCnt << 1); // overflow OK here + protected final void setRefCnt(int refCnt) { + updater.setRefCnt(this, refCnt); + } + + /** + * An unsafe operation intended for use by a subclass that resets the reference count of the buffer to 1 + */ + protected final void resetRefCnt() { + updater.resetRefCnt(this); } @Override public ByteBuf retain() { - return retain0(1); + return updater.retain(this); } @Override public ByteBuf retain(int increment) { - return retain0(checkPositive(increment, "increment")); - } - - private ByteBuf retain0(final int increment) { - // all changes to the raw count are 2x the "real" change - int adjustedIncrement = increment << 1; // overflow OK here - int oldRef = refCntUpdater.getAndAdd(this, adjustedIncrement); - if ((oldRef & 1) != 0) { - throw new IllegalReferenceCountException(0, increment); - } - // don't pass 0! - if ((oldRef <= 0 && oldRef + adjustedIncrement >= 0) - || (oldRef >= 0 && oldRef + adjustedIncrement < oldRef)) { - // overflow case - refCntUpdater.getAndAdd(this, -adjustedIncrement); - throw new IllegalReferenceCountException(realRefCnt(oldRef), increment); - } - return this; + return updater.retain(this, increment); } @Override @@ -130,64 +97,19 @@ public abstract class AbstractReferenceCountedByteBuf extends AbstractByteBuf { @Override public boolean release() { - return release0(1); + return handleRelease(updater.release(this)); } @Override public boolean release(int decrement) { - return release0(checkPositive(decrement, "decrement")); + return handleRelease(updater.release(this, decrement)); } - private boolean release0(int decrement) { - int rawCnt = nonVolatileRawCnt(), realCnt = toLiveRealCnt(rawCnt, decrement); - if (decrement == realCnt) { - if (refCntUpdater.compareAndSet(this, rawCnt, 1)) { - deallocate(); - return true; - } - return retryRelease0(decrement); + private boolean handleRelease(boolean result) { + if (result) { + deallocate(); } - return releaseNonFinal0(decrement, rawCnt, realCnt); - } - - private boolean releaseNonFinal0(int decrement, int rawCnt, int realCnt) { - if (decrement < realCnt - // all changes to the raw count are 2x the "real" change - && refCntUpdater.compareAndSet(this, rawCnt, rawCnt - (decrement << 1))) { - return false; - } - return retryRelease0(decrement); - } - - private boolean retryRelease0(int decrement) { - for (;;) { - int rawCnt = refCntUpdater.get(this), realCnt = toLiveRealCnt(rawCnt, decrement); - if (decrement == realCnt) { - if (refCntUpdater.compareAndSet(this, rawCnt, 1)) { - deallocate(); - return true; - } - } else if (decrement < realCnt) { - // all changes to the raw count are 2x the "real" change - if (refCntUpdater.compareAndSet(this, rawCnt, rawCnt - (decrement << 1))) { - return false; - } - } else { - throw new IllegalReferenceCountException(realCnt, -decrement); - } - Thread.yield(); // this benefits throughput under high contention - } - } - - /** - * Like {@link #realRefCnt(int)} but throws if refCnt == 0 - */ - private static int toLiveRealCnt(int rawCnt, int decrement) { - if ((rawCnt & 1) == 0) { - return rawCnt >>> 1; - } - // odd rawCnt => already deallocated - throw new IllegalReferenceCountException(0, -decrement); + return result; } /** diff --git a/buffer/src/main/java/io/netty/buffer/ByteBufUtil.java b/buffer/src/main/java/io/netty/buffer/ByteBufUtil.java index 54a9f0c9fa..b1e86afce6 100644 --- a/buffer/src/main/java/io/netty/buffer/ByteBufUtil.java +++ b/buffer/src/main/java/io/netty/buffer/ByteBufUtil.java @@ -1139,7 +1139,7 @@ public final class ByteBufUtil { static ThreadLocalUnsafeDirectByteBuf newInstance() { ThreadLocalUnsafeDirectByteBuf buf = RECYCLER.get(); - buf.setRefCnt(1); + buf.resetRefCnt(); return buf; } @@ -1172,7 +1172,7 @@ public final class ByteBufUtil { static ThreadLocalDirectByteBuf newInstance() { ThreadLocalDirectByteBuf buf = RECYCLER.get(); - buf.setRefCnt(1); + buf.resetRefCnt(); return buf; } diff --git a/buffer/src/main/java/io/netty/buffer/PooledByteBuf.java b/buffer/src/main/java/io/netty/buffer/PooledByteBuf.java index beffbb07ee..3b17087e11 100644 --- a/buffer/src/main/java/io/netty/buffer/PooledByteBuf.java +++ b/buffer/src/main/java/io/netty/buffer/PooledByteBuf.java @@ -72,7 +72,7 @@ abstract class PooledByteBuf extends AbstractReferenceCountedByteBuf { */ final void reuse(int maxCapacity) { maxCapacity(maxCapacity); - setRefCnt(1); + resetRefCnt(); setIndex0(0, 0); discardMarks(); } diff --git a/common/src/main/java/io/netty/util/AbstractReferenceCounted.java b/common/src/main/java/io/netty/util/AbstractReferenceCounted.java index b7480c4eb8..b5ffb4f4b3 100644 --- a/common/src/main/java/io/netty/util/AbstractReferenceCounted.java +++ b/common/src/main/java/io/netty/util/AbstractReferenceCounted.java @@ -15,85 +15,55 @@ */ package io.netty.util; -import static io.netty.util.internal.ObjectUtil.checkPositive; - import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import io.netty.util.internal.PlatformDependent; +import io.netty.util.internal.ReferenceCountUpdater; /** * Abstract base class for classes wants to implement {@link ReferenceCounted}. */ public abstract class AbstractReferenceCounted implements ReferenceCounted { - private static final long REFCNT_FIELD_OFFSET; - private static final AtomicIntegerFieldUpdater refCntUpdater = + private static final long REFCNT_FIELD_OFFSET = + ReferenceCountUpdater.getUnsafeOffset(AbstractReferenceCounted.class, "refCnt"); + private static final AtomicIntegerFieldUpdater AIF_UPDATER = AtomicIntegerFieldUpdater.newUpdater(AbstractReferenceCounted.class, "refCnt"); - // even => "real" refcount is (refCnt >>> 1); odd => "real" refcount is 0 - @SuppressWarnings("unused") - private volatile int refCnt = 2; - - static { - long refCntFieldOffset = -1; - try { - if (PlatformDependent.hasUnsafe()) { - refCntFieldOffset = PlatformDependent.objectFieldOffset( - AbstractReferenceCounted.class.getDeclaredField("refCnt")); - } - } catch (Throwable ignore) { - refCntFieldOffset = -1; + private static final ReferenceCountUpdater updater = + new ReferenceCountUpdater() { + @Override + protected AtomicIntegerFieldUpdater updater() { + return AIF_UPDATER; } + @Override + protected long unsafeOffset() { + return REFCNT_FIELD_OFFSET; + } + }; - REFCNT_FIELD_OFFSET = refCntFieldOffset; - } - - private static int realRefCnt(int rawCnt) { - return (rawCnt & 1) != 0 ? 0 : rawCnt >>> 1; - } - - private int nonVolatileRawCnt() { - // TODO: Once we compile against later versions of Java we can replace the Unsafe usage here by varhandles. - return REFCNT_FIELD_OFFSET != -1 ? PlatformDependent.getInt(this, REFCNT_FIELD_OFFSET) - : refCntUpdater.get(this); - } + // Value might not equal "real" reference count, all access should be via the updater + @SuppressWarnings("unused") + private volatile int refCnt = updater.initialValue(); @Override public int refCnt() { - return realRefCnt(refCntUpdater.get(this)); + return updater.refCnt(this); } /** * An unsafe operation intended for use by a subclass that sets the reference count of the buffer directly */ - protected final void setRefCnt(int newRefCnt) { - refCntUpdater.set(this, newRefCnt << 1); // overflow OK here + protected final void setRefCnt(int refCnt) { + updater.setRefCnt(this, refCnt); } @Override public ReferenceCounted retain() { - return retain0(1); + return updater.retain(this); } @Override public ReferenceCounted retain(int increment) { - return retain0(checkPositive(increment, "increment")); - } - - private ReferenceCounted retain0(final int increment) { - // all changes to the raw count are 2x the "real" change - int adjustedIncrement = increment << 1; // overflow OK here - int oldRef = refCntUpdater.getAndAdd(this, adjustedIncrement); - if ((oldRef & 1) != 0) { - throw new IllegalReferenceCountException(0, increment); - } - // don't pass 0! - if ((oldRef <= 0 && oldRef + adjustedIncrement >= 0) - || (oldRef >= 0 && oldRef + adjustedIncrement < oldRef)) { - // overflow case - refCntUpdater.getAndAdd(this, -adjustedIncrement); - throw new IllegalReferenceCountException(realRefCnt(oldRef), increment); - } - return this; + return updater.retain(this, increment); } @Override @@ -103,64 +73,19 @@ public abstract class AbstractReferenceCounted implements ReferenceCounted { @Override public boolean release() { - return release0(1); + return handleRelease(updater.release(this)); } @Override public boolean release(int decrement) { - return release0(checkPositive(decrement, "decrement")); + return handleRelease(updater.release(this, decrement)); } - private boolean release0(int decrement) { - int rawCnt = nonVolatileRawCnt(), realCnt = toLiveRealCnt(rawCnt, decrement); - if (decrement == realCnt) { - if (refCntUpdater.compareAndSet(this, rawCnt, 1)) { - deallocate(); - return true; - } - return retryRelease0(decrement); + private boolean handleRelease(boolean result) { + if (result) { + deallocate(); } - return releaseNonFinal0(decrement, rawCnt, realCnt); - } - - private boolean releaseNonFinal0(int decrement, int rawCnt, int realCnt) { - if (decrement < realCnt - // all changes to the raw count are 2x the "real" change - && refCntUpdater.compareAndSet(this, rawCnt, rawCnt - (decrement << 1))) { - return false; - } - return retryRelease0(decrement); - } - - private boolean retryRelease0(int decrement) { - for (;;) { - int rawCnt = refCntUpdater.get(this), realCnt = toLiveRealCnt(rawCnt, decrement); - if (decrement == realCnt) { - if (refCntUpdater.compareAndSet(this, rawCnt, 1)) { - deallocate(); - return true; - } - } else if (decrement < realCnt) { - // all changes to the raw count are 2x the "real" change - if (refCntUpdater.compareAndSet(this, rawCnt, rawCnt - (decrement << 1))) { - return false; - } - } else { - throw new IllegalReferenceCountException(realCnt, -decrement); - } - Thread.yield(); // this benefits throughput under high contention - } - } - - /** - * Like {@link #realRefCnt(int)} but throws if refCnt == 0 - */ - private static int toLiveRealCnt(int rawCnt, int decrement) { - if ((rawCnt & 1) == 0) { - return rawCnt >>> 1; - } - // odd rawCnt => already deallocated - throw new IllegalReferenceCountException(0, -decrement); + return result; } /** diff --git a/common/src/main/java/io/netty/util/internal/ReferenceCountUpdater.java b/common/src/main/java/io/netty/util/internal/ReferenceCountUpdater.java new file mode 100644 index 0000000000..d6d131bddb --- /dev/null +++ b/common/src/main/java/io/netty/util/internal/ReferenceCountUpdater.java @@ -0,0 +1,179 @@ +/* + * Copyright 2019 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package io.netty.util.internal; + +import static io.netty.util.internal.ObjectUtil.checkPositive; + +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; + +import io.netty.util.IllegalReferenceCountException; +import io.netty.util.ReferenceCounted; + +/** + * Common logic for {@link ReferenceCounted} implementations + */ +public abstract class ReferenceCountUpdater { + /* + * Implementation notes: + * + * For the updated int field: + * Even => "real" refcount is (refCnt >>> 1) + * Odd => "real" refcount is 0 + * + * (x & y) appears to be surprisingly expensive relative to (x == y). Thus this class uses + * a fast-path in some places for most common low values when checking for live (even) refcounts, + * for example: if (rawCnt == 2 || rawCnt == 4 || (rawCnt & 1) == 0) { ... + */ + + protected ReferenceCountUpdater() { } + + public static long getUnsafeOffset(Class clz, String fieldName) { + try { + if (PlatformDependent.hasUnsafe()) { + return PlatformDependent.objectFieldOffset(clz.getDeclaredField(fieldName)); + } + } catch (Throwable ignore) { + // fall-back + } + return -1; + } + + protected abstract AtomicIntegerFieldUpdater updater(); + + protected abstract long unsafeOffset(); + + public final int initialValue() { + return 2; + } + + private static int realRefCnt(int rawCnt) { + return rawCnt != 2 && rawCnt != 4 && (rawCnt & 1) != 0 ? 0 : rawCnt >>> 1; + } + + /** + * Like {@link #realRefCnt(int)} but throws if refCnt == 0 + */ + private static int toLiveRealRefCnt(int rawCnt, int decrement) { + if (rawCnt == 2 || rawCnt == 4 || (rawCnt & 1) == 0) { + return rawCnt >>> 1; + } + // odd rawCnt => already deallocated + throw new IllegalReferenceCountException(0, -decrement); + } + + private int nonVolatileRawCnt(T instance) { + // TODO: Once we compile against later versions of Java we can replace the Unsafe usage here by varhandles. + final long offset = unsafeOffset(); + return offset != -1 ? PlatformDependent.getInt(instance, offset) : updater().get(instance); + } + + public final int refCnt(T instance) { + return realRefCnt(updater().get(instance)); + } + + public final boolean isLiveNonVolatile(T instance) { + final long offset = unsafeOffset(); + final int rawCnt = offset != -1 ? PlatformDependent.getInt(instance, offset) : updater().get(instance); + + // The "real" ref count is > 0 if the rawCnt is even. + return rawCnt == 2 || rawCnt == 4 || rawCnt == 6 || rawCnt == 8 || (rawCnt & 1) == 0; + } + + /** + * An unsafe operation that sets the reference count directly + */ + public final void setRefCnt(T instance, int refCnt) { + updater().set(instance, refCnt > 0 ? refCnt << 1 : 1); // overflow OK here + } + + /** + * Resets the reference count to 1 + */ + public final void resetRefCnt(T instance) { + updater().set(instance, initialValue()); + } + + public final T retain(T instance) { + return retain0(instance, 1, 2); + } + + public final T retain(T instance, int increment) { + // all changes to the raw count are 2x the "real" change - overflow is OK + int rawIncrement = checkPositive(increment, "increment") << 1; + return retain0(instance, increment, rawIncrement); + } + + // rawIncrement == increment << 1 + private T retain0(T instance, final int increment, final int rawIncrement) { + int oldRef = updater().getAndAdd(instance, rawIncrement); + if (oldRef != 2 && oldRef != 4 && (oldRef & 1) != 0) { + throw new IllegalReferenceCountException(0, increment); + } + // don't pass 0! + if ((oldRef <= 0 && oldRef + rawIncrement >= 0) + || (oldRef >= 0 && oldRef + rawIncrement < oldRef)) { + // overflow case + updater().getAndAdd(instance, -rawIncrement); + throw new IllegalReferenceCountException(realRefCnt(oldRef), increment); + } + return instance; + } + + public final boolean release(T instance) { + int rawCnt = nonVolatileRawCnt(instance); + return rawCnt == 2 ? tryFinalRelease0(instance, 2) || retryRelease0(instance, 1) + : nonFinalRelease0(instance, 1, rawCnt, toLiveRealRefCnt(rawCnt, 1)); + } + + public final boolean release(T instance, int decrement) { + int rawCnt = nonVolatileRawCnt(instance); + int realCnt = toLiveRealRefCnt(rawCnt, checkPositive(decrement, "decrement")); + return decrement == realCnt ? tryFinalRelease0(instance, rawCnt) || retryRelease0(instance, decrement) + : nonFinalRelease0(instance, decrement, rawCnt, realCnt); + } + + private boolean tryFinalRelease0(T instance, int expectRawCnt) { + return updater().compareAndSet(instance, expectRawCnt, 1); // any odd number will work + } + + private boolean nonFinalRelease0(T instance, int decrement, int rawCnt, int realCnt) { + if (decrement < realCnt + // all changes to the raw count are 2x the "real" change - overflow is OK + && updater().compareAndSet(instance, rawCnt, rawCnt - (decrement << 1))) { + return false; + } + return retryRelease0(instance, decrement); + } + + private boolean retryRelease0(T instance, int decrement) { + for (;;) { + int rawCnt = updater().get(instance), realCnt = toLiveRealRefCnt(rawCnt, decrement); + if (decrement == realCnt) { + if (tryFinalRelease0(instance, rawCnt)) { + return true; + } + } else if (decrement < realCnt) { + // all changes to the raw count are 2x the "real" change + if (updater().compareAndSet(instance, rawCnt, rawCnt - (decrement << 1))) { + return false; + } + } else { + throw new IllegalReferenceCountException(realCnt, -decrement); + } + Thread.yield(); // this benefits throughput under high contention + } + } +}