From e6a238b14d9613b7905f152945b8458f83292d2e Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Fri, 7 May 2021 16:35:44 +0200 Subject: [PATCH 01/13] Add features to MemoryManager The ability to allocate a buffer on a sub-region of some recoverable memory will be useful when porting over the arena-based pooling allocator from Netty. --- .../io/netty/buffer/api/MemoryManager.java | 3 +++ .../buffer/api/SizeClassedMemoryPool.java | 10 +++------- .../bytebuffer/ByteBufferMemoryManager.java | 13 +++++++++++++ .../memseg/AbstractMemorySegmentManager.java | 18 ++++++++++++++++++ .../netty/buffer/api/unsafe/UnsafeMemory.java | 4 ++++ .../buffer/api/unsafe/UnsafeMemoryManager.java | 13 +++++++++++++ 6 files changed, 54 insertions(+), 7 deletions(-) diff --git a/src/main/java/io/netty/buffer/api/MemoryManager.java b/src/main/java/io/netty/buffer/api/MemoryManager.java index 0f2088d..8c3008f 100644 --- a/src/main/java/io/netty/buffer/api/MemoryManager.java +++ b/src/main/java/io/netty/buffer/api/MemoryManager.java @@ -23,6 +23,9 @@ public interface MemoryManager { Drop drop(); Object unwrapRecoverableMemory(Buffer buf); int capacityOfRecoverableMemory(Object memory); + void discardRecoverableMemory(Object recoverableMemory); // todo should recoverMemory re-attach a cleaner? Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemory, Drop drop); + Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemoryBase, + int offset, int length, Drop drop); } diff --git a/src/main/java/io/netty/buffer/api/SizeClassedMemoryPool.java b/src/main/java/io/netty/buffer/api/SizeClassedMemoryPool.java index be6c8ea..a759639 100644 --- a/src/main/java/io/netty/buffer/api/SizeClassedMemoryPool.java +++ b/src/main/java/io/netty/buffer/api/SizeClassedMemoryPool.java @@ -82,7 +82,7 @@ class SizeClassedMemoryPool implements BufferAllocator, AllocatorControl, Drop getSizeClassPool(int size) { return pool.computeIfAbsent(size, k -> new ConcurrentLinkedQueue<>()); } - - private void dispose(Buffer buf) { - manager.drop().drop(buf); - } } diff --git a/src/main/java/io/netty/buffer/api/bytebuffer/ByteBufferMemoryManager.java b/src/main/java/io/netty/buffer/api/bytebuffer/ByteBufferMemoryManager.java index b9389ae..40bdb73 100644 --- a/src/main/java/io/netty/buffer/api/bytebuffer/ByteBufferMemoryManager.java +++ b/src/main/java/io/netty/buffer/api/bytebuffer/ByteBufferMemoryManager.java @@ -65,9 +65,22 @@ public class ByteBufferMemoryManager implements MemoryManager { return ((ByteBuffer) memory).capacity(); } + @Override + public void discardRecoverableMemory(Object recoverableMemory) { + // ByteBuffers have their memory released by the GC, so there is nothing for us to do. + } + @Override public Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemory, Drop drop) { ByteBuffer memory = (ByteBuffer) recoverableMemory; return new NioBuffer(memory, memory, allocatorControl, convert(drop)); } + + @Override + public Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemoryBase, + int offset, int length, Drop drop) { + ByteBuffer memory = (ByteBuffer) recoverableMemoryBase; + memory = memory.slice(offset, length); + return new NioBuffer(memory, memory, allocatorControl, convert(drop)); + } } diff --git a/src/main/java/io/netty/buffer/api/memseg/AbstractMemorySegmentManager.java b/src/main/java/io/netty/buffer/api/memseg/AbstractMemorySegmentManager.java index 6941143..2cc7c82 100644 --- a/src/main/java/io/netty/buffer/api/memseg/AbstractMemorySegmentManager.java +++ b/src/main/java/io/netty/buffer/api/memseg/AbstractMemorySegmentManager.java @@ -21,6 +21,7 @@ import io.netty.buffer.api.Drop; import io.netty.buffer.api.MemoryManager; import io.netty.buffer.api.internal.ArcDrop; import jdk.incubator.foreign.MemorySegment; +import jdk.incubator.foreign.ResourceScope; import java.lang.ref.Cleaner; @@ -58,9 +59,26 @@ public abstract class AbstractMemorySegmentManager implements MemoryManager { return (int) ((MemorySegment) memory).byteSize(); } + @Override + public void discardRecoverableMemory(Object recoverableMemory) { + var segment = (MemorySegment) recoverableMemory; + ResourceScope scope = segment.scope(); + if (!scope.isImplicit()) { + scope.close(); + } + } + @Override public Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemory, Drop drop) { var segment = (MemorySegment) recoverableMemory; return new MemSegBuffer(segment, segment, convert(ArcDrop.acquire(drop)), allocatorControl); } + + @Override + public Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemoryBase, + int offset, int length, Drop drop) { + var segment = (MemorySegment) recoverableMemoryBase; + segment = segment.asSlice(offset, length); + return new MemSegBuffer(segment, segment, convert(ArcDrop.acquire(drop)), allocatorControl); + } } diff --git a/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemory.java b/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemory.java index d4721d9..27de302 100644 --- a/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemory.java +++ b/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemory.java @@ -25,4 +25,8 @@ class UnsafeMemory { this.address = address; this.size = size; } + + public UnsafeMemory slice(int offset, int length) { + return new UnsafeMemory(base, address + offset, length); + } } diff --git a/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemoryManager.java b/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemoryManager.java index 97f794a..3bdce07 100644 --- a/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemoryManager.java +++ b/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemoryManager.java @@ -79,9 +79,22 @@ public class UnsafeMemoryManager implements MemoryManager { return ((UnsafeMemory) memory).size; } + @Override + public void discardRecoverableMemory(Object recoverableMemory) { + // We cannot reliably drop unsafe memory. We have to rely on the cleaner to do that. + } + @Override public Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemory, Drop drop) { UnsafeMemory memory = (UnsafeMemory) recoverableMemory; return new UnsafeBuffer(memory, 0, memory.size, allocatorControl, convert(drop)); } + + @Override + public Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemoryBase, + int offset, int length, Drop drop) { + UnsafeMemory memory = (UnsafeMemory) recoverableMemoryBase; + memory = memory.slice(offset, length); + return new UnsafeBuffer(memory, 0, memory.size, allocatorControl, convert(drop)); + } } From ae2abdd2aa6c533b1286702bf364d230442c4317 Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Tue, 11 May 2021 11:24:06 +0200 Subject: [PATCH 02/13] First incomplete draft of porting over the pooling allocator --- .../io/netty/buffer/api/MemoryManager.java | 1 + .../bytebuffer/ByteBufferMemoryManager.java | 5 + .../api/memseg/HeapMemorySegmentManager.java | 5 + .../memseg/NativeMemorySegmentManager.java | 5 + .../api/pool/ByteBufAllocatorMetric.java | 10 + .../pool/ByteBufAllocatorMetricProvider.java | 11 + .../buffer/api/pool/LongLongHashMap.java | 114 ++++ .../buffer/api/pool/LongPriorityQueue.java | 92 +++ .../io/netty/buffer/api/pool/PoolArena.java | 457 +++++++++++++ .../buffer/api/pool/PoolArenaMetric.java | 123 ++++ .../io/netty/buffer/api/pool/PoolChunk.java | 613 ++++++++++++++++++ .../netty/buffer/api/pool/PoolChunkList.java | 234 +++++++ .../buffer/api/pool/PoolChunkListMetric.java | 17 + .../buffer/api/pool/PoolChunkMetric.java | 22 + .../io/netty/buffer/api/pool/PoolSubpage.java | 272 ++++++++ .../buffer/api/pool/PoolSubpageMetric.java | 27 + .../buffer/api/pool/PoolThreadCache.java | 375 +++++++++++ .../api/pool/PooledByteBufAllocator.java | 522 +++++++++++++++ .../pool/PooledByteBufAllocatorMetric.java | 78 +++ .../io/netty/buffer/api/pool/SizeClasses.java | 392 +++++++++++ .../buffer/api/pool/SizeClassesMetric.java | 72 ++ .../api/unsafe/UnsafeMemoryManager.java | 5 + 22 files changed, 3452 insertions(+) create mode 100644 src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetric.java create mode 100644 src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetricProvider.java create mode 100644 src/main/java/io/netty/buffer/api/pool/LongLongHashMap.java create mode 100644 src/main/java/io/netty/buffer/api/pool/LongPriorityQueue.java create mode 100644 src/main/java/io/netty/buffer/api/pool/PoolArena.java create mode 100644 src/main/java/io/netty/buffer/api/pool/PoolArenaMetric.java create mode 100644 src/main/java/io/netty/buffer/api/pool/PoolChunk.java create mode 100644 src/main/java/io/netty/buffer/api/pool/PoolChunkList.java create mode 100644 src/main/java/io/netty/buffer/api/pool/PoolChunkListMetric.java create mode 100644 src/main/java/io/netty/buffer/api/pool/PoolChunkMetric.java create mode 100644 src/main/java/io/netty/buffer/api/pool/PoolSubpage.java create mode 100644 src/main/java/io/netty/buffer/api/pool/PoolSubpageMetric.java create mode 100644 src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java create mode 100644 src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocator.java create mode 100644 src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocatorMetric.java create mode 100644 src/main/java/io/netty/buffer/api/pool/SizeClasses.java create mode 100644 src/main/java/io/netty/buffer/api/pool/SizeClassesMetric.java diff --git a/src/main/java/io/netty/buffer/api/MemoryManager.java b/src/main/java/io/netty/buffer/api/MemoryManager.java index 8c3008f..81e72cf 100644 --- a/src/main/java/io/netty/buffer/api/MemoryManager.java +++ b/src/main/java/io/netty/buffer/api/MemoryManager.java @@ -18,6 +18,7 @@ package io.netty.buffer.api; import java.lang.ref.Cleaner; public interface MemoryManager { + boolean isNative(); Buffer allocateShared(AllocatorControl allocatorControl, long size, Drop drop, Cleaner cleaner); Buffer allocateConstChild(Buffer readOnlyConstParent); Drop drop(); diff --git a/src/main/java/io/netty/buffer/api/bytebuffer/ByteBufferMemoryManager.java b/src/main/java/io/netty/buffer/api/bytebuffer/ByteBufferMemoryManager.java index 40bdb73..26c3017 100644 --- a/src/main/java/io/netty/buffer/api/bytebuffer/ByteBufferMemoryManager.java +++ b/src/main/java/io/netty/buffer/api/bytebuffer/ByteBufferMemoryManager.java @@ -34,6 +34,11 @@ public class ByteBufferMemoryManager implements MemoryManager { this.direct = direct; } + @Override + public boolean isNative() { + return direct; + } + @Override public Buffer allocateShared(AllocatorControl allocatorControl, long size, Drop drop, Cleaner cleaner) { int capacity = Math.toIntExact(size); diff --git a/src/main/java/io/netty/buffer/api/memseg/HeapMemorySegmentManager.java b/src/main/java/io/netty/buffer/api/memseg/HeapMemorySegmentManager.java index 5b7a095..d52900d 100644 --- a/src/main/java/io/netty/buffer/api/memseg/HeapMemorySegmentManager.java +++ b/src/main/java/io/netty/buffer/api/memseg/HeapMemorySegmentManager.java @@ -24,4 +24,9 @@ public class HeapMemorySegmentManager extends AbstractMemorySegmentManager { protected MemorySegment createSegment(long size, Cleaner cleaner) { return MemorySegment.ofArray(new byte[Math.toIntExact(size)]); } + + @Override + public boolean isNative() { + return false; + } } diff --git a/src/main/java/io/netty/buffer/api/memseg/NativeMemorySegmentManager.java b/src/main/java/io/netty/buffer/api/memseg/NativeMemorySegmentManager.java index c71d1bf..1575ff6 100644 --- a/src/main/java/io/netty/buffer/api/memseg/NativeMemorySegmentManager.java +++ b/src/main/java/io/netty/buffer/api/memseg/NativeMemorySegmentManager.java @@ -52,6 +52,11 @@ public class NativeMemorySegmentManager extends AbstractMemorySegmentManager { } } + @Override + public boolean isNative() { + return true; + } + @Override protected MemorySegment createSegment(long size, Cleaner cleaner) { final ResourceScope scope = cleaner == null ? newSharedScope() : newSharedScope(cleaner); diff --git a/src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetric.java b/src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetric.java new file mode 100644 index 0000000..9411cdc --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetric.java @@ -0,0 +1,10 @@ +package io.netty.buffer.api.pool; + +import io.netty.buffer.api.BufferAllocator; + +public interface ByteBufAllocatorMetric { + /** + * Returns the number of bytes of heap memory used by a {@link BufferAllocator} or {@code -1} if unknown. + */ + long usedMemory(); +} diff --git a/src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetricProvider.java b/src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetricProvider.java new file mode 100644 index 0000000..a69519e --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetricProvider.java @@ -0,0 +1,11 @@ +package io.netty.buffer.api.pool; + +import io.netty.buffer.api.BufferAllocator; + +public interface ByteBufAllocatorMetricProvider { + + /** + * Returns a {@link ByteBufAllocatorMetric} for a {@link BufferAllocator}. + */ + ByteBufAllocatorMetric metric(); +} diff --git a/src/main/java/io/netty/buffer/api/pool/LongLongHashMap.java b/src/main/java/io/netty/buffer/api/pool/LongLongHashMap.java new file mode 100644 index 0000000..b272452 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/LongLongHashMap.java @@ -0,0 +1,114 @@ +package io.netty.buffer.api.pool; + +/** + * Internal primitive map implementation that is specifically optimised for the runs availability map use case in + * {@link PoolChunk}. + */ +final class LongLongHashMap { + private static final int MASK_TEMPLATE = ~1; + private int mask; + private long[] array; + private int maxProbe; + private long zeroVal; + private final long emptyVal; + + LongLongHashMap(long emptyVal) { + this.emptyVal = emptyVal; + zeroVal = emptyVal; + int initialSize = 32; + array = new long[initialSize]; + mask = initialSize - 1; + computeMaskAndProbe(); + } + + public long put(long key, long value) { + if (key == 0) { + long prev = zeroVal; + zeroVal = value; + return prev; + } + + for (;;) { + int index = index(key); + for (int i = 0; i < maxProbe; i++) { + long existing = array[index]; + if (existing == key || existing == 0) { + long prev = existing == 0? emptyVal : array[index + 1]; + array[index] = key; + array[index + 1] = value; + for (; i < maxProbe; i++) { // Nerf any existing misplaced entries. + index = index + 2 & mask; + if (array[index] == key) { + array[index] = 0; + prev = array[index + 1]; + break; + } + } + return prev; + } + index = index + 2 & mask; + } + expand(); // Grow array and re-hash. + } + } + + public void remove(long key) { + if (key == 0) { + zeroVal = emptyVal; + return; + } + int index = index(key); + for (int i = 0; i < maxProbe; i++) { + long existing = array[index]; + if (existing == key) { + array[index] = 0; + break; + } + index = index + 2 & mask; + } + } + + public long get(long key) { + if (key == 0) { + return zeroVal; + } + int index = index(key); + for (int i = 0; i < maxProbe; i++) { + long existing = array[index]; + if (existing == key) { + return array[index + 1]; + } + index = index + 2 & mask; + } + return emptyVal; + } + + private int index(long key) { + // Hash with murmur64, and mask. + key ^= key >>> 33; + key *= 0xff51afd7ed558ccdL; + key ^= key >>> 33; + key *= 0xc4ceb9fe1a85ec53L; + key ^= key >>> 33; + return (int) key & mask; + } + + private void expand() { + long[] prev = array; + array = new long[prev.length * 2]; + computeMaskAndProbe(); + for (int i = 0; i < prev.length; i += 2) { + long key = prev[i]; + if (key != 0) { + long val = prev[i + 1]; + put(key, val); + } + } + } + + private void computeMaskAndProbe() { + int length = array.length; + mask = length - 1 & MASK_TEMPLATE; + maxProbe = (int) Math.log(length); + } +} diff --git a/src/main/java/io/netty/buffer/api/pool/LongPriorityQueue.java b/src/main/java/io/netty/buffer/api/pool/LongPriorityQueue.java new file mode 100644 index 0000000..9329a60 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/LongPriorityQueue.java @@ -0,0 +1,92 @@ +package io.netty.buffer.api.pool; + +import java.util.Arrays; + +/** + * Internal primitive priority queue, used by {@link PoolChunk}. + * The implementation is based on the binary heap, as described in Algorithms by Sedgewick and Wayne. + */ +final class LongPriorityQueue { + public static final int NO_VALUE = -1; + private long[] array = new long[9]; + private int size; + + public void offer(long handle) { + if (handle == NO_VALUE) { + throw new IllegalArgumentException("The NO_VALUE (" + NO_VALUE + ") cannot be added to the queue."); + } + size++; + if (size == array.length) { + // Grow queue capacity. + array = Arrays.copyOf(array, 1 + (array.length - 1) * 2); + } + array[size] = handle; + lift(size); + } + + public void remove(long value) { + for (int i = 1; i <= size; i++) { + if (array[i] == value) { + array[i] = array[size--]; + lift(i); + sink(i); + return; + } + } + } + + public long peek() { + if (size == 0) { + return NO_VALUE; + } + return array[1]; + } + + public long poll() { + if (size == 0) { + return NO_VALUE; + } + long val = array[1]; + array[1] = array[size]; + array[size] = 0; + size--; + sink(1); + return val; + } + + public boolean isEmpty() { + return size == 0; + } + + private void lift(int index) { + int parentIndex; + while (index > 1 && subord(parentIndex = index >> 1, index)) { + swap(index, parentIndex); + index = parentIndex; + } + } + + private void sink(int index) { + int child; + while ((child = index << 1) <= size) { + if (child < size && subord(child, child + 1)) { + child++; + } + if (!subord(index, child)) { + break; + } + swap(index, child); + index = child; + } + } + + private boolean subord(int a, int b) { + return array[a] > array[b]; + } + + private void swap(int a, int b) { + long value = array[a]; + array[a] = array[b]; + array[b] = value; + } +} diff --git a/src/main/java/io/netty/buffer/api/pool/PoolArena.java b/src/main/java/io/netty/buffer/api/pool/PoolArena.java new file mode 100644 index 0000000..61afa4e --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/PoolArena.java @@ -0,0 +1,457 @@ +package io.netty.buffer.api.pool; + +import io.netty.buffer.api.Buffer; +import io.netty.buffer.api.BufferAllocator; +import io.netty.buffer.api.MemoryManager; +import io.netty.buffer.api.internal.Statics; +import io.netty.util.internal.StringUtil; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.LongAdder; + +import static io.netty.buffer.api.pool.PoolChunk.isSubpage; +import static java.lang.Math.max; + +class PoolArena extends SizeClasses implements PoolArenaMetric { + enum SizeClass { + Small, + Normal + } + + final PooledByteBufAllocator parent; + final MemoryManager manager; + + final int numSmallSubpagePools; + final int directMemoryCacheAlignment; + private final PoolSubpage[] smallSubpagePools; + + private final PoolChunkList q050; + private final PoolChunkList q025; + private final PoolChunkList q000; + private final PoolChunkList qInit; + private final PoolChunkList q075; + private final PoolChunkList q100; + + private final List chunkListMetrics; + + // Metrics for allocations and deallocations + private long allocationsNormal; + + // We need to use the LongAdder here as this is not guarded via synchronized block. + private final LongAdder allocationsSmall = new LongAdder(); + private final LongAdder allocationsHuge = new LongAdder(); + private final LongAdder activeBytesHuge = new LongAdder(); + + private long deallocationsSmall; + private long deallocationsNormal; + + // We need to use the LongAdder here as this is not guarded via synchronized block. + private final LongAdder deallocationsHuge = new LongAdder(); + + // Number of thread caches backed by this arena. + final AtomicInteger numThreadCaches = new AtomicInteger(); + + protected PoolArena(PooledByteBufAllocator parent, MemoryManager manager, int pageSize, + int pageShifts, int chunkSize, int cacheAlignment) { + super(pageSize, pageShifts, chunkSize, cacheAlignment); + this.parent = parent; + this.manager = manager; + directMemoryCacheAlignment = cacheAlignment; + + numSmallSubpagePools = nSubpages; + smallSubpagePools = newSubpagePoolArray(numSmallSubpagePools); + for (int i = 0; i < smallSubpagePools.length; i ++) { + smallSubpagePools[i] = newSubpagePoolHead(); + } + + q100 = new PoolChunkList(this, null, 100, Integer.MAX_VALUE, chunkSize); + q075 = new PoolChunkList(this, q100, 75, 100, chunkSize); + q050 = new PoolChunkList(this, q075, 50, 100, chunkSize); + q025 = new PoolChunkList(this, q050, 25, 75, chunkSize); + q000 = new PoolChunkList(this, q025, 1, 50, chunkSize); + qInit = new PoolChunkList(this, q000, Integer.MIN_VALUE, 25, chunkSize); + + q100.prevList(q075); + q075.prevList(q050); + q050.prevList(q025); + q025.prevList(q000); + q000.prevList(null); + qInit.prevList(qInit); + + chunkListMetrics = List.of(qInit, q000, q025, q050, q075, q100); + } + + private static PoolSubpage newSubpagePoolHead() { + PoolSubpage head = new PoolSubpage(); + head.prev = head; + head.next = head; + return head; + } + + private static PoolSubpage[] newSubpagePoolArray(int size) { + return new PoolSubpage[size]; + } + + boolean isDirect() { + return manager.isNative(); + } + + Buffer allocate(PoolThreadCache cache, int size) { + final int sizeIdx = size2SizeIdx(size); + + if (sizeIdx <= smallMaxSizeIdx) { + return tcacheAllocateSmall(cache, size, sizeIdx); + } else if (sizeIdx < nSizes) { + return tcacheAllocateNormal(cache, size, sizeIdx); + } else { + int normCapacity = directMemoryCacheAlignment > 0 + ? normalizeSize(size) : size; + // Huge allocations are never served via the cache so just call allocateHuge + return allocateHuge(normCapacity); + } + } + + private Buffer tcacheAllocateSmall(PoolThreadCache cache, final int size, + final int sizeIdx) { + Buffer buffer = cache.allocateSmall(size, sizeIdx); + if (buffer != null) { + // was able to allocate out of the cache so move on + return buffer; + } + + /* + * Synchronize on the head. This is needed as {@link PoolChunk#allocateSubpage(int)} and + * {@link PoolChunk#free(long)} may modify the doubly linked list as well. + */ + final PoolSubpage head = smallSubpagePools[sizeIdx]; + final boolean needsNormalAllocation; + synchronized (head) { + final PoolSubpage s = head.next; + needsNormalAllocation = s == head; + if (!needsNormalAllocation) { + assert s.doNotDestroy && s.elemSize == sizeIdx2size(sizeIdx); + long handle = s.allocate(); + assert handle >= 0; + buffer = s.chunk.allocateBufferWithSubpage(handle, size, cache); + } + } + + if (needsNormalAllocation) { + synchronized (this) { + buffer = allocateNormal(size, sizeIdx, cache); + } + } + + incSmallAllocation(); + return buffer; + } + + private Buffer tcacheAllocateNormal(PoolThreadCache cache, int size, int sizeIdx) { + Buffer buffer = cache.allocateNormal(this, size, sizeIdx); + if (buffer != null) { + // was able to allocate out of the cache so move on + return buffer; + } + synchronized (this) { + buffer = allocateNormal(size, sizeIdx, cache); + allocationsNormal++; + } + return buffer; + } + + // Method must be called inside synchronized(this) { ... } block + private Buffer allocateNormal(int size, int sizeIdx, PoolThreadCache threadCache) { + Buffer buffer = q050.allocate(size, sizeIdx, threadCache); + if (buffer != null) { + return buffer; + } + buffer = q025.allocate(size, sizeIdx, threadCache); + if (buffer != null) { + return buffer; + } + buffer = q000.allocate(size, sizeIdx, threadCache); + if (buffer != null) { + return buffer; + } + buffer = qInit.allocate(size, sizeIdx, threadCache); + if (buffer != null) { + return buffer; + } + buffer = q075.allocate(size, sizeIdx, threadCache); + if (buffer != null) { + return buffer; + } + + // Add a new chunk. + PoolChunk c = newChunk(pageSize, nPSizes, pageShifts, chunkSize); + buffer = c.allocate(size, sizeIdx, threadCache); + assert buffer != null; + qInit.add(c); + return buffer; + } + + private void incSmallAllocation() { + allocationsSmall.increment(); + } + + private Buffer allocateHuge(int size) { + activeBytesHuge.add(size); + allocationsHuge.increment(); + BufferAllocator allocator = isDirect()? BufferAllocator.direct() : BufferAllocator.heap(); + return allocator.allocate(size); + } + + void free(PoolChunk chunk, long handle, int normCapacity, PoolThreadCache cache) { + if (chunk.unpooled) { + int size = chunk.chunkSize(); + chunk.destroy(); + activeBytesHuge.add(-size); + deallocationsHuge.increment(); + } else { + SizeClass sizeClass = sizeClass(handle); + if (cache != null && cache.add(this, chunk, handle, normCapacity, sizeClass)) { + // cached so not free it. + return; + } + + freeChunk(chunk, handle, normCapacity, sizeClass); + } + } + + private static SizeClass sizeClass(long handle) { + return isSubpage(handle) ? SizeClass.Small : SizeClass.Normal; + } + + void freeChunk(PoolChunk chunk, long handle, int normCapacity, SizeClass sizeClass) { + final boolean destroyChunk; + synchronized (this) { + switch (sizeClass) { + case Normal: + ++deallocationsNormal; + break; + case Small: + ++deallocationsSmall; + break; + default: + throw new Error(); + } + destroyChunk = !chunk.parent.free(chunk, handle, normCapacity); + } + if (destroyChunk) { + // destroyChunk not need to be called while holding the synchronized lock. + chunk.destroy(); + } + } + + PoolSubpage findSubpagePoolHead(int sizeIdx) { + return smallSubpagePools[sizeIdx]; + } + + @Override + public int numThreadCaches() { + return numThreadCaches.get(); + } + + @Override + public int numSmallSubpages() { + return smallSubpagePools.length; + } + + @Override + public int numChunkLists() { + return chunkListMetrics.size(); + } + + @Override + public List smallSubpages() { + return subPageMetricList(smallSubpagePools); + } + + @Override + public List chunkLists() { + return chunkListMetrics; + } + + private static List subPageMetricList(PoolSubpage[] pages) { + List metrics = new ArrayList<>(); + for (PoolSubpage head : pages) { + if (head.next == head) { + continue; + } + PoolSubpage s = head.next; + do { + metrics.add(s); + s = s.next; + } while (s != head); + } + return metrics; + } + + @Override + public long numAllocations() { + final long allocsNormal; + synchronized (this) { + allocsNormal = allocationsNormal; + } + + return allocationsSmall.longValue() + allocsNormal + allocationsHuge.longValue(); + } + + @Override + public long numTinyAllocations() { + return 0; + } + + @Override + public long numSmallAllocations() { + return allocationsSmall.longValue(); + } + + @Override + public synchronized long numNormalAllocations() { + return allocationsNormal; + } + + @Override + public long numDeallocations() { + final long deallocs; + synchronized (this) { + deallocs = deallocationsSmall + deallocationsNormal; + } + return deallocs + deallocationsHuge.longValue(); + } + + @Override + public long numTinyDeallocations() { + return 0; + } + + @Override + public synchronized long numSmallDeallocations() { + return deallocationsSmall; + } + + @Override + public synchronized long numNormalDeallocations() { + return deallocationsNormal; + } + + @Override + public long numHugeAllocations() { + return allocationsHuge.longValue(); + } + + @Override + public long numHugeDeallocations() { + return deallocationsHuge.longValue(); + } + + @Override + public long numActiveAllocations() { + + long val = allocationsSmall.longValue() + allocationsHuge.longValue() + - deallocationsHuge.longValue(); + synchronized (this) { + val += allocationsNormal - (deallocationsSmall + deallocationsNormal); + } + return max(val, 0); + } + + @Override + public long numActiveTinyAllocations() { + return 0; + } + + @Override + public long numActiveSmallAllocations() { + return max(numSmallAllocations() - numSmallDeallocations(), 0); + } + + @Override + public long numActiveNormalAllocations() { + final long val; + synchronized (this) { + val = allocationsNormal - deallocationsNormal; + } + return max(val, 0); + } + + @Override + public long numActiveHugeAllocations() { + return max(numHugeAllocations() - numHugeDeallocations(), 0); + } + + @Override + public long numActiveBytes() { + long val = activeBytesHuge.longValue(); + synchronized (this) { + for (int i = 0; i < chunkListMetrics.size(); i++) { + for (PoolChunkMetric m: chunkListMetrics.get(i)) { + val += m.chunkSize(); + } + } + } + return max(0, val); + } + + protected final PoolChunk newChunk(int pageSize, int maxPageIdx, int pageShifts, int chunkSize) { + Buffer base = manager.allocateShared(parent, chunkSize, manager.drop(), Statics.CLEANER); + Object memory = manager.unwrapRecoverableMemory(base); + return new PoolChunk( + this, base, memory, pageSize, pageShifts, chunkSize, maxPageIdx); + } + + @Override + public synchronized String toString() { + StringBuilder buf = new StringBuilder() + .append("Chunk(s) at 0~25%:") + .append(StringUtil.NEWLINE) + .append(qInit) + .append(StringUtil.NEWLINE) + .append("Chunk(s) at 0~50%:") + .append(StringUtil.NEWLINE) + .append(q000) + .append(StringUtil.NEWLINE) + .append("Chunk(s) at 25~75%:") + .append(StringUtil.NEWLINE) + .append(q025) + .append(StringUtil.NEWLINE) + .append("Chunk(s) at 50~100%:") + .append(StringUtil.NEWLINE) + .append(q050) + .append(StringUtil.NEWLINE) + .append("Chunk(s) at 75~100%:") + .append(StringUtil.NEWLINE) + .append(q075) + .append(StringUtil.NEWLINE) + .append("Chunk(s) at 100%:") + .append(StringUtil.NEWLINE) + .append(q100) + .append(StringUtil.NEWLINE) + .append("small subpages:"); + appendPoolSubPages(buf, smallSubpagePools); + buf.append(StringUtil.NEWLINE); + + return buf.toString(); + } + + private static void appendPoolSubPages(StringBuilder buf, PoolSubpage[] subpages) { + for (int i = 0; i < subpages.length; i ++) { + PoolSubpage head = subpages[i]; + if (head.next == head) { + continue; + } + + buf.append(StringUtil.NEWLINE) + .append(i) + .append(": "); + PoolSubpage s = head.next; + do { + buf.append(s); + s = s.next; + } while (s != head); + } + } +} diff --git a/src/main/java/io/netty/buffer/api/pool/PoolArenaMetric.java b/src/main/java/io/netty/buffer/api/pool/PoolArenaMetric.java new file mode 100644 index 0000000..e119f1e --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/PoolArenaMetric.java @@ -0,0 +1,123 @@ +package io.netty.buffer.api.pool; + +import java.util.List; + +/** + * Expose metrics for an arena. + */ +public interface PoolArenaMetric extends SizeClassesMetric { + + /** + * Returns the number of thread caches backed by this arena. + */ + int numThreadCaches(); + + /** + * Returns the number of small sub-pages for the arena. + */ + int numSmallSubpages(); + + /** + * Returns the number of chunk lists for the arena. + */ + int numChunkLists(); + + /** + * Returns an unmodifiable {@link List} which holds {@link PoolSubpageMetric}s for small sub-pages. + */ + List smallSubpages(); + + /** + * Returns an unmodifiable {@link List} which holds {@link PoolChunkListMetric}s. + */ + List chunkLists(); + + /** + * Return the number of allocations done via the arena. This includes all sizes. + */ + long numAllocations(); + + /** + * Return the number of tiny allocations done via the arena. + * + * @deprecated Tiny allocations have been merged into small allocations. + */ + @Deprecated + long numTinyAllocations(); + + /** + * Return the number of small allocations done via the arena. + */ + long numSmallAllocations(); + + /** + * Return the number of normal allocations done via the arena. + */ + long numNormalAllocations(); + + /** + * Return the number of huge allocations done via the arena. + */ + long numHugeAllocations(); + + /** + * Return the number of deallocations done via the arena. This includes all sizes. + */ + long numDeallocations(); + + /** + * Return the number of tiny deallocations done via the arena. + * + * @deprecated Tiny deallocations have been merged into small deallocations. + */ + @Deprecated + long numTinyDeallocations(); + + /** + * Return the number of small deallocations done via the arena. + */ + long numSmallDeallocations(); + + /** + * Return the number of normal deallocations done via the arena. + */ + long numNormalDeallocations(); + + /** + * Return the number of huge deallocations done via the arena. + */ + long numHugeDeallocations(); + + /** + * Return the number of currently active allocations. + */ + long numActiveAllocations(); + + /** + * Return the number of currently active tiny allocations. + * + * @deprecated Tiny allocations have been merged into small allocations. + */ + @Deprecated + long numActiveTinyAllocations(); + + /** + * Return the number of currently active small allocations. + */ + long numActiveSmallAllocations(); + + /** + * Return the number of currently active normal allocations. + */ + long numActiveNormalAllocations(); + + /** + * Return the number of currently active huge allocations. + */ + long numActiveHugeAllocations(); + + /** + * Return the number of active bytes that are currently allocated by the arena. + */ + long numActiveBytes(); +} diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java new file mode 100644 index 0000000..1804a51 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java @@ -0,0 +1,613 @@ +package io.netty.buffer.api.pool; + +import io.netty.buffer.api.Buffer; +import io.netty.buffer.api.Drop; + +import java.util.PriorityQueue; + +/** + * Description of algorithm for PageRun/PoolSubpage allocation from PoolChunk + * + * Notation: The following terms are important to understand the code + * > page - a page is the smallest unit of memory chunk that can be allocated + * > run - a run is a collection of pages + * > chunk - a chunk is a collection of runs + * > in this code chunkSize = maxPages * pageSize + * + * To begin we allocate a byte array of size = chunkSize + * Whenever a ByteBuf of given size needs to be created we search for the first position + * in the byte array that has enough empty space to accommodate the requested size and + * return a (long) handle that encodes this offset information, (this memory segment is then + * marked as reserved, so it is always used by exactly one ByteBuf and no more) + * + * For simplicity all sizes are normalized according to {@link PoolArena#size2SizeIdx(int)} method. + * This ensures that when we request for memory segments of size > pageSize the normalizedCapacity + * equals the next nearest size in {@link SizeClasses}. + * + * + * A chunk has the following layout: + * + * /-----------------\ + * | run | + * | | + * | | + * |-----------------| + * | run | + * | | + * |-----------------| + * | unalloctated | + * | (freed) | + * | | + * |-----------------| + * | subpage | + * |-----------------| + * | unallocated | + * | (freed) | + * | ... | + * | ... | + * | ... | + * | | + * | | + * | | + * \-----------------/ + * + * + * handle: + * ------- + * a handle is a long number, the bit layout of a run looks like: + * + * oooooooo ooooooos ssssssss ssssssue bbbbbbbb bbbbbbbb bbbbbbbb bbbbbbbb + * + * o: runOffset (page offset in the chunk), 15bit + * s: size (number of pages) of this run, 15bit + * u: isUsed?, 1bit + * e: isSubpage?, 1bit + * b: bitmapIdx of subpage, zero if it's not subpage, 32bit + * + * runsAvailMap: + * ------ + * a map which manages all runs (used and not in used). + * For each run, the first runOffset and last runOffset are stored in runsAvailMap. + * key: runOffset + * value: handle + * + * runsAvail: + * ---------- + * an array of {@link PriorityQueue}. + * Each queue manages same size of runs. + * Runs are sorted by offset, so that we always allocate runs with smaller offset. + * + * + * Algorithm: + * ---------- + * + * As we allocate runs, we update values stored in runsAvailMap and runsAvail so that the property is maintained. + * + * Initialization - + * In the beginning we store the initial run which is the whole chunk. + * The initial run: + * runOffset = 0 + * size = chunkSize + * isUsed = no + * isSubpage = no + * bitmapIdx = 0 + * + * + * Algorithm: [allocateRun(size)] + * ---------- + * 1) find the first avail run using in runsAvails according to size + * 2) if pages of run is larger than request pages then split it, and save the tailing run + * for later using + * + * Algorithm: [allocateSubpage(size)] + * ---------- + * 1) find a not full subpage according to size. + * if it already exists just return, otherwise allocate a new PoolSubpage and call init() + * note that this subpage object is added to subpagesPool in the PoolArena when we init() it + * 2) call subpage.allocate() + * + * Algorithm: [free(handle, length, nioBuffer)] + * ---------- + * 1) if it is a subpage, return the slab back into this subpage + * 2) if the subpage is not used, or it is a run, then start free this run + * 3) merge continuous avail runs + * 4) save the merged run + * + */ +final class PoolChunk implements PoolChunkMetric { + private static final int SIZE_BIT_LENGTH = 15; + private static final int INUSED_BIT_LENGTH = 1; + private static final int SUBPAGE_BIT_LENGTH = 1; + private static final int BITMAP_IDX_BIT_LENGTH = 32; + + static final int IS_SUBPAGE_SHIFT = BITMAP_IDX_BIT_LENGTH; + static final int IS_USED_SHIFT = SUBPAGE_BIT_LENGTH + IS_SUBPAGE_SHIFT; + static final int SIZE_SHIFT = INUSED_BIT_LENGTH + IS_USED_SHIFT; + static final int RUN_OFFSET_SHIFT = SIZE_BIT_LENGTH + SIZE_SHIFT; + + final PoolArena arena; + final Buffer base; // The buffer that is the source of the memory. Closing it will free the memory. + final Object memory; + final boolean unpooled; + + /** + * store the first page and last page of each avail run + */ + private final LongLongHashMap runsAvailMap; + + /** + * manage all avail runs + */ + private final LongPriorityQueue[] runsAvail; + + /** + * manage all subpages in this chunk + */ + private final PoolSubpage[] subpages; + + private final int pageSize; + private final int pageShifts; + private final int chunkSize; + + int freeBytes; + + PoolChunkList parent; + PoolChunk prev; + PoolChunk next; + + PoolChunk(PoolArena arena, Buffer base, Object memory, int pageSize, int pageShifts, int chunkSize, int maxPageIdx) { + unpooled = false; + this.arena = arena; + this.base = base; + this.memory = memory; + this.pageSize = pageSize; + this.pageShifts = pageShifts; + this.chunkSize = chunkSize; + freeBytes = chunkSize; + + runsAvail = newRunsAvailqueueArray(maxPageIdx); + runsAvailMap = new LongLongHashMap(-1); + subpages = new PoolSubpage[chunkSize >> pageShifts]; + + //insert initial run, offset = 0, pages = chunkSize / pageSize + int pages = chunkSize >> pageShifts; + long initHandle = (long) pages << SIZE_SHIFT; + insertAvailRun(0, pages, initHandle); + } + + /** Creates a special chunk that is not pooled. */ + PoolChunk(PoolArena arena, Buffer base, Object memory, int size) { + unpooled = true; + this.arena = arena; + this.base = base; + this.memory = memory; + pageSize = 0; + pageShifts = 0; + runsAvailMap = null; + runsAvail = null; + subpages = null; + chunkSize = size; + } + + private static LongPriorityQueue[] newRunsAvailqueueArray(int size) { + LongPriorityQueue[] queueArray = new LongPriorityQueue[size]; + for (int i = 0; i < queueArray.length; i++) { + queueArray[i] = new LongPriorityQueue(); + } + return queueArray; + } + + private void insertAvailRun(int runOffset, int pages, long handle) { + int pageIdxFloor = arena.pages2pageIdxFloor(pages); + LongPriorityQueue queue = runsAvail[pageIdxFloor]; + queue.offer(handle); + + //insert first page of run + insertAvailRun0(runOffset, handle); + if (pages > 1) { + //insert last page of run + insertAvailRun0(lastPage(runOffset, pages), handle); + } + } + + private void insertAvailRun0(int runOffset, long handle) { + long pre = runsAvailMap.put(runOffset, handle); + assert pre == -1; + } + + private void removeAvailRun(long handle) { + int pageIdxFloor = arena.pages2pageIdxFloor(runPages(handle)); + LongPriorityQueue queue = runsAvail[pageIdxFloor]; + removeAvailRun(queue, handle); + } + + private void removeAvailRun(LongPriorityQueue queue, long handle) { + queue.remove(handle); + + int runOffset = runOffset(handle); + int pages = runPages(handle); + //remove first page of run + runsAvailMap.remove(runOffset); + if (pages > 1) { + //remove last page of run + runsAvailMap.remove(lastPage(runOffset, pages)); + } + } + + private static int lastPage(int runOffset, int pages) { + return runOffset + pages - 1; + } + + private long getAvailRunByOffset(int runOffset) { + return runsAvailMap.get(runOffset); + } + + @Override + public int usage() { + final int freeBytes; + synchronized (arena) { + freeBytes = this.freeBytes; + } + return usage(freeBytes); + } + + private int usage(int freeBytes) { + if (freeBytes == 0) { + return 100; + } + + int freePercentage = (int) (freeBytes * 100L / chunkSize); + if (freePercentage == 0) { + return 99; + } + return 100 - freePercentage; + } + + Buffer allocate(int size, int sizeIdx, PoolThreadCache cache) { + final long handle; + if (sizeIdx <= arena.smallMaxSizeIdx) { + // small + handle = allocateSubpage(sizeIdx); + if (handle < 0) { + return null; + } + assert isSubpage(handle); + } else { + // normal + // runSize must be multiple of pageSize + int runSize = arena.sizeIdx2size(sizeIdx); + handle = allocateRun(runSize); + if (handle < 0) { + return null; + } + } + + return allocateBuffer(handle, size, cache); + } + + private long allocateRun(int runSize) { + int pages = runSize >> pageShifts; + int pageIdx = arena.pages2pageIdx(pages); + + synchronized (runsAvail) { + //find first queue which has at least one big enough run + int queueIdx = runFirstBestFit(pageIdx); + if (queueIdx == -1) { + return -1; + } + + //get run with min offset in this queue + LongPriorityQueue queue = runsAvail[queueIdx]; + long handle = queue.poll(); + + assert handle != LongPriorityQueue.NO_VALUE && !isUsed(handle) : "invalid handle: " + handle; + + removeAvailRun(queue, handle); + + if (handle != -1) { + handle = splitLargeRun(handle, pages); + } + + freeBytes -= runSize(pageShifts, handle); + return handle; + } + } + + private int calculateRunSize(int sizeIdx) { + int maxElements = 1 << pageShifts - SizeClasses.LOG2_QUANTUM; + int runSize = 0; + int nElements; + + final int elemSize = arena.sizeIdx2size(sizeIdx); + + // Find the lowest common multiple of pageSize and elemSize + do { + runSize += pageSize; + nElements = runSize / elemSize; + } while (nElements < maxElements && runSize != nElements * elemSize); + + while (nElements > maxElements) { + runSize -= pageSize; + nElements = runSize / elemSize; + } + + assert nElements > 0; + assert runSize <= chunkSize; + assert runSize >= elemSize; + + return runSize; + } + + private int runFirstBestFit(int pageIdx) { + if (freeBytes == chunkSize) { + return arena.nPSizes - 1; + } + for (int i = pageIdx; i < arena.nPSizes; i++) { + LongPriorityQueue queue = runsAvail[i]; + if (queue != null && !queue.isEmpty()) { + return i; + } + } + return -1; + } + + private long splitLargeRun(long handle, int needPages) { + assert needPages > 0; + + int totalPages = runPages(handle); + assert needPages <= totalPages; + + int remPages = totalPages - needPages; + + if (remPages > 0) { + int runOffset = runOffset(handle); + + // keep track of trailing unused pages for later use + int availOffset = runOffset + needPages; + long availRun = toRunHandle(availOffset, remPages, 0); + insertAvailRun(availOffset, remPages, availRun); + + // not avail + return toRunHandle(runOffset, needPages, 1); + } + + //mark it as used + handle |= 1L << IS_USED_SHIFT; + return handle; + } + + /** + * Create / initialize a new PoolSubpage of normCapacity. Any PoolSubpage created / initialized here is added to + * subpage pool in the PoolArena that owns this PoolChunk + * + * @param sizeIdx sizeIdx of normalized size + * + * @return index in memoryMap + */ + private long allocateSubpage(int sizeIdx) { + // Obtain the head of the PoolSubPage pool that is owned by the PoolArena and synchronize on it. + // This is need as we may add it back and so alter the linked-list structure. + PoolSubpage head = arena.findSubpagePoolHead(sizeIdx); + synchronized (head) { + //allocate a new run + int runSize = calculateRunSize(sizeIdx); + //runSize must be multiples of pageSize + long runHandle = allocateRun(runSize); + if (runHandle < 0) { + return -1; + } + + int runOffset = runOffset(runHandle); + assert subpages[runOffset] == null; + int elemSize = arena.sizeIdx2size(sizeIdx); + + PoolSubpage subpage = new PoolSubpage(head, this, pageShifts, runOffset, + runSize(pageShifts, runHandle), elemSize); + + subpages[runOffset] = subpage; + return subpage.allocate(); + } + } + + /** + * Free a subpage, or a run of pages When a subpage is freed from PoolSubpage, it might be added back to subpage + * pool of the owning PoolArena. If the subpage pool in PoolArena has at least one other PoolSubpage of given + * elemSize, we can completely free the owning Page, so it is available for subsequent allocations. + * + * @param handle handle to free + */ + void free(long handle, int normCapacity) { + if (isSubpage(handle)) { + int sizeIdx = arena.size2SizeIdx(normCapacity); + PoolSubpage head = arena.findSubpagePoolHead(sizeIdx); + + int sIdx = runOffset(handle); + PoolSubpage subpage = subpages[sIdx]; + assert subpage != null && subpage.doNotDestroy; + + // Obtain the head of the PoolSubPage pool that is owned by the PoolArena and synchronize on it. + // This is need as we may add it back and so alter the linked-list structure. + synchronized (head) { + if (subpage.free(head, bitmapIdx(handle))) { + //the subpage is still used, do not free it + return; + } + assert !subpage.doNotDestroy; + // Null out slot in the array as it was freed, and we should not use it anymore. + subpages[sIdx] = null; + } + } + + //start free run + int pages = runPages(handle); + + synchronized (runsAvail) { + // collapse continuous runs, successfully collapsed runs + // will be removed from runsAvail and runsAvailMap + long finalRun = collapseRuns(handle); + + //set run as not used + finalRun &= ~(1L << IS_USED_SHIFT); + //if it is a subpage, set it to run + finalRun &= ~(1L << IS_SUBPAGE_SHIFT); + + insertAvailRun(runOffset(finalRun), runPages(finalRun), finalRun); + freeBytes += pages << pageShifts; + } + } + + private long collapseRuns(long handle) { + return collapseNext(collapsePast(handle)); + } + + private long collapsePast(long handle) { + for (;;) { + int runOffset = runOffset(handle); + int runPages = runPages(handle); + + long pastRun = getAvailRunByOffset(runOffset - 1); + if (pastRun == -1) { + return handle; + } + + int pastOffset = runOffset(pastRun); + int pastPages = runPages(pastRun); + + //is continuous + if (pastRun != handle && pastOffset + pastPages == runOffset) { + //remove past run + removeAvailRun(pastRun); + handle = toRunHandle(pastOffset, pastPages + runPages, 0); + } else { + return handle; + } + } + } + + private long collapseNext(long handle) { + for (;;) { + int runOffset = runOffset(handle); + int runPages = runPages(handle); + + long nextRun = getAvailRunByOffset(runOffset + runPages); + if (nextRun == -1) { + return handle; + } + + int nextOffset = runOffset(nextRun); + int nextPages = runPages(nextRun); + + //is continuous + if (nextRun != handle && runOffset + runPages == nextOffset) { + //remove next run + removeAvailRun(nextRun); + handle = toRunHandle(runOffset, runPages + nextPages, 0); + } else { + return handle; + } + } + } + + private static long toRunHandle(int runOffset, int runPages, int inUsed) { + return (long) runOffset << RUN_OFFSET_SHIFT + | (long) runPages << SIZE_SHIFT + | (long) inUsed << IS_USED_SHIFT; + } + + Buffer allocateBuffer(long handle, int size, PoolThreadCache threadCache) { + if (isRun(handle)) { + int offset = runOffset(handle) << pageShifts; + int maxLength = runSize(pageShifts, handle); + PoolThreadCache poolThreadCache = arena.parent.threadCache(); + return arena.manager.recoverMemory(arena.parent, memory, offset, size, new Drop() { + @Override + public void drop(Buffer obj) { + arena.free(PoolChunk.this, handle, maxLength, poolThreadCache); + } + }); + } else { + return allocateBufferWithSubpage(handle, size, threadCache); + } + } + + Buffer allocateBufferWithSubpage(long handle, int size, PoolThreadCache threadCache) { + int runOffset = runOffset(handle); + int bitmapIdx = bitmapIdx(handle); + + PoolSubpage s = subpages[runOffset]; + assert s.doNotDestroy; + assert size <= s.elemSize; + + int offset = (runOffset << pageShifts) + bitmapIdx * s.elemSize; + return arena.manager.recoverMemory(arena.parent, memory, offset, size, new Drop() { + @Override + public void drop(Buffer obj) { + arena.free(PoolChunk.this, handle, s.elemSize, threadCache); + } + }); + } + + @Override + public int chunkSize() { + return chunkSize; + } + + @Override + public int freeBytes() { + synchronized (arena) { + return freeBytes; + } + } + + @Override + public String toString() { + final int freeBytes; + synchronized (arena) { + freeBytes = this.freeBytes; + } + + return new StringBuilder() + .append("Chunk(") + .append(Integer.toHexString(System.identityHashCode(this))) + .append(": ") + .append(usage(freeBytes)) + .append("%, ") + .append(chunkSize - freeBytes) + .append('/') + .append(chunkSize) + .append(')') + .toString(); + } + + void destroy() { + base.close(); + } + + static int runOffset(long handle) { + return (int) (handle >> RUN_OFFSET_SHIFT); + } + + static int runSize(int pageShifts, long handle) { + return runPages(handle) << pageShifts; + } + + static int runPages(long handle) { + return (int) (handle >> SIZE_SHIFT & 0x7fff); + } + + static boolean isUsed(long handle) { + return (handle >> IS_USED_SHIFT & 1) == 1L; + } + + static boolean isRun(long handle) { + return !isSubpage(handle); + } + + static boolean isSubpage(long handle) { + return (handle >> IS_SUBPAGE_SHIFT & 1) == 1L; + } + + static int bitmapIdx(long handle) { + return (int) handle; + } +} diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java b/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java new file mode 100644 index 0000000..4c18f95 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java @@ -0,0 +1,234 @@ +package io.netty.buffer.api.pool; + +import io.netty.buffer.api.Buffer; +import io.netty.util.internal.StringUtil; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import static java.lang.Math.*; + +final class PoolChunkList implements PoolChunkListMetric { + private static final Iterator EMPTY_METRICS = Collections.emptyIterator(); + private final PoolArena arena; + private final PoolChunkList nextList; + private final int minUsage; + private final int maxUsage; + private final int maxCapacity; + private PoolChunk head; + private final int freeMinThreshold; + private final int freeMaxThreshold; + + // This is only update once when create the linked like list of PoolChunkList in PoolArena constructor. + private PoolChunkList prevList; + + PoolChunkList(PoolArena arena, PoolChunkList nextList, int minUsage, int maxUsage, int chunkSize) { + assert minUsage <= maxUsage; + this.arena = arena; + this.nextList = nextList; + this.minUsage = minUsage; + this.maxUsage = maxUsage; + maxCapacity = calculateMaxCapacity(minUsage, chunkSize); + + // the thresholds are aligned with PoolChunk.usage() logic: + // 1) basic logic: usage() = 100 - freeBytes * 100L / chunkSize + // so, for example: (usage() >= maxUsage) condition can be transformed in the following way: + // 100 - freeBytes * 100L / chunkSize >= maxUsage + // freeBytes <= chunkSize * (100 - maxUsage) / 100 + // let freeMinThreshold = chunkSize * (100 - maxUsage) / 100, then freeBytes <= freeMinThreshold + // + // 2) usage() returns an int value and has a floor rounding during a calculation, + // to be aligned absolute thresholds should be shifted for "the rounding step": + // freeBytes * 100 / chunkSize < 1 + // the condition can be converted to: freeBytes < 1 * chunkSize / 100 + // this is why we have + 0.99999999 shifts. A example why just +1 shift cannot be used: + // freeBytes = 16777216 == freeMaxThreshold: 16777216, usage = 0 < minUsage: 1, chunkSize: 16777216 + // At the same time we want to have zero thresholds in case of (maxUsage == 100) and (minUsage == 100). + // + freeMinThreshold = maxUsage == 100 ? 0 : (int) (chunkSize * (100.0 - maxUsage + 0.99999999) / 100L); + freeMaxThreshold = minUsage == 100 ? 0 : (int) (chunkSize * (100.0 - minUsage + 0.99999999) / 100L); + } + + /** + * Calculates the maximum capacity of a buffer that will ever be possible to allocate out of the {@link PoolChunk}s + * that belong to the {@link PoolChunkList} with the given {@code minUsage} and {@code maxUsage} settings. + */ + private static int calculateMaxCapacity(int minUsage, int chunkSize) { + minUsage = minUsage0(minUsage); + + if (minUsage == 100) { + // If the minUsage is 100 we can not allocate anything out of this list. + return 0; + } + + // Calculate the maximum amount of bytes that can be allocated from a PoolChunk in this PoolChunkList. + // + // As an example: + // - If a PoolChunkList has minUsage == 25 we are allowed to allocate at most 75% of the chunkSize because + // this is the maximum amount available in any PoolChunk in this PoolChunkList. + return (int) (chunkSize * (100L - minUsage) / 100L); + } + + void prevList(PoolChunkList prevList) { + assert this.prevList == null; + this.prevList = prevList; + } + + Buffer allocate(int size, int sizeIdx, PoolThreadCache threadCache) { + int normCapacity = arena.sizeIdx2size(sizeIdx); + if (normCapacity > maxCapacity) { + // Either this PoolChunkList is empty, or the requested capacity is larger than the capacity which can + // be handled by the PoolChunks that are contained in this PoolChunkList. + return null; + } + + for (PoolChunk cur = head; cur != null; cur = cur.next) { + Buffer buffer = cur.allocate(size, sizeIdx, threadCache); + if (buffer != null) { + if (cur.freeBytes <= freeMinThreshold) { + remove(cur); + nextList.add(cur); + } + return buffer; + } + } + return null; + } + + boolean free(PoolChunk chunk, long handle, int normCapacity) { + chunk.free(handle, normCapacity); + if (chunk.freeBytes > freeMaxThreshold) { + remove(chunk); + // Move the PoolChunk down the PoolChunkList linked-list. + return move0(chunk); + } + return true; + } + + private boolean move(PoolChunk chunk) { + if (chunk.freeBytes > freeMaxThreshold) { + // Move the PoolChunk down the PoolChunkList linked-list. + return move0(chunk); + } + + // PoolChunk fits into this PoolChunkList, adding it here. + add0(chunk); + return true; + } + + /** + * Moves the {@link PoolChunk} down the {@link PoolChunkList} linked-list, so it will end up in the right + * {@link PoolChunkList} that has the correct minUsage / maxUsage in respect to {@link PoolChunk#usage()}. + */ + private boolean move0(PoolChunk chunk) { + if (prevList == null) { + // There is no previous PoolChunkList so return false which result in having the PoolChunk destroyed and + // all memory associated with the PoolChunk will be released. + return false; + } + return prevList.move(chunk); + } + + void add(PoolChunk chunk) { + if (chunk.freeBytes <= freeMinThreshold) { + nextList.add(chunk); + return; + } + add0(chunk); + } + + /** + * Adds the {@link PoolChunk} to this {@link PoolChunkList}. + */ + void add0(PoolChunk chunk) { + chunk.parent = this; + if (head == null) { + head = chunk; + chunk.prev = null; + chunk.next = null; + } else { + chunk.prev = null; + chunk.next = head; + head.prev = chunk; + head = chunk; + } + } + + private void remove(PoolChunk cur) { + if (cur == head) { + head = cur.next; + if (head != null) { + head.prev = null; + } + } else { + PoolChunk next = cur.next; + cur.prev.next = next; + if (next != null) { + next.prev = cur.prev; + } + } + } + + @Override + public int minUsage() { + return minUsage0(minUsage); + } + + @Override + public int maxUsage() { + return min(maxUsage, 100); + } + + private static int minUsage0(int value) { + return max(1, value); + } + + @Override + public Iterator iterator() { + synchronized (arena) { + if (head == null) { + return EMPTY_METRICS; + } + List metrics = new ArrayList<>(); + for (PoolChunk cur = head;;) { + metrics.add(cur); + cur = cur.next; + if (cur == null) { + break; + } + } + return metrics.iterator(); + } + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(); + synchronized (arena) { + if (head == null) { + return "none"; + } + + for (PoolChunk cur = head;;) { + buf.append(cur); + cur = cur.next; + if (cur == null) { + break; + } + buf.append(StringUtil.NEWLINE); + } + } + return buf.toString(); + } + + void destroy(PoolArena arena) { + PoolChunk chunk = head; + while (chunk != null) { + arena.destroyChunk(chunk); + chunk = chunk.next; + } + head = null; + } +} diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunkListMetric.java b/src/main/java/io/netty/buffer/api/pool/PoolChunkListMetric.java new file mode 100644 index 0000000..8c3d89f --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunkListMetric.java @@ -0,0 +1,17 @@ +package io.netty.buffer.api.pool; + +/** + * Metrics for a list of chunks. + */ +public interface PoolChunkListMetric extends Iterable { + + /** + * Return the minimum usage of the chunk list before which chunks are promoted to the previous list. + */ + int minUsage(); + + /** + * Return the maximum usage of the chunk list after which chunks are promoted to the next list. + */ + int maxUsage(); +} diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunkMetric.java b/src/main/java/io/netty/buffer/api/pool/PoolChunkMetric.java new file mode 100644 index 0000000..69868e7 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunkMetric.java @@ -0,0 +1,22 @@ +package io.netty.buffer.api.pool; + +/** + * Metrics for a chunk. + */ +public interface PoolChunkMetric { + + /** + * Return the percentage of the current usage of the chunk. + */ + int usage(); + + /** + * Return the size of the chunk in bytes, this is the maximum of bytes that can be served out of the chunk. + */ + int chunkSize(); + + /** + * Return the number of free bytes in the chunk. + */ + int freeBytes(); +} diff --git a/src/main/java/io/netty/buffer/api/pool/PoolSubpage.java b/src/main/java/io/netty/buffer/api/pool/PoolSubpage.java new file mode 100644 index 0000000..a91c854 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/PoolSubpage.java @@ -0,0 +1,272 @@ +package io.netty.buffer.api.pool; + +import static io.netty.buffer.api.pool.PoolChunk.RUN_OFFSET_SHIFT; +import static io.netty.buffer.api.pool.PoolChunk.SIZE_SHIFT; +import static io.netty.buffer.api.pool.PoolChunk.IS_USED_SHIFT; +import static io.netty.buffer.api.pool.PoolChunk.IS_SUBPAGE_SHIFT; +import static io.netty.buffer.api.pool.SizeClasses.LOG2_QUANTUM; + +final class PoolSubpage implements PoolSubpageMetric { + final PoolChunk chunk; + private final int pageShifts; + private final int runOffset; + private final int runSize; + private final long[] bitmap; + + PoolSubpage prev; + PoolSubpage next; + + boolean doNotDestroy; + int elemSize; + private int maxNumElems; + private int bitmapLength; + private int nextAvail; + private int numAvail; + + /** Special constructor that creates a linked list head */ + PoolSubpage() { + chunk = null; + pageShifts = -1; + runOffset = -1; + elemSize = -1; + runSize = -1; + bitmap = null; + } + + PoolSubpage(PoolSubpage head, PoolChunk chunk, int pageShifts, int runOffset, int runSize, int elemSize) { + this.chunk = chunk; + this.pageShifts = pageShifts; + this.runOffset = runOffset; + this.runSize = runSize; + this.elemSize = elemSize; + bitmap = new long[runSize >>> 6 + LOG2_QUANTUM]; // runSize / 64 / QUANTUM + + doNotDestroy = true; + if (elemSize != 0) { + maxNumElems = numAvail = runSize / elemSize; + nextAvail = 0; + bitmapLength = maxNumElems >>> 6; + if ((maxNumElems & 63) != 0) { + bitmapLength ++; + } + + for (int i = 0; i < bitmapLength; i ++) { + bitmap[i] = 0; + } + } + addToPool(head); + } + + /** + * Returns the bitmap index of the subpage allocation. + */ + long allocate() { + if (numAvail == 0 || !doNotDestroy) { + return -1; + } + + final int bitmapIdx = getNextAvail(); + int q = bitmapIdx >>> 6; + int r = bitmapIdx & 63; + assert (bitmap[q] >>> r & 1) == 0; + bitmap[q] |= 1L << r; + + if (-- numAvail == 0) { + removeFromPool(); + } + + return toHandle(bitmapIdx); + } + + /** + * @return {@code true} if this subpage is in use. + * {@code false} if this subpage is not used by its chunk and thus it's OK to be released. + */ + boolean free(PoolSubpage head, int bitmapIdx) { + if (elemSize == 0) { + return true; + } + int q = bitmapIdx >>> 6; + int r = bitmapIdx & 63; + assert (bitmap[q] >>> r & 1) != 0; + bitmap[q] ^= 1L << r; + + setNextAvail(bitmapIdx); + + if (numAvail ++ == 0) { + addToPool(head); + // When maxNumElems == 1, the maximum numAvail is also 1. + // Each of these PoolSubpages will go in here when they do free operation. + // If they return true directly from here, then the rest of the code will be unreachable, + // and they will not actually be recycled. So return true only on maxNumElems > 1. + if (maxNumElems > 1) { + return true; + } + } + + if (numAvail != maxNumElems) { + return true; + } else { + // Subpage not in use (numAvail == maxNumElems) + if (prev == next) { + // Do not remove if this subpage is the only one left in the pool. + return true; + } + + // Remove this subpage from the pool if there are other subpages left in the pool. + doNotDestroy = false; + removeFromPool(); + return false; + } + } + + private void addToPool(PoolSubpage head) { + assert prev == null && next == null; + prev = head; + next = head.next; + next.prev = this; + head.next = this; + } + + private void removeFromPool() { + assert prev != null && next != null; + prev.next = next; + next.prev = prev; + next = null; + prev = null; + } + + private void setNextAvail(int bitmapIdx) { + nextAvail = bitmapIdx; + } + + private int getNextAvail() { + int nextAvail = this.nextAvail; + if (nextAvail >= 0) { + this.nextAvail = -1; + return nextAvail; + } + return findNextAvail(); + } + + private int findNextAvail() { + final long[] bitmap = this.bitmap; + final int bitmapLength = this.bitmapLength; + for (int i = 0; i < bitmapLength; i ++) { + long bits = bitmap[i]; + if (~bits != 0) { + return findNextAvail0(i, bits); + } + } + return -1; + } + + private int findNextAvail0(int i, long bits) { + final int maxNumElems = this.maxNumElems; + final int baseVal = i << 6; + + for (int j = 0; j < 64; j ++) { + if ((bits & 1) == 0) { + int val = baseVal | j; + if (val < maxNumElems) { + return val; + } else { + break; + } + } + bits >>>= 1; + } + return -1; + } + + private long toHandle(int bitmapIdx) { + int pages = runSize >> pageShifts; + return (long) runOffset << RUN_OFFSET_SHIFT + | (long) pages << SIZE_SHIFT + | 1L << IS_USED_SHIFT + | 1L << IS_SUBPAGE_SHIFT + | bitmapIdx; + } + + @Override + public String toString() { + final boolean doNotDestroy; + final int maxNumElems; + final int numAvail; + final int elemSize; + if (chunk == null) { + // This is the head so there is no need to synchronize at all as these never change. + doNotDestroy = true; + maxNumElems = 0; + numAvail = 0; + elemSize = -1; + } else { + synchronized (chunk.arena) { + if (!this.doNotDestroy) { + doNotDestroy = false; + // Not used for creating the String. + maxNumElems = numAvail = elemSize = -1; + } else { + doNotDestroy = true; + maxNumElems = this.maxNumElems; + numAvail = this.numAvail; + elemSize = this.elemSize; + } + } + } + + if (!doNotDestroy) { + return "(" + runOffset + ": not in use)"; + } + + return "(" + runOffset + ": " + (maxNumElems - numAvail) + '/' + maxNumElems + + ", offset: " + runOffset + ", length: " + runSize + ", elemSize: " + elemSize + ')'; + } + + @Override + public int maxNumElements() { + if (chunk == null) { + // It's the head. + return 0; + } + + synchronized (chunk.arena) { + return maxNumElems; + } + } + + @Override + public int numAvailable() { + if (chunk == null) { + // It's the head. + return 0; + } + + synchronized (chunk.arena) { + return numAvail; + } + } + + @Override + public int elementSize() { + if (chunk == null) { + // It's the head. + return -1; + } + + synchronized (chunk.arena) { + return elemSize; + } + } + + @Override + public int pageSize() { + return 1 << pageShifts; + } + + void destroy() { + if (chunk != null) { + chunk.destroy(); + } + } +} diff --git a/src/main/java/io/netty/buffer/api/pool/PoolSubpageMetric.java b/src/main/java/io/netty/buffer/api/pool/PoolSubpageMetric.java new file mode 100644 index 0000000..75b2a53 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/PoolSubpageMetric.java @@ -0,0 +1,27 @@ +package io.netty.buffer.api.pool; + +/** + * Metrics for a sub-page. + */ +public interface PoolSubpageMetric { + + /** + * Return the number of maximal elements that can be allocated out of the sub-page. + */ + int maxNumElements(); + + /** + * Return the number of available elements to be allocated. + */ + int numAvailable(); + + /** + * Return the size (in bytes) of the elements that will be allocated. + */ + int elementSize(); + + /** + * Return the page size (in bytes) of this page. + */ + int pageSize(); +} diff --git a/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java b/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java new file mode 100644 index 0000000..5e986b1 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java @@ -0,0 +1,375 @@ +package io.netty.buffer.api.pool; + +import static io.netty.buffer.api.pool.PoolArena.SizeClass.*; +import static io.netty.util.internal.ObjectUtil.checkPositiveOrZero; + +import io.netty.buffer.api.Buffer; +import io.netty.buffer.api.pool.PoolArena.SizeClass; +import io.netty.util.internal.MathUtil; +import io.netty.util.internal.ObjectPool; +import io.netty.util.internal.ObjectPool.Handle; +import io.netty.util.internal.PlatformDependent; +import io.netty.util.internal.logging.InternalLogger; +import io.netty.util.internal.logging.InternalLoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; + +/** + * Acts a Thread cache for allocations. This implementation is modelled after + * jemalloc and the described + * techniques of + * + * Scalable memory allocation using jemalloc. + */ +final class PoolThreadCache { + + private static final InternalLogger logger = InternalLoggerFactory.getInstance(PoolThreadCache.class); + private static final int INTEGER_SIZE_MINUS_ONE = Integer.SIZE - 1; + + final PoolArena arena; + + // Hold the caches for the different size classes, which are tiny, small and normal. + private final MemoryRegionCache[] smallSubPageCaches; + private final MemoryRegionCache[] normalCaches; + + private final int freeSweepAllocationThreshold; + + private int allocations; + + PoolThreadCache(PoolArena arena, + int smallCacheSize, int normalCacheSize, int maxCachedBufferCapacity, + int freeSweepAllocationThreshold) { + checkPositiveOrZero(maxCachedBufferCapacity, "maxCachedBufferCapacity"); + this.freeSweepAllocationThreshold = freeSweepAllocationThreshold; + this.arena = arena; + if (arena != null) { + // Create the caches for the heap allocations + smallSubPageCaches = createSubPageCaches( + smallCacheSize, arena.numSmallSubpagePools); + + normalCaches = createNormalCaches( + normalCacheSize, maxCachedBufferCapacity, arena); + + arena.numThreadCaches.getAndIncrement(); + } else { + // No heapArea is configured so just null out all caches + smallSubPageCaches = null; + normalCaches = null; + } + + // Only check if there are caches in use. + if ((smallSubPageCaches != null || normalCaches != null) + && freeSweepAllocationThreshold < 1) { + throw new IllegalArgumentException("freeSweepAllocationThreshold: " + + freeSweepAllocationThreshold + " (expected: > 0)"); + } + } + + private static MemoryRegionCache[] createSubPageCaches( + int cacheSize, int numCaches) { + if (cacheSize > 0 && numCaches > 0) { + MemoryRegionCache[] cache = new MemoryRegionCache[numCaches]; + for (int i = 0; i < cache.length; i++) { + // TODO: maybe use cacheSize / cache.length + cache[i] = new SubPageMemoryRegionCache(cacheSize); + } + return cache; + } else { + return null; + } + } + + private static MemoryRegionCache[] createNormalCaches( + int cacheSize, int maxCachedBufferCapacity, PoolArena area) { + if (cacheSize > 0 && maxCachedBufferCapacity > 0) { + int max = Math.min(area.chunkSize, maxCachedBufferCapacity); + + // Create as many normal caches as we support based on how many sizeIdx we have and what the upper + // bound is that we want to cache in general. + List cache = new ArrayList<>() ; + for (int idx = area.numSmallSubpagePools; idx < area.nSizes && area.sizeIdx2size(idx) <= max ; idx++) { + cache.add(new NormalMemoryRegionCache(cacheSize)); + } + return cache.toArray(MemoryRegionCache[]::new); + } else { + return null; + } + } + + // val > 0 + static int log2(int val) { + return INTEGER_SIZE_MINUS_ONE - Integer.numberOfLeadingZeros(val); + } + + /** + * Try to allocate a small buffer out of the cache. Returns {@code true} if successful {@code false} otherwise + */ + Buffer allocateSmall(int size, int sizeIdx) { + return allocate(cacheForSmall(sizeIdx), size); + } + + /** + * Try to allocate a normal buffer out of the cache. Returns {@code true} if successful {@code false} otherwise + */ + Buffer allocateNormal(PoolArena area, int size, int sizeIdx) { + return allocate(cacheForNormal(area, sizeIdx), size); + } + + private Buffer allocate(MemoryRegionCache cache, int size) { + if (cache == null) { + // no cache found so just return false here + return null; + } + Buffer allocated = cache.allocate(size, this); + if (++ allocations >= freeSweepAllocationThreshold) { + allocations = 0; + trim(); + } + return allocated; + } + + /** + * Add {@link PoolChunk} and {@code handle} to the cache if there is enough room. + * Returns {@code true} if it fit into the cache {@code false} otherwise. + */ + boolean add(PoolArena area, PoolChunk chunk, + long handle, int normCapacity, SizeClass sizeClass) { + int sizeIdx = area.size2SizeIdx(normCapacity); + MemoryRegionCache cache = cache(area, sizeIdx, sizeClass); + if (cache == null) { + return false; + } + return cache.add(chunk, handle, normCapacity); + } + + private MemoryRegionCache cache(PoolArena area, int sizeIdx, SizeClass sizeClass) { + switch (sizeClass) { + case Normal: + return cacheForNormal(area, sizeIdx); + case Small: + return cacheForSmall(sizeIdx); + default: + throw new Error(); + } + } + + /** + * Should be called if the Thread that uses this cache is about to exist to release resources out of the cache + */ + void free() { + int numFreed = free(smallSubPageCaches) + free(normalCaches); + + if (numFreed > 0 && logger.isDebugEnabled()) { + logger.debug("Freed {} thread-local buffer(s) from thread: {}", numFreed, + Thread.currentThread().getName()); + } + + if (arena != null) { + arena.numThreadCaches.getAndDecrement(); + } + } + + private static int free(MemoryRegionCache[] caches) { + if (caches == null) { + return 0; + } + + int numFreed = 0; + for (MemoryRegionCache c: caches) { + numFreed += free(c); + } + return numFreed; + } + + private static int free(MemoryRegionCache cache) { + if (cache == null) { + return 0; + } + return cache.free(); + } + + void trim() { + trim(smallSubPageCaches); + trim(normalCaches); + } + + private static void trim(MemoryRegionCache[] caches) { + if (caches == null) { + return; + } + for (MemoryRegionCache c: caches) { + trim(c); + } + } + + private static void trim(MemoryRegionCache cache) { + if (cache == null) { + return; + } + cache.trim(); + } + + private MemoryRegionCache cacheForSmall(int sizeIdx) { + return cache(smallSubPageCaches, sizeIdx); + } + + private MemoryRegionCache cacheForNormal(PoolArena area, int sizeIdx) { + // We need to substract area.numSmallSubpagePools as sizeIdx is the overall index for all sizes. + int idx = sizeIdx - area.numSmallSubpagePools; + return cache(normalCaches, idx); + } + + private static MemoryRegionCache cache(MemoryRegionCache[] cache, int sizeIdx) { + if (cache == null || sizeIdx > cache.length - 1) { + return null; + } + return cache[sizeIdx]; + } + + /** + * Cache used for buffers which are backed by SMALL size. + */ + private static final class SubPageMemoryRegionCache extends MemoryRegionCache { + SubPageMemoryRegionCache(int size) { + super(size, Small); + } + + @Override + protected Buffer allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache) { + return chunk.allocateBufferWithSubpage(handle, size, threadCache); + } + } + + /** + * Cache used for buffers which are backed by NORMAL size. + */ + private static final class NormalMemoryRegionCache extends MemoryRegionCache { + NormalMemoryRegionCache(int size) { + super(size, Normal); + } + + @Override + protected Buffer allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache) { + return chunk.allocateBuffer(handle, size, threadCache); + } + } + + private abstract static class MemoryRegionCache { + private final int size; + private final Queue queue; + private final SizeClass sizeClass; + private int allocations; + + MemoryRegionCache(int size, SizeClass sizeClass) { + this.size = MathUtil.safeFindNextPositivePowerOfTwo(size); + queue = PlatformDependent.newFixedMpscQueue(this.size); + this.sizeClass = sizeClass; + } + + /** + * Allocate a new {@link Buffer} using the provided chunk and handle with the capacity restrictions. + */ + protected abstract Buffer allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache); + + /** + * Add to cache if not already full. + */ + public final boolean add(PoolChunk chunk, long handle, int normCapacity) { + Entry entry = newEntry(chunk, handle, normCapacity); + boolean queued = queue.offer(entry); + if (!queued) { + // If it was not possible to cache the chunk, immediately recycle the entry + entry.recycle(); + } + + return queued; + } + + /** + * Allocate something out of the cache if possible and remove the entry from the cache. + */ + public final Buffer allocate(int size, PoolThreadCache threadCache) { + Entry entry = queue.poll(); + if (entry == null) { + return null; + } + Buffer buffer = allocBuf(entry.chunk, entry.handle, size, threadCache); + entry.recycle(); + + // allocations are not thread-safe which is fine as this is only called from the same thread all time. + allocations++; + return buffer; + } + + /** + * Clear out this cache and free up all previous cached {@link PoolChunk}s and {@code handle}s. + */ + public final int free() { + return free(Integer.MAX_VALUE); + } + + private int free(int max) { + int numFreed = 0; + for (; numFreed < max; numFreed++) { + Entry entry = queue.poll(); + if (entry != null) { + freeEntry(entry); + } else { + // all cleared + return numFreed; + } + } + return numFreed; + } + + /** + * Free up cached {@link PoolChunk}s if not allocated frequently enough. + */ + public final void trim() { + int free = size - allocations; + allocations = 0; + + // We not even allocated all the number that are + if (free > 0) { + free(free); + } + } + + private void freeEntry(Entry entry) { + PoolChunk chunk = entry.chunk; + long handle = entry.handle; + + entry.recycle(); + chunk.arena.freeChunk(chunk, handle, entry.normCapacity, sizeClass); + } + + static final class Entry { + final Handle recyclerHandle; + PoolChunk chunk; + long handle = -1; + int normCapacity; + + Entry(Handle recyclerHandle) { + this.recyclerHandle = recyclerHandle; + } + + void recycle() { + chunk = null; + handle = -1; + recyclerHandle.recycle(this); + } + } + + private static Entry newEntry(PoolChunk chunk, long handle, int normCapacity) { + Entry entry = RECYCLER.get(); + entry.chunk = chunk; + entry.handle = handle; + entry.normCapacity = normCapacity; + return entry; + } + + private static final ObjectPool RECYCLER = ObjectPool.newPool(handle -> new Entry(handle)); + } +} diff --git a/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocator.java b/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocator.java new file mode 100644 index 0000000..b9120d3 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocator.java @@ -0,0 +1,522 @@ +package io.netty.buffer.api.pool; + +import static io.netty.util.internal.ObjectUtil.checkPositiveOrZero; +import static java.util.Objects.requireNonNull; + +import io.netty.buffer.api.AllocatorControl; +import io.netty.buffer.api.Buffer; +import io.netty.buffer.api.BufferAllocator; +import io.netty.buffer.api.MemoryManager; +import io.netty.util.NettyRuntime; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.FastThreadLocal; +import io.netty.util.concurrent.FastThreadLocalThread; +import io.netty.util.internal.PlatformDependent; +import io.netty.util.internal.StringUtil; +import io.netty.util.internal.SystemPropertyUtil; +import io.netty.util.internal.ThreadExecutorMap; +import io.netty.util.internal.logging.InternalLogger; +import io.netty.util.internal.logging.InternalLoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class PooledByteBufAllocator implements BufferAllocator, ByteBufAllocatorMetricProvider, AllocatorControl { + + private static final InternalLogger logger = InternalLoggerFactory.getInstance(PooledByteBufAllocator.class); + private static final int DEFAULT_NUM_HEAP_ARENA; + private static final int DEFAULT_NUM_DIRECT_ARENA; + + private static final int DEFAULT_PAGE_SIZE; + private static final int DEFAULT_MAX_ORDER; // 8192 << 11 = 16 MiB per chunk + private static final int DEFAULT_SMALL_CACHE_SIZE; + private static final int DEFAULT_NORMAL_CACHE_SIZE; + static final int DEFAULT_MAX_CACHED_BUFFER_CAPACITY; + private static final int DEFAULT_CACHE_TRIM_INTERVAL; + private static final long DEFAULT_CACHE_TRIM_INTERVAL_MILLIS; + private static final boolean DEFAULT_USE_CACHE_FOR_ALL_THREADS; + private static final int DEFAULT_DIRECT_MEMORY_CACHE_ALIGNMENT; + static final int DEFAULT_MAX_CACHED_BYTEBUFFERS_PER_CHUNK; + + private static final int MIN_PAGE_SIZE = 4096; + private static final int MAX_CHUNK_SIZE = (int) (((long) Integer.MAX_VALUE + 1) / 2); + + private final Runnable trimTask = this::trimCurrentThreadCache; + + static { + int defaultAlignment = SystemPropertyUtil.getInt( + "io.netty.allocator.directMemoryCacheAlignment", 0); + int defaultPageSize = SystemPropertyUtil.getInt("io.netty.allocator.pageSize", 8192); + Throwable pageSizeFallbackCause = null; + try { + validateAndCalculatePageShifts(defaultPageSize, defaultAlignment); + } catch (Throwable t) { + pageSizeFallbackCause = t; + defaultPageSize = 8192; + defaultAlignment = 0; + } + DEFAULT_PAGE_SIZE = defaultPageSize; + DEFAULT_DIRECT_MEMORY_CACHE_ALIGNMENT = defaultAlignment; + + int defaultMaxOrder = SystemPropertyUtil.getInt("io.netty.allocator.maxOrder", 11); + Throwable maxOrderFallbackCause = null; + try { + validateAndCalculateChunkSize(DEFAULT_PAGE_SIZE, defaultMaxOrder); + } catch (Throwable t) { + maxOrderFallbackCause = t; + defaultMaxOrder = 11; + } + DEFAULT_MAX_ORDER = defaultMaxOrder; + + // Determine reasonable default for nHeapArena and nDirectArena. + // Assuming each arena has 3 chunks, the pool should not consume more than 50% of max memory. + final Runtime runtime = Runtime.getRuntime(); + + /* + * We use 2 * available processors by default to reduce contention as we use 2 * available processors for the + * number of EventLoops in NIO and EPOLL as well. If we choose a smaller number we will run into hot spots as + * allocation and de-allocation needs to be synchronized on the PoolArena. + * + * See https://github.com/netty/netty/issues/3888. + */ + final int defaultMinNumArena = NettyRuntime.availableProcessors() * 2; + final int defaultChunkSize = DEFAULT_PAGE_SIZE << DEFAULT_MAX_ORDER; + DEFAULT_NUM_HEAP_ARENA = Math.max(0, + SystemPropertyUtil.getInt( + "io.netty.allocator.numArenas", + (int) Math.min( + defaultMinNumArena, + runtime.maxMemory() / defaultChunkSize / 2 / 3))); + DEFAULT_NUM_DIRECT_ARENA = Math.max(0, + SystemPropertyUtil.getInt( + "io.netty.allocator.numDirectArenas", + (int) Math.min( + defaultMinNumArena, + PlatformDependent.maxDirectMemory() / defaultChunkSize / 2 / 3))); + + // cache sizes + DEFAULT_SMALL_CACHE_SIZE = SystemPropertyUtil.getInt("io.netty.allocator.smallCacheSize", 256); + DEFAULT_NORMAL_CACHE_SIZE = SystemPropertyUtil.getInt("io.netty.allocator.normalCacheSize", 64); + + // 32 kb is the default maximum capacity of the cached buffer. Similar to what is explained in + // 'Scalable memory allocation using jemalloc' + DEFAULT_MAX_CACHED_BUFFER_CAPACITY = SystemPropertyUtil.getInt( + "io.netty.allocator.maxCachedBufferCapacity", 32 * 1024); + + // the number of threshold of allocations when cached entries will be freed up if not frequently used + DEFAULT_CACHE_TRIM_INTERVAL = SystemPropertyUtil.getInt( + "io.netty.allocator.cacheTrimInterval", 8192); + + DEFAULT_CACHE_TRIM_INTERVAL_MILLIS = SystemPropertyUtil.getLong( + "io.netty.allocator.cacheTrimIntervalMillis", 0); + + DEFAULT_USE_CACHE_FOR_ALL_THREADS = SystemPropertyUtil.getBoolean( + "io.netty.allocator.useCacheForAllThreads", false); + + // Use 1023 by default as we use an ArrayDeque as backing storage which will then allocate an internal array + // of 1024 elements. Otherwise, we would allocate 2048 and only use 1024 which is wasteful. + DEFAULT_MAX_CACHED_BYTEBUFFERS_PER_CHUNK = SystemPropertyUtil.getInt( + "io.netty.allocator.maxCachedByteBuffersPerChunk", 1023); + + if (logger.isDebugEnabled()) { + logger.debug("-Dio.netty.allocator.numArenas: {}", DEFAULT_NUM_HEAP_ARENA); + logger.debug("-Dio.netty.allocator.numDirectArenas: {}", DEFAULT_NUM_DIRECT_ARENA); + if (pageSizeFallbackCause == null) { + logger.debug("-Dio.netty.allocator.pageSize: {}", DEFAULT_PAGE_SIZE); + } else { + logger.debug("-Dio.netty.allocator.pageSize: {}", DEFAULT_PAGE_SIZE, pageSizeFallbackCause); + } + if (maxOrderFallbackCause == null) { + logger.debug("-Dio.netty.allocator.maxOrder: {}", DEFAULT_MAX_ORDER); + } else { + logger.debug("-Dio.netty.allocator.maxOrder: {}", DEFAULT_MAX_ORDER, maxOrderFallbackCause); + } + logger.debug("-Dio.netty.allocator.chunkSize: {}", DEFAULT_PAGE_SIZE << DEFAULT_MAX_ORDER); + logger.debug("-Dio.netty.allocator.smallCacheSize: {}", DEFAULT_SMALL_CACHE_SIZE); + logger.debug("-Dio.netty.allocator.normalCacheSize: {}", DEFAULT_NORMAL_CACHE_SIZE); + logger.debug("-Dio.netty.allocator.maxCachedBufferCapacity: {}", DEFAULT_MAX_CACHED_BUFFER_CAPACITY); + logger.debug("-Dio.netty.allocator.cacheTrimInterval: {}", DEFAULT_CACHE_TRIM_INTERVAL); + logger.debug("-Dio.netty.allocator.cacheTrimIntervalMillis: {}", DEFAULT_CACHE_TRIM_INTERVAL_MILLIS); + logger.debug("-Dio.netty.allocator.useCacheForAllThreads: {}", DEFAULT_USE_CACHE_FOR_ALL_THREADS); + logger.debug("-Dio.netty.allocator.maxCachedByteBuffersPerChunk: {}", + DEFAULT_MAX_CACHED_BYTEBUFFERS_PER_CHUNK); + } + } + + private final MemoryManager manager; + private final PoolArena[] arenas; + private final int smallCacheSize; + private final int normalCacheSize; + private final List arenaMetrics; + private final PoolThreadLocalCache threadCache; + private final int chunkSize; + private final PooledByteBufAllocatorMetric metric; + + public PooledByteBufAllocator(MemoryManager manager) { + this(manager, manager.isNative()? DEFAULT_NUM_DIRECT_ARENA : DEFAULT_NUM_HEAP_ARENA, + DEFAULT_PAGE_SIZE, DEFAULT_MAX_ORDER, DEFAULT_SMALL_CACHE_SIZE, + DEFAULT_NORMAL_CACHE_SIZE, DEFAULT_USE_CACHE_FOR_ALL_THREADS, + DEFAULT_DIRECT_MEMORY_CACHE_ALIGNMENT); + } + + public PooledByteBufAllocator(MemoryManager manager, int numArenas, int pageSize, int maxOrder) { + this(manager, numArenas, pageSize, maxOrder, DEFAULT_SMALL_CACHE_SIZE, + DEFAULT_NORMAL_CACHE_SIZE, DEFAULT_USE_CACHE_FOR_ALL_THREADS, + DEFAULT_DIRECT_MEMORY_CACHE_ALIGNMENT); + } + + public PooledByteBufAllocator(MemoryManager manager, int numArenas, int pageSize, int maxOrder, + int smallCacheSize, int normalCacheSize, + boolean useCacheForAllThreads) { + this(manager, numArenas, pageSize, maxOrder, + smallCacheSize, normalCacheSize, + useCacheForAllThreads, DEFAULT_DIRECT_MEMORY_CACHE_ALIGNMENT); + } + + public PooledByteBufAllocator(MemoryManager manager, int numArenas, int pageSize, int maxOrder, + int smallCacheSize, int normalCacheSize, + boolean useCacheForAllThreads, int directMemoryCacheAlignment) { + this.manager = requireNonNull(manager, "MemoryManager"); + threadCache = new PoolThreadLocalCache(useCacheForAllThreads); + this.smallCacheSize = smallCacheSize; + this.normalCacheSize = normalCacheSize; + + if (directMemoryCacheAlignment != 0) { + if (!PlatformDependent.hasAlignDirectByteBuffer()) { + throw new UnsupportedOperationException("Buffer alignment is not supported. " + + "Either Unsafe or ByteBuffer.alignSlice() must be available."); + } + + // Ensure page size is a whole multiple of the alignment, or bump it to the next whole multiple. + pageSize = (int) PlatformDependent.align(pageSize, directMemoryCacheAlignment); + } + + chunkSize = validateAndCalculateChunkSize(pageSize, maxOrder); + + checkPositiveOrZero(numArenas, "numArenas"); + + checkPositiveOrZero(directMemoryCacheAlignment, "directMemoryCacheAlignment"); + if (directMemoryCacheAlignment > 0 && !isDirectMemoryCacheAlignmentSupported()) { + throw new IllegalArgumentException("directMemoryCacheAlignment is not supported"); + } + + if ((directMemoryCacheAlignment & -directMemoryCacheAlignment) != directMemoryCacheAlignment) { + throw new IllegalArgumentException("directMemoryCacheAlignment: " + + directMemoryCacheAlignment + " (expected: power of two)"); + } + + int pageShifts = validateAndCalculatePageShifts(pageSize, directMemoryCacheAlignment); + + if (numArenas > 0) { + arenas = newArenaArray(numArenas); + List metrics = new ArrayList<>(arenas.length); + for (int i = 0; i < arenas.length; i ++) { + PoolArena arena = new PoolArena(this, manager, + pageSize, pageShifts, chunkSize, + directMemoryCacheAlignment); + arenas[i] = arena; + metrics.add(arena); + } + arenaMetrics = Collections.unmodifiableList(metrics); + } else { + arenas = null; + arenaMetrics = Collections.emptyList(); + } + + metric = new PooledByteBufAllocatorMetric(this); + } + + private static PoolArena[] newArenaArray(int size) { + return new PoolArena[size]; + } + + private static int validateAndCalculatePageShifts(int pageSize, int alignment) { + if (pageSize < MIN_PAGE_SIZE) { + throw new IllegalArgumentException("pageSize: " + pageSize + " (expected: " + MIN_PAGE_SIZE + ')'); + } + + if ((pageSize & pageSize - 1) != 0) { + throw new IllegalArgumentException("pageSize: " + pageSize + " (expected: power of 2)"); + } + + if (pageSize < alignment) { + throw new IllegalArgumentException("Alignment cannot be greater than page size. " + + "Alignment: " + alignment + ", page size: " + pageSize + '.'); + } + + // Logarithm base 2. At this point we know that pageSize is a power of two. + return Integer.SIZE - 1 - Integer.numberOfLeadingZeros(pageSize); + } + + private static int validateAndCalculateChunkSize(int pageSize, int maxOrder) { + if (maxOrder > 14) { + throw new IllegalArgumentException("maxOrder: " + maxOrder + " (expected: 0-14)"); + } + + // Ensure the resulting chunkSize does not overflow. + int chunkSize = pageSize; + for (int i = maxOrder; i > 0; i --) { + if (chunkSize > MAX_CHUNK_SIZE / 2) { + throw new IllegalArgumentException(String.format( + "pageSize (%d) << maxOrder (%d) must not exceed %d", pageSize, maxOrder, MAX_CHUNK_SIZE)); + } + chunkSize <<= 1; + } + return chunkSize; + } + + @Override + public Buffer allocate(int size) { + PoolThreadCache cache = threadCache.get(); + PoolArena arena = cache.arena; + + if (arena != null) { + return arena.allocate(cache, size); + } + BufferAllocator unpooled = manager.isNative()? BufferAllocator.direct() : BufferAllocator.heap(); + return unpooled.allocate(size); + } + + @Override + public Object allocateUntethered(Buffer originator, int size) { + // TODO + return null; + } + + @Override + public void recoverMemory(Object memory) { + // TODO + } + + /** + * Default number of heap arenas - System Property: io.netty.allocator.numHeapArenas - default 2 * cores + */ + public static int defaultNumHeapArena() { + return DEFAULT_NUM_HEAP_ARENA; + } + + /** + * Default number of direct arenas - System Property: io.netty.allocator.numDirectArenas - default 2 * cores + */ + public static int defaultNumDirectArena() { + return DEFAULT_NUM_DIRECT_ARENA; + } + + /** + * Default buffer page size - System Property: io.netty.allocator.pageSize - default 8192 + */ + public static int defaultPageSize() { + return DEFAULT_PAGE_SIZE; + } + + /** + * Default maximum order - System Property: io.netty.allocator.maxOrder - default 11 + */ + public static int defaultMaxOrder() { + return DEFAULT_MAX_ORDER; + } + + /** + * Default thread caching behavior - System Property: io.netty.allocator.useCacheForAllThreads - default true + */ + public static boolean defaultUseCacheForAllThreads() { + return DEFAULT_USE_CACHE_FOR_ALL_THREADS; + } + + /** + * Default prefer direct - System Property: io.netty.noPreferDirect - default false + */ + public static boolean defaultPreferDirect() { + return PlatformDependent.directBufferPreferred(); + } + + /** + * Default small cache size - System Property: io.netty.allocator.smallCacheSize - default 256 + */ + public static int defaultSmallCacheSize() { + return DEFAULT_SMALL_CACHE_SIZE; + } + + /** + * Default normal cache size - System Property: io.netty.allocator.normalCacheSize - default 64 + */ + public static int defaultNormalCacheSize() { + return DEFAULT_NORMAL_CACHE_SIZE; + } + + /** + * Return {@code true} if direct memory cache alignment is supported, {@code false} otherwise. + */ + public static boolean isDirectMemoryCacheAlignmentSupported() { + return PlatformDependent.hasUnsafe(); + } + + public boolean isDirectBufferPooled() { + return manager.isNative(); + } + + public int numArenas() { + return arenas.length; + } + + final class PoolThreadLocalCache extends FastThreadLocal { + private final boolean useCacheForAllThreads; + + PoolThreadLocalCache(boolean useCacheForAllThreads) { + this.useCacheForAllThreads = useCacheForAllThreads; + } + + @Override + protected synchronized PoolThreadCache initialValue() { + final PoolArena arena = leastUsedArena(arenas); + + final Thread current = Thread.currentThread(); + if (useCacheForAllThreads || current instanceof FastThreadLocalThread) { + final PoolThreadCache cache = new PoolThreadCache( + arena, smallCacheSize, normalCacheSize, + DEFAULT_MAX_CACHED_BUFFER_CAPACITY, DEFAULT_CACHE_TRIM_INTERVAL); + + if (DEFAULT_CACHE_TRIM_INTERVAL_MILLIS > 0) { + final EventExecutor executor = ThreadExecutorMap.currentExecutor(); + if (executor != null) { + executor.scheduleAtFixedRate(trimTask, DEFAULT_CACHE_TRIM_INTERVAL_MILLIS, + DEFAULT_CACHE_TRIM_INTERVAL_MILLIS, TimeUnit.MILLISECONDS); + } + } + return cache; + } + // No caching so just use 0 as sizes. + return new PoolThreadCache(arena, 0, 0, 0, 0); + } + + @Override + protected void onRemoval(PoolThreadCache threadCache) { + threadCache.free(); + } + + private PoolArena leastUsedArena(PoolArena[] arenas) { + if (arenas == null || arenas.length == 0) { + return null; + } + + PoolArena minArena = arenas[0]; + for (int i = 1; i < arenas.length; i++) { + PoolArena arena = arenas[i]; + if (arena.numThreadCaches.get() < minArena.numThreadCaches.get()) { + minArena = arena; + } + } + + return minArena; + } + } + + @Override + public PooledByteBufAllocatorMetric metric() { + return metric; + } + + /** + * Return a {@link List} of all heap {@link PoolArenaMetric}s that are provided by this pool. + */ + List arenaMetrics() { + return arenaMetrics; + } + + /** + * Return the number of thread local caches used by this {@link PooledByteBufAllocator}. + */ + int numThreadLocalCaches() { + if (arenas == null) { + return 0; + } + + int total = 0; + for (PoolArena arena : arenas) { + total += arena.numThreadCaches.get(); + } + + return total; + } + + /** + * Return the size of the small cache. + */ + int smallCacheSize() { + return smallCacheSize; + } + + /** + * Return the size of the normal cache. + */ + int normalCacheSize() { + return normalCacheSize; + } + + /** + * Return the chunk size for an arena. + */ + final int chunkSize() { + return chunkSize; + } + + final long usedMemory() { + return usedMemory(arenas); + } + + private static long usedMemory(PoolArena[] arenas) { + if (arenas == null) { + return -1; + } + long used = 0; + for (PoolArena arena : arenas) { + used += arena.numActiveBytes(); + if (used < 0) { + return Long.MAX_VALUE; + } + } + return used; + } + + final PoolThreadCache threadCache() { + PoolThreadCache cache = threadCache.get(); + assert cache != null; + return cache; + } + + /** + * Trim thread local cache for the current {@link Thread}, which will give back any cached memory that was not + * allocated frequently since the last trim operation. + * + * Returns {@code true} if a cache for the current {@link Thread} exists and so was trimmed, false otherwise. + */ + public boolean trimCurrentThreadCache() { + PoolThreadCache cache = threadCache.getIfExists(); + if (cache != null) { + cache.trim(); + return true; + } + return false; + } + + /** + * Returns the status of the allocator (which contains all metrics) as string. Be aware this may be expensive + * and so should not be called too frequently. + */ + public String dumpStats() { + int heapArenasLen = arenas == null ? 0 : arenas.length; + StringBuilder buf = new StringBuilder(512) + .append(heapArenasLen) + .append(" arena(s):") + .append(StringUtil.NEWLINE); + if (heapArenasLen > 0) { + for (PoolArena a: arenas) { + buf.append(a); + } + } + + return buf.toString(); + } +} diff --git a/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocatorMetric.java b/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocatorMetric.java new file mode 100644 index 0000000..3d70c72 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocatorMetric.java @@ -0,0 +1,78 @@ +package io.netty.buffer.api.pool; + +import io.netty.util.internal.StringUtil; + +import java.util.List; + +/** + * Exposed metric for {@link PooledByteBufAllocator}. + */ +@SuppressWarnings("deprecation") +public final class PooledByteBufAllocatorMetric implements ByteBufAllocatorMetric { + + private final PooledByteBufAllocator allocator; + + PooledByteBufAllocatorMetric(PooledByteBufAllocator allocator) { + this.allocator = allocator; + } + + /** + * Return the number of arenas. + */ + public int numArenas() { + return allocator.numArenas(); + } + + /** + * Return a {@link List} of all {@link PoolArenaMetric}s that are provided by this pool. + */ + public List arenaMetrics() { + return allocator.arenaMetrics(); + } + + /** + * Return the number of thread local caches used by this {@link PooledByteBufAllocator}. + */ + public int numThreadLocalCaches() { + return allocator.numThreadLocalCaches(); + } + + /** + * Return the size of the small cache. + */ + public int smallCacheSize() { + return allocator.smallCacheSize(); + } + + /** + * Return the size of the normal cache. + */ + public int normalCacheSize() { + return allocator.normalCacheSize(); + } + + /** + * Return the chunk size for an arena. + */ + public int chunkSize() { + return allocator.chunkSize(); + } + + @Override + public long usedMemory() { + return allocator.usedMemory(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(256); + sb.append(StringUtil.simpleClassName(this)) + .append("(usedMemory: ").append(usedMemory()) + .append("; numArenas: ").append(numArenas()) + .append("; smallCacheSize: ").append(smallCacheSize()) + .append("; normalCacheSize: ").append(normalCacheSize()) + .append("; numThreadLocalCaches: ").append(numThreadLocalCaches()) + .append("; chunkSize: ").append(chunkSize()).append(')'); + return sb.toString(); + } +} diff --git a/src/main/java/io/netty/buffer/api/pool/SizeClasses.java b/src/main/java/io/netty/buffer/api/pool/SizeClasses.java new file mode 100644 index 0000000..6022cd7 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/SizeClasses.java @@ -0,0 +1,392 @@ +package io.netty.buffer.api.pool; + +import static io.netty.buffer.api.pool.PoolThreadCache.*; + +/** + * SizeClasses requires {@code pageShifts} to be defined prior to inclusion, + * and it in turn defines: + *

+ * LOG2_SIZE_CLASS_GROUP: Log of size class count for each size doubling. + * LOG2_MAX_LOOKUP_SIZE: Log of max size class in the lookup table. + * sizeClasses: Complete table of [index, log2Group, log2Delta, nDelta, isMultiPageSize, + * isSubPage, log2DeltaLookup] tuples. + * index: Size class index. + * log2Group: Log of group base size (no deltas added). + * log2Delta: Log of delta to previous size class. + * nDelta: Delta multiplier. + * isMultiPageSize: 'yes' if a multiple of the page size, 'no' otherwise. + * isSubPage: 'yes' if a subpage size class, 'no' otherwise. + * log2DeltaLookup: Same as log2Delta if a lookup table size class, 'no' + * otherwise. + *

+ * nSubpages: Number of subpages size classes. + * nSizes: Number of size classes. + * nPSizes: Number of size classes that are multiples of pageSize. + * + * smallMaxSizeIdx: Maximum small size class index. + * + * lookupMaxclass: Maximum size class included in lookup table. + * log2NormalMinClass: Log of minimum normal size class. + *

+ * The first size class and spacing are 1 << LOG2_QUANTUM. + * Each group has 1 << LOG2_SIZE_CLASS_GROUP of size classes. + * + * size = 1 << log2Group + nDelta * (1 << log2Delta) + * + * The first size class has an unusual encoding, because the size has to be + * split between group and delta*nDelta. + * + * If pageShift = 13, sizeClasses looks like this: + * + * (index, log2Group, log2Delta, nDelta, isMultiPageSize, isSubPage, log2DeltaLookup) + *

+ * ( 0, 4, 4, 0, no, yes, 4) + * ( 1, 4, 4, 1, no, yes, 4) + * ( 2, 4, 4, 2, no, yes, 4) + * ( 3, 4, 4, 3, no, yes, 4) + *

+ * ( 4, 6, 4, 1, no, yes, 4) + * ( 5, 6, 4, 2, no, yes, 4) + * ( 6, 6, 4, 3, no, yes, 4) + * ( 7, 6, 4, 4, no, yes, 4) + *

+ * ( 8, 7, 5, 1, no, yes, 5) + * ( 9, 7, 5, 2, no, yes, 5) + * ( 10, 7, 5, 3, no, yes, 5) + * ( 11, 7, 5, 4, no, yes, 5) + * ... + * ... + * ( 72, 23, 21, 1, yes, no, no) + * ( 73, 23, 21, 2, yes, no, no) + * ( 74, 23, 21, 3, yes, no, no) + * ( 75, 23, 21, 4, yes, no, no) + *

+ * ( 76, 24, 22, 1, yes, no, no) + */ +abstract class SizeClasses implements SizeClassesMetric { + + static final int LOG2_QUANTUM = 4; + + private static final int LOG2_SIZE_CLASS_GROUP = 2; + private static final int LOG2_MAX_LOOKUP_SIZE = 12; + + private static final int INDEX_IDX = 0; + private static final int LOG2GROUP_IDX = 1; + private static final int LOG2DELTA_IDX = 2; + private static final int NDELTA_IDX = 3; + private static final int PAGESIZE_IDX = 4; + private static final int SUBPAGE_IDX = 5; + private static final int LOG2_DELTA_LOOKUP_IDX = 6; + + private static final byte no = 0, yes = 1; + + protected SizeClasses(int pageSize, int pageShifts, int chunkSize, int directMemoryCacheAlignment) { + this.pageSize = pageSize; + this.pageShifts = pageShifts; + this.chunkSize = chunkSize; + this.directMemoryCacheAlignment = directMemoryCacheAlignment; + + int group = log2(chunkSize) + 1 - LOG2_QUANTUM; + + //generate size classes + //[index, log2Group, log2Delta, nDelta, isMultiPageSize, isSubPage, log2DeltaLookup] + sizeClasses = new short[group << LOG2_SIZE_CLASS_GROUP][7]; + nSizes = sizeClasses(); + + //generate lookup table + sizeIdx2sizeTab = new int[nSizes]; + pageIdx2sizeTab = new int[nPSizes]; + idx2SizeTab(sizeIdx2sizeTab, pageIdx2sizeTab); + + size2idxTab = new int[lookupMaxSize >> LOG2_QUANTUM]; + size2idxTab(size2idxTab); + } + + protected final int pageSize; + protected final int pageShifts; + protected final int chunkSize; + protected final int directMemoryCacheAlignment; + + final int nSizes; + int nSubpages; + int nPSizes; + + int smallMaxSizeIdx; + + private int lookupMaxSize; + + private final short[][] sizeClasses; + + private final int[] pageIdx2sizeTab; + + // lookup table for sizeIdx <= smallMaxSizeIdx + private final int[] sizeIdx2sizeTab; + + // lookup table used for size <= lookupMaxclass + // spacing is 1 << LOG2_QUANTUM, so the size of array is lookupMaxclass >> LOG2_QUANTUM + private final int[] size2idxTab; + + private int sizeClasses() { + int normalMaxSize = -1; + + int index = 0; + int size = 0; + + int log2Group = LOG2_QUANTUM; + int log2Delta = LOG2_QUANTUM; + int ndeltaLimit = 1 << LOG2_SIZE_CLASS_GROUP; + + //First small group, nDelta start at 0. + //first size class is 1 << LOG2_QUANTUM + int nDelta = 0; + while (nDelta < ndeltaLimit) { + size = sizeClass(index++, log2Group, log2Delta, nDelta++); + } + log2Group += LOG2_SIZE_CLASS_GROUP; + + //All remaining groups, nDelta start at 1. + while (size < chunkSize) { + nDelta = 1; + + while (nDelta <= ndeltaLimit && size < chunkSize) { + size = sizeClass(index++, log2Group, log2Delta, nDelta++); + normalMaxSize = size; + } + + log2Group++; + log2Delta++; + } + + //chunkSize must be normalMaxSize + assert chunkSize == normalMaxSize; + + //return number of size index + return index; + } + + //calculate size class + private int sizeClass(int index, int log2Group, int log2Delta, int nDelta) { + short isMultiPageSize; + if (log2Delta >= pageShifts) { + isMultiPageSize = yes; + } else { + int pageSize = 1 << pageShifts; + int size = (1 << log2Group) + (1 << log2Delta) * nDelta; + + isMultiPageSize = size == size / pageSize * pageSize? yes : no; + } + + int log2Ndelta = nDelta == 0? 0 : log2(nDelta); + + byte remove = 1 << log2Ndelta < nDelta? yes : no; + + int log2Size = log2Delta + log2Ndelta == log2Group? log2Group + 1 : log2Group; + if (log2Size == log2Group) { + remove = yes; + } + + short isSubpage = log2Size < pageShifts + LOG2_SIZE_CLASS_GROUP? yes : no; + + int log2DeltaLookup = log2Size < LOG2_MAX_LOOKUP_SIZE || + log2Size == LOG2_MAX_LOOKUP_SIZE && remove == no + ? log2Delta : no; + + short[] sz = { + (short) index, (short) log2Group, (short) log2Delta, + (short) nDelta, isMultiPageSize, isSubpage, (short) log2DeltaLookup + }; + + sizeClasses[index] = sz; + int size = (1 << log2Group) + (nDelta << log2Delta); + + if (sz[PAGESIZE_IDX] == yes) { + nPSizes++; + } + if (sz[SUBPAGE_IDX] == yes) { + nSubpages++; + smallMaxSizeIdx = index; + } + if (sz[LOG2_DELTA_LOOKUP_IDX] != no) { + lookupMaxSize = size; + } + return size; + } + + private void idx2SizeTab(int[] sizeIdx2sizeTab, int[] pageIdx2sizeTab) { + int pageIdx = 0; + + for (int i = 0; i < nSizes; i++) { + short[] sizeClass = sizeClasses[i]; + int log2Group = sizeClass[LOG2GROUP_IDX]; + int log2Delta = sizeClass[LOG2DELTA_IDX]; + int nDelta = sizeClass[NDELTA_IDX]; + + int size = (1 << log2Group) + (nDelta << log2Delta); + sizeIdx2sizeTab[i] = size; + + if (sizeClass[PAGESIZE_IDX] == yes) { + pageIdx2sizeTab[pageIdx++] = size; + } + } + } + + private void size2idxTab(int[] size2idxTab) { + int idx = 0; + int size = 0; + + for (int i = 0; size <= lookupMaxSize; i++) { + int log2Delta = sizeClasses[i][LOG2DELTA_IDX]; + int times = 1 << log2Delta - LOG2_QUANTUM; + + while (size <= lookupMaxSize && times-- > 0) { + size2idxTab[idx++] = i; + size = idx + 1 << LOG2_QUANTUM; + } + } + } + + @Override + public int sizeIdx2size(int sizeIdx) { + return sizeIdx2sizeTab[sizeIdx]; + } + + @Override + public int sizeIdx2sizeCompute(int sizeIdx) { + int group = sizeIdx >> LOG2_SIZE_CLASS_GROUP; + int mod = sizeIdx & (1 << LOG2_SIZE_CLASS_GROUP) - 1; + + int groupSize = group == 0? 0 : + 1 << LOG2_QUANTUM + LOG2_SIZE_CLASS_GROUP - 1 << group; + + int shift = group == 0? 1 : group; + int lgDelta = shift + LOG2_QUANTUM - 1; + int modSize = mod + 1 << lgDelta; + + return groupSize + modSize; + } + + @Override + public long pageIdx2size(int pageIdx) { + return pageIdx2sizeTab[pageIdx]; + } + + @Override + public long pageIdx2sizeCompute(int pageIdx) { + int group = pageIdx >> LOG2_SIZE_CLASS_GROUP; + int mod = pageIdx & (1 << LOG2_SIZE_CLASS_GROUP) - 1; + + long groupSize = group == 0? 0 : + 1L << pageShifts + LOG2_SIZE_CLASS_GROUP - 1 << group; + + int shift = group == 0? 1 : group; + int log2Delta = shift + pageShifts - 1; + int modSize = mod + 1 << log2Delta; + + return groupSize + modSize; + } + + @Override + public int size2SizeIdx(int size) { + if (size == 0) { + return 0; + } + if (size > chunkSize) { + return nSizes; + } + + if (directMemoryCacheAlignment > 0) { + size = alignSize(size); + } + + if (size <= lookupMaxSize) { + //size-1 / MIN_TINY + return size2idxTab[size - 1 >> LOG2_QUANTUM]; + } + + int x = log2((size << 1) - 1); + int shift = x < LOG2_SIZE_CLASS_GROUP + LOG2_QUANTUM + 1 + ? 0 : x - (LOG2_SIZE_CLASS_GROUP + LOG2_QUANTUM); + + int group = shift << LOG2_SIZE_CLASS_GROUP; + + int log2Delta = x < LOG2_SIZE_CLASS_GROUP + LOG2_QUANTUM + 1 + ? LOG2_QUANTUM : x - LOG2_SIZE_CLASS_GROUP - 1; + + int deltaInverseMask = -1 << log2Delta; + int mod = (size - 1 & deltaInverseMask) >> log2Delta & + (1 << LOG2_SIZE_CLASS_GROUP) - 1; + + return group + mod; + } + + @Override + public int pages2pageIdx(int pages) { + return pages2pageIdxCompute(pages, false); + } + + @Override + public int pages2pageIdxFloor(int pages) { + return pages2pageIdxCompute(pages, true); + } + + private int pages2pageIdxCompute(int pages, boolean floor) { + int pageSize = pages << pageShifts; + if (pageSize > chunkSize) { + return nPSizes; + } + + int x = log2((pageSize << 1) - 1); + + int shift = x < LOG2_SIZE_CLASS_GROUP + pageShifts + ? 0 : x - (LOG2_SIZE_CLASS_GROUP + pageShifts); + + int group = shift << LOG2_SIZE_CLASS_GROUP; + + int log2Delta = x < LOG2_SIZE_CLASS_GROUP + pageShifts + 1? + pageShifts : x - LOG2_SIZE_CLASS_GROUP - 1; + + int deltaInverseMask = -1 << log2Delta; + int mod = (pageSize - 1 & deltaInverseMask) >> log2Delta & + (1 << LOG2_SIZE_CLASS_GROUP) - 1; + + int pageIdx = group + mod; + + if (floor && pageIdx2sizeTab[pageIdx] > pages << pageShifts) { + pageIdx--; + } + + return pageIdx; + } + + // Round size up to the nearest multiple of alignment. + private int alignSize(int size) { + int delta = size & directMemoryCacheAlignment - 1; + return delta == 0? size : size + directMemoryCacheAlignment - delta; + } + + @Override + public int normalizeSize(int size) { + if (size == 0) { + return sizeIdx2sizeTab[0]; + } + if (directMemoryCacheAlignment > 0) { + size = alignSize(size); + } + + if (size <= lookupMaxSize) { + int ret = sizeIdx2sizeTab[size2idxTab[size - 1 >> LOG2_QUANTUM]]; + assert ret == normalizeSizeCompute(size); + return ret; + } + return normalizeSizeCompute(size); + } + + private static int normalizeSizeCompute(int size) { + int x = log2((size << 1) - 1); + int log2Delta = x < LOG2_SIZE_CLASS_GROUP + LOG2_QUANTUM + 1 + ? LOG2_QUANTUM : x - LOG2_SIZE_CLASS_GROUP - 1; + int delta = 1 << log2Delta; + int delta_mask = delta - 1; + return size + delta_mask & ~delta_mask; + } +} diff --git a/src/main/java/io/netty/buffer/api/pool/SizeClassesMetric.java b/src/main/java/io/netty/buffer/api/pool/SizeClassesMetric.java new file mode 100644 index 0000000..5fcb830 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/SizeClassesMetric.java @@ -0,0 +1,72 @@ +package io.netty.buffer.api.pool; + +/** + * Expose metrics for an SizeClasses. + */ +public interface SizeClassesMetric { + + /** + * Computes size from lookup table according to sizeIdx. + * + * @return size + */ + int sizeIdx2size(int sizeIdx); + + /** + * Computes size according to sizeIdx. + * + * @return size + */ + int sizeIdx2sizeCompute(int sizeIdx); + + /** + * Computes size from lookup table according to pageIdx. + * + * @return size which is multiples of pageSize. + */ + long pageIdx2size(int pageIdx); + + /** + * Computes size according to pageIdx. + * + * @return size which is multiples of pageSize + */ + long pageIdx2sizeCompute(int pageIdx); + + /** + * Normalizes request size up to the nearest size class. + * + * @param size request size + * + * @return sizeIdx of the size class + */ + int size2SizeIdx(int size); + + /** + * Normalizes request size up to the nearest pageSize class. + * + * @param pages multiples of pageSizes + * + * @return pageIdx of the pageSize class + */ + int pages2pageIdx(int pages); + + /** + * Normalizes request size down to the nearest pageSize class. + * + * @param pages multiples of pageSizes + * + * @return pageIdx of the pageSize class + */ + int pages2pageIdxFloor(int pages); + + /** + * Normalizes usable size that would result from allocating an object with the + * specified size and alignment. + * + * @param size request size + * + * @return normalized size + */ + int normalizeSize(int size); +} diff --git a/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemoryManager.java b/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemoryManager.java index 3bdce07..1298795 100644 --- a/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemoryManager.java +++ b/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemoryManager.java @@ -33,6 +33,11 @@ public class UnsafeMemoryManager implements MemoryManager { this.offheap = offheap; } + @Override + public boolean isNative() { + return offheap; + } + @Override public Buffer allocateShared(AllocatorControl allocatorControl, long size, Drop drop, Cleaner cleaner) { final Object base; From b4b0afd7875cb4617fd2af84a8dca3ff3cd92b77 Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Wed, 12 May 2021 10:44:33 +0200 Subject: [PATCH 03/13] Second, more complete draft of porting over the pooling allocator from Netty --- .../io/netty/buffer/api/pool/PoolArena.java | 62 +++++++++++++------ .../io/netty/buffer/api/pool/PoolChunk.java | 43 ++++++------- .../netty/buffer/api/pool/PoolChunkList.java | 8 +-- .../buffer/api/pool/PoolThreadCache.java | 31 +++++----- .../api/pool/PooledAllocatorControl.java | 26 ++++++++ .../api/pool/PooledByteBufAllocator.java | 22 +++---- 6 files changed, 122 insertions(+), 70 deletions(-) create mode 100644 src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java diff --git a/src/main/java/io/netty/buffer/api/pool/PoolArena.java b/src/main/java/io/netty/buffer/api/pool/PoolArena.java index 61afa4e..4a2c179 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolArena.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolArena.java @@ -1,5 +1,6 @@ package io.netty.buffer.api.pool; +import io.netty.buffer.api.AllocatorControl; import io.netty.buffer.api.Buffer; import io.netty.buffer.api.BufferAllocator; import io.netty.buffer.api.MemoryManager; @@ -14,7 +15,7 @@ import java.util.concurrent.atomic.LongAdder; import static io.netty.buffer.api.pool.PoolChunk.isSubpage; import static java.lang.Math.max; -class PoolArena extends SizeClasses implements PoolArenaMetric { +class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl { enum SizeClass { Small, Normal @@ -98,13 +99,13 @@ class PoolArena extends SizeClasses implements PoolArenaMetric { return manager.isNative(); } - Buffer allocate(PoolThreadCache cache, int size) { + Buffer allocate(PooledAllocatorControl control, PoolThreadCache cache, int size) { final int sizeIdx = size2SizeIdx(size); if (sizeIdx <= smallMaxSizeIdx) { - return tcacheAllocateSmall(cache, size, sizeIdx); + return tcacheAllocateSmall(control, cache, size, sizeIdx); } else if (sizeIdx < nSizes) { - return tcacheAllocateNormal(cache, size, sizeIdx); + return tcacheAllocateNormal(control, cache, size, sizeIdx); } else { int normCapacity = directMemoryCacheAlignment > 0 ? normalizeSize(size) : size; @@ -113,9 +114,9 @@ class PoolArena extends SizeClasses implements PoolArenaMetric { } } - private Buffer tcacheAllocateSmall(PoolThreadCache cache, final int size, - final int sizeIdx) { - Buffer buffer = cache.allocateSmall(size, sizeIdx); + private Buffer tcacheAllocateSmall(PooledAllocatorControl control, PoolThreadCache cache, final int size, + final int sizeIdx) { + Buffer buffer = cache.allocateSmall(control, size, sizeIdx); if (buffer != null) { // was able to allocate out of the cache so move on return buffer; @@ -134,13 +135,13 @@ class PoolArena extends SizeClasses implements PoolArenaMetric { assert s.doNotDestroy && s.elemSize == sizeIdx2size(sizeIdx); long handle = s.allocate(); assert handle >= 0; - buffer = s.chunk.allocateBufferWithSubpage(handle, size, cache); + buffer = s.chunk.allocateBufferWithSubpage(handle, size, cache, control); } } if (needsNormalAllocation) { synchronized (this) { - buffer = allocateNormal(size, sizeIdx, cache); + buffer = allocateNormal(size, sizeIdx, cache, control); } } @@ -148,45 +149,45 @@ class PoolArena extends SizeClasses implements PoolArenaMetric { return buffer; } - private Buffer tcacheAllocateNormal(PoolThreadCache cache, int size, int sizeIdx) { - Buffer buffer = cache.allocateNormal(this, size, sizeIdx); + private Buffer tcacheAllocateNormal(PooledAllocatorControl control, PoolThreadCache cache, int size, int sizeIdx) { + Buffer buffer = cache.allocateNormal(this, control, size, sizeIdx); if (buffer != null) { // was able to allocate out of the cache so move on return buffer; } synchronized (this) { - buffer = allocateNormal(size, sizeIdx, cache); + buffer = allocateNormal(size, sizeIdx, cache, control); allocationsNormal++; } return buffer; } // Method must be called inside synchronized(this) { ... } block - private Buffer allocateNormal(int size, int sizeIdx, PoolThreadCache threadCache) { - Buffer buffer = q050.allocate(size, sizeIdx, threadCache); + private Buffer allocateNormal(int size, int sizeIdx, PoolThreadCache threadCache, PooledAllocatorControl control) { + Buffer buffer = q050.allocate(size, sizeIdx, threadCache, control); if (buffer != null) { return buffer; } - buffer = q025.allocate(size, sizeIdx, threadCache); + buffer = q025.allocate(size, sizeIdx, threadCache, control); if (buffer != null) { return buffer; } - buffer = q000.allocate(size, sizeIdx, threadCache); + buffer = q000.allocate(size, sizeIdx, threadCache, control); if (buffer != null) { return buffer; } - buffer = qInit.allocate(size, sizeIdx, threadCache); + buffer = qInit.allocate(size, sizeIdx, threadCache, control); if (buffer != null) { return buffer; } - buffer = q075.allocate(size, sizeIdx, threadCache); + buffer = q075.allocate(size, sizeIdx, threadCache, control); if (buffer != null) { return buffer; } // Add a new chunk. PoolChunk c = newChunk(pageSize, nPSizes, pageShifts, chunkSize); - buffer = c.allocate(size, sizeIdx, threadCache); + buffer = c.allocate(size, sizeIdx, threadCache, control); assert buffer != null; qInit.add(c); return buffer; @@ -249,6 +250,18 @@ class PoolArena extends SizeClasses implements PoolArenaMetric { return smallSubpagePools[sizeIdx]; } + @Override + public Object allocateUntethered(Buffer originator, int size) { + throw new AssertionError("PoolChunk base buffers should never need to reallocate."); + } + + @Override + public void recoverMemory(Object memory) { + // This means we've lost all strong references to a PoolChunk. + // Probably means we don't need it anymore, so just free its memory. + manager.discardRecoverableMemory(memory); + } + @Override public int numThreadCaches() { return numThreadCaches.get(); @@ -397,7 +410,7 @@ class PoolArena extends SizeClasses implements PoolArenaMetric { } protected final PoolChunk newChunk(int pageSize, int maxPageIdx, int pageShifts, int chunkSize) { - Buffer base = manager.allocateShared(parent, chunkSize, manager.drop(), Statics.CLEANER); + Buffer base = manager.allocateShared(this, chunkSize, manager.drop(), Statics.CLEANER); Object memory = manager.unwrapRecoverableMemory(base); return new PoolChunk( this, base, memory, pageSize, pageShifts, chunkSize, maxPageIdx); @@ -454,4 +467,13 @@ class PoolArena extends SizeClasses implements PoolArenaMetric { } while (s != head); } } + + public void close() { + for (PoolSubpage page : smallSubpagePools) { + page.destroy(); + } + for (PoolChunkList list : new PoolChunkList[] {qInit, q000, q025, q050, q100}) { + list.destroy(); + } + } } diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java index 1804a51..c9b65e9 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java @@ -175,20 +175,6 @@ final class PoolChunk implements PoolChunkMetric { insertAvailRun(0, pages, initHandle); } - /** Creates a special chunk that is not pooled. */ - PoolChunk(PoolArena arena, Buffer base, Object memory, int size) { - unpooled = true; - this.arena = arena; - this.base = base; - this.memory = memory; - pageSize = 0; - pageShifts = 0; - runsAvailMap = null; - runsAvail = null; - subpages = null; - chunkSize = size; - } - private static LongPriorityQueue[] newRunsAvailqueueArray(int size) { LongPriorityQueue[] queueArray = new LongPriorityQueue[size]; for (int i = 0; i < queueArray.length; i++) { @@ -263,7 +249,7 @@ final class PoolChunk implements PoolChunkMetric { return 100 - freePercentage; } - Buffer allocate(int size, int sizeIdx, PoolThreadCache cache) { + Buffer allocate(int size, int sizeIdx, PoolThreadCache cache, PooledAllocatorControl control) { final long handle; if (sizeIdx <= arena.smallMaxSizeIdx) { // small @@ -282,7 +268,7 @@ final class PoolChunk implements PoolChunkMetric { } } - return allocateBuffer(handle, size, cache); + return allocateBuffer(handle, size, cache, control); } private long allocateRun(int runSize) { @@ -514,23 +500,25 @@ final class PoolChunk implements PoolChunkMetric { | (long) inUsed << IS_USED_SHIFT; } - Buffer allocateBuffer(long handle, int size, PoolThreadCache threadCache) { + Buffer allocateBuffer(long handle, int size, PoolThreadCache threadCache, PooledAllocatorControl control) { if (isRun(handle)) { int offset = runOffset(handle) << pageShifts; int maxLength = runSize(pageShifts, handle); PoolThreadCache poolThreadCache = arena.parent.threadCache(); - return arena.manager.recoverMemory(arena.parent, memory, offset, size, new Drop() { + initAllocatorControl(control, poolThreadCache, handle, maxLength, offset, size); + return arena.manager.recoverMemory(control, memory, offset, size, new Drop() { @Override public void drop(Buffer obj) { arena.free(PoolChunk.this, handle, maxLength, poolThreadCache); } }); } else { - return allocateBufferWithSubpage(handle, size, threadCache); + return allocateBufferWithSubpage(handle, size, threadCache, control); } } - Buffer allocateBufferWithSubpage(long handle, int size, PoolThreadCache threadCache) { + Buffer allocateBufferWithSubpage(long handle, int size, PoolThreadCache threadCache, + PooledAllocatorControl control) { int runOffset = runOffset(handle); int bitmapIdx = bitmapIdx(handle); @@ -539,7 +527,8 @@ final class PoolChunk implements PoolChunkMetric { assert size <= s.elemSize; int offset = (runOffset << pageShifts) + bitmapIdx * s.elemSize; - return arena.manager.recoverMemory(arena.parent, memory, offset, size, new Drop() { + initAllocatorControl(control, threadCache, handle, s.elemSize, offset, size); + return arena.manager.recoverMemory(control, memory, offset, size, new Drop() { @Override public void drop(Buffer obj) { arena.free(PoolChunk.this, handle, s.elemSize, threadCache); @@ -547,6 +536,18 @@ final class PoolChunk implements PoolChunkMetric { }); } + private void initAllocatorControl(PooledAllocatorControl control, PoolThreadCache threadCache, long handle, + int normSize, int offset, int size) { + control.arena = arena; + control.chunk = this; + control.threadCache = threadCache; + control.handle = handle; + control.normSize = normSize; + control.memory = memory; + control.offset = offset; + control.size = size; + } + @Override public int chunkSize() { return chunkSize; diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java b/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java index 4c18f95..9211939 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java @@ -76,7 +76,7 @@ final class PoolChunkList implements PoolChunkListMetric { this.prevList = prevList; } - Buffer allocate(int size, int sizeIdx, PoolThreadCache threadCache) { + Buffer allocate(int size, int sizeIdx, PoolThreadCache threadCache, PooledAllocatorControl control) { int normCapacity = arena.sizeIdx2size(sizeIdx); if (normCapacity > maxCapacity) { // Either this PoolChunkList is empty, or the requested capacity is larger than the capacity which can @@ -85,7 +85,7 @@ final class PoolChunkList implements PoolChunkListMetric { } for (PoolChunk cur = head; cur != null; cur = cur.next) { - Buffer buffer = cur.allocate(size, sizeIdx, threadCache); + Buffer buffer = cur.allocate(size, sizeIdx, threadCache, control); if (buffer != null) { if (cur.freeBytes <= freeMinThreshold) { remove(cur); @@ -223,10 +223,10 @@ final class PoolChunkList implements PoolChunkListMetric { return buf.toString(); } - void destroy(PoolArena arena) { + void destroy() { PoolChunk chunk = head; while (chunk != null) { - arena.destroyChunk(chunk); + chunk.destroy(); chunk = chunk.next; } head = null; diff --git a/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java b/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java index 5e986b1..8fb2e93 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java @@ -106,24 +106,24 @@ final class PoolThreadCache { /** * Try to allocate a small buffer out of the cache. Returns {@code true} if successful {@code false} otherwise */ - Buffer allocateSmall(int size, int sizeIdx) { - return allocate(cacheForSmall(sizeIdx), size); + Buffer allocateSmall(PooledAllocatorControl control, int size, int sizeIdx) { + return allocate(cacheForSmall(sizeIdx), control, size); } /** * Try to allocate a normal buffer out of the cache. Returns {@code true} if successful {@code false} otherwise */ - Buffer allocateNormal(PoolArena area, int size, int sizeIdx) { - return allocate(cacheForNormal(area, sizeIdx), size); + Buffer allocateNormal(PoolArena area, PooledAllocatorControl control, int size, int sizeIdx) { + return allocate(cacheForNormal(area, sizeIdx), control, size); } - private Buffer allocate(MemoryRegionCache cache, int size) { + private Buffer allocate(MemoryRegionCache cache, PooledAllocatorControl control, int size) { if (cache == null) { // no cache found so just return false here return null; } - Buffer allocated = cache.allocate(size, this); - if (++ allocations >= freeSweepAllocationThreshold) { + Buffer allocated = cache.allocate(size, this, control); + if (++allocations >= freeSweepAllocationThreshold) { allocations = 0; trim(); } @@ -237,8 +237,9 @@ final class PoolThreadCache { } @Override - protected Buffer allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache) { - return chunk.allocateBufferWithSubpage(handle, size, threadCache); + protected Buffer allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache, + PooledAllocatorControl control) { + return chunk.allocateBufferWithSubpage(handle, size, threadCache, control); } } @@ -251,8 +252,9 @@ final class PoolThreadCache { } @Override - protected Buffer allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache) { - return chunk.allocateBuffer(handle, size, threadCache); + protected Buffer allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache, + PooledAllocatorControl control) { + return chunk.allocateBuffer(handle, size, threadCache, control); } } @@ -271,7 +273,8 @@ final class PoolThreadCache { /** * Allocate a new {@link Buffer} using the provided chunk and handle with the capacity restrictions. */ - protected abstract Buffer allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache); + protected abstract Buffer allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache, + PooledAllocatorControl control); /** * Add to cache if not already full. @@ -290,12 +293,12 @@ final class PoolThreadCache { /** * Allocate something out of the cache if possible and remove the entry from the cache. */ - public final Buffer allocate(int size, PoolThreadCache threadCache) { + public final Buffer allocate(int size, PoolThreadCache threadCache, PooledAllocatorControl control) { Entry entry = queue.poll(); if (entry == null) { return null; } - Buffer buffer = allocBuf(entry.chunk, entry.handle, size, threadCache); + Buffer buffer = allocBuf(entry.chunk, entry.handle, size, threadCache, control); entry.recycle(); // allocations are not thread-safe which is fine as this is only called from the same thread all time. diff --git a/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java b/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java new file mode 100644 index 0000000..1a33990 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java @@ -0,0 +1,26 @@ +package io.netty.buffer.api.pool; + +import io.netty.buffer.api.AllocatorControl; +import io.netty.buffer.api.Buffer; + +class PooledAllocatorControl implements AllocatorControl { + public PoolArena arena; + public PoolChunk chunk; + public PoolThreadCache threadCache; + public long handle; + public int normSize; + public Object memory; + public int offset; + public int size; + + @Override + public Object allocateUntethered(Buffer originator, int size) { + Buffer allocate = arena.parent.allocate(this, size); + return arena.manager.unwrapRecoverableMemory(allocate); + } + + @Override + public void recoverMemory(Object memory) { + arena.free(chunk, handle, normSize, threadCache); + } +} diff --git a/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocator.java b/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocator.java index b9120d3..f5a7aad 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocator.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocator.java @@ -23,7 +23,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; -public class PooledByteBufAllocator implements BufferAllocator, ByteBufAllocatorMetricProvider, AllocatorControl { +public class PooledByteBufAllocator implements BufferAllocator, ByteBufAllocatorMetricProvider { private static final InternalLogger logger = InternalLoggerFactory.getInstance(PooledByteBufAllocator.class); private static final int DEFAULT_NUM_HEAP_ARENA; @@ -257,7 +257,7 @@ public class PooledByteBufAllocator implements BufferAllocator, ByteBufAllocator // Ensure the resulting chunkSize does not overflow. int chunkSize = pageSize; - for (int i = maxOrder; i > 0; i --) { + for (int i = maxOrder; i > 0; i--) { if (chunkSize > MAX_CHUNK_SIZE / 2) { throw new IllegalArgumentException(String.format( "pageSize (%d) << maxOrder (%d) must not exceed %d", pageSize, maxOrder, MAX_CHUNK_SIZE)); @@ -269,25 +269,25 @@ public class PooledByteBufAllocator implements BufferAllocator, ByteBufAllocator @Override public Buffer allocate(int size) { + return allocate(new PooledAllocatorControl(), size); + } + + Buffer allocate(PooledAllocatorControl control, int size) { PoolThreadCache cache = threadCache.get(); PoolArena arena = cache.arena; if (arena != null) { - return arena.allocate(cache, size); + return arena.allocate(control, cache, size); } BufferAllocator unpooled = manager.isNative()? BufferAllocator.direct() : BufferAllocator.heap(); return unpooled.allocate(size); } @Override - public Object allocateUntethered(Buffer originator, int size) { - // TODO - return null; - } - - @Override - public void recoverMemory(Object memory) { - // TODO + public void close() { + for (PoolArena arena : arenas) { + arena.close(); + } } /** From 0c49c887e695aa0e0eaa6d8a7e68212f91d96963 Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Wed, 12 May 2021 10:46:15 +0200 Subject: [PATCH 04/13] Renames to align with the new API --- ...Metric.java => BufferAllocatorMetric.java} | 2 +- .../pool/BufferAllocatorMetricProvider.java | 11 +++++++ .../pool/ByteBufAllocatorMetricProvider.java | 11 ------- .../io/netty/buffer/api/pool/PoolArena.java | 4 +-- ...ocator.java => PooledBufferAllocator.java} | 29 +++++++++---------- ....java => PooledBufferAllocatorMetric.java} | 10 +++---- 6 files changed, 33 insertions(+), 34 deletions(-) rename src/main/java/io/netty/buffer/api/pool/{ByteBufAllocatorMetric.java => BufferAllocatorMetric.java} (84%) create mode 100644 src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetricProvider.java delete mode 100644 src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetricProvider.java rename src/main/java/io/netty/buffer/api/pool/{PooledByteBufAllocator.java => PooledBufferAllocator.java} (94%) rename src/main/java/io/netty/buffer/api/pool/{PooledByteBufAllocatorMetric.java => PooledBufferAllocatorMetric.java} (86%) diff --git a/src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetric.java b/src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetric.java similarity index 84% rename from src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetric.java rename to src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetric.java index 9411cdc..9e5ecb8 100644 --- a/src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetric.java +++ b/src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetric.java @@ -2,7 +2,7 @@ package io.netty.buffer.api.pool; import io.netty.buffer.api.BufferAllocator; -public interface ByteBufAllocatorMetric { +public interface BufferAllocatorMetric { /** * Returns the number of bytes of heap memory used by a {@link BufferAllocator} or {@code -1} if unknown. */ diff --git a/src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetricProvider.java b/src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetricProvider.java new file mode 100644 index 0000000..faac14d --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetricProvider.java @@ -0,0 +1,11 @@ +package io.netty.buffer.api.pool; + +import io.netty.buffer.api.BufferAllocator; + +public interface BufferAllocatorMetricProvider { + + /** + * Returns a {@link BufferAllocatorMetric} for a {@link BufferAllocator}. + */ + BufferAllocatorMetric metric(); +} diff --git a/src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetricProvider.java b/src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetricProvider.java deleted file mode 100644 index a69519e..0000000 --- a/src/main/java/io/netty/buffer/api/pool/ByteBufAllocatorMetricProvider.java +++ /dev/null @@ -1,11 +0,0 @@ -package io.netty.buffer.api.pool; - -import io.netty.buffer.api.BufferAllocator; - -public interface ByteBufAllocatorMetricProvider { - - /** - * Returns a {@link ByteBufAllocatorMetric} for a {@link BufferAllocator}. - */ - ByteBufAllocatorMetric metric(); -} diff --git a/src/main/java/io/netty/buffer/api/pool/PoolArena.java b/src/main/java/io/netty/buffer/api/pool/PoolArena.java index 4a2c179..743c5b9 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolArena.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolArena.java @@ -21,7 +21,7 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl Normal } - final PooledByteBufAllocator parent; + final PooledBufferAllocator parent; final MemoryManager manager; final int numSmallSubpagePools; @@ -54,7 +54,7 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl // Number of thread caches backed by this arena. final AtomicInteger numThreadCaches = new AtomicInteger(); - protected PoolArena(PooledByteBufAllocator parent, MemoryManager manager, int pageSize, + protected PoolArena(PooledBufferAllocator parent, MemoryManager manager, int pageSize, int pageShifts, int chunkSize, int cacheAlignment) { super(pageSize, pageShifts, chunkSize, cacheAlignment); this.parent = parent; diff --git a/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocator.java b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java similarity index 94% rename from src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocator.java rename to src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java index f5a7aad..f8ecf7f 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocator.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java @@ -3,7 +3,6 @@ package io.netty.buffer.api.pool; import static io.netty.util.internal.ObjectUtil.checkPositiveOrZero; import static java.util.Objects.requireNonNull; -import io.netty.buffer.api.AllocatorControl; import io.netty.buffer.api.Buffer; import io.netty.buffer.api.BufferAllocator; import io.netty.buffer.api.MemoryManager; @@ -23,9 +22,9 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; -public class PooledByteBufAllocator implements BufferAllocator, ByteBufAllocatorMetricProvider { +public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMetricProvider { - private static final InternalLogger logger = InternalLoggerFactory.getInstance(PooledByteBufAllocator.class); + private static final InternalLogger logger = InternalLoggerFactory.getInstance(PooledBufferAllocator.class); private static final int DEFAULT_NUM_HEAP_ARENA; private static final int DEFAULT_NUM_DIRECT_ARENA; @@ -152,32 +151,32 @@ public class PooledByteBufAllocator implements BufferAllocator, ByteBufAllocator private final List arenaMetrics; private final PoolThreadLocalCache threadCache; private final int chunkSize; - private final PooledByteBufAllocatorMetric metric; + private final PooledBufferAllocatorMetric metric; - public PooledByteBufAllocator(MemoryManager manager) { + public PooledBufferAllocator(MemoryManager manager) { this(manager, manager.isNative()? DEFAULT_NUM_DIRECT_ARENA : DEFAULT_NUM_HEAP_ARENA, DEFAULT_PAGE_SIZE, DEFAULT_MAX_ORDER, DEFAULT_SMALL_CACHE_SIZE, DEFAULT_NORMAL_CACHE_SIZE, DEFAULT_USE_CACHE_FOR_ALL_THREADS, DEFAULT_DIRECT_MEMORY_CACHE_ALIGNMENT); } - public PooledByteBufAllocator(MemoryManager manager, int numArenas, int pageSize, int maxOrder) { + public PooledBufferAllocator(MemoryManager manager, int numArenas, int pageSize, int maxOrder) { this(manager, numArenas, pageSize, maxOrder, DEFAULT_SMALL_CACHE_SIZE, DEFAULT_NORMAL_CACHE_SIZE, DEFAULT_USE_CACHE_FOR_ALL_THREADS, DEFAULT_DIRECT_MEMORY_CACHE_ALIGNMENT); } - public PooledByteBufAllocator(MemoryManager manager, int numArenas, int pageSize, int maxOrder, - int smallCacheSize, int normalCacheSize, - boolean useCacheForAllThreads) { + public PooledBufferAllocator(MemoryManager manager, int numArenas, int pageSize, int maxOrder, + int smallCacheSize, int normalCacheSize, + boolean useCacheForAllThreads) { this(manager, numArenas, pageSize, maxOrder, smallCacheSize, normalCacheSize, useCacheForAllThreads, DEFAULT_DIRECT_MEMORY_CACHE_ALIGNMENT); } - public PooledByteBufAllocator(MemoryManager manager, int numArenas, int pageSize, int maxOrder, - int smallCacheSize, int normalCacheSize, - boolean useCacheForAllThreads, int directMemoryCacheAlignment) { + public PooledBufferAllocator(MemoryManager manager, int numArenas, int pageSize, int maxOrder, + int smallCacheSize, int normalCacheSize, + boolean useCacheForAllThreads, int directMemoryCacheAlignment) { this.manager = requireNonNull(manager, "MemoryManager"); threadCache = new PoolThreadLocalCache(useCacheForAllThreads); this.smallCacheSize = smallCacheSize; @@ -225,7 +224,7 @@ public class PooledByteBufAllocator implements BufferAllocator, ByteBufAllocator arenaMetrics = Collections.emptyList(); } - metric = new PooledByteBufAllocatorMetric(this); + metric = new PooledBufferAllocatorMetric(this); } private static PoolArena[] newArenaArray(int size) { @@ -414,7 +413,7 @@ public class PooledByteBufAllocator implements BufferAllocator, ByteBufAllocator } @Override - public PooledByteBufAllocatorMetric metric() { + public PooledBufferAllocatorMetric metric() { return metric; } @@ -426,7 +425,7 @@ public class PooledByteBufAllocator implements BufferAllocator, ByteBufAllocator } /** - * Return the number of thread local caches used by this {@link PooledByteBufAllocator}. + * Return the number of thread local caches used by this {@link PooledBufferAllocator}. */ int numThreadLocalCaches() { if (arenas == null) { diff --git a/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocatorMetric.java b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocatorMetric.java similarity index 86% rename from src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocatorMetric.java rename to src/main/java/io/netty/buffer/api/pool/PooledBufferAllocatorMetric.java index 3d70c72..f02b820 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledByteBufAllocatorMetric.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocatorMetric.java @@ -5,14 +5,14 @@ import io.netty.util.internal.StringUtil; import java.util.List; /** - * Exposed metric for {@link PooledByteBufAllocator}. + * Exposed metric for {@link PooledBufferAllocator}. */ @SuppressWarnings("deprecation") -public final class PooledByteBufAllocatorMetric implements ByteBufAllocatorMetric { +public final class PooledBufferAllocatorMetric implements BufferAllocatorMetric { - private final PooledByteBufAllocator allocator; + private final PooledBufferAllocator allocator; - PooledByteBufAllocatorMetric(PooledByteBufAllocator allocator) { + PooledBufferAllocatorMetric(PooledBufferAllocator allocator) { this.allocator = allocator; } @@ -31,7 +31,7 @@ public final class PooledByteBufAllocatorMetric implements ByteBufAllocatorMetri } /** - * Return the number of thread local caches used by this {@link PooledByteBufAllocator}. + * Return the number of thread local caches used by this {@link PooledBufferAllocator}. */ public int numThreadLocalCaches() { return allocator.numThreadLocalCaches(); From 1daa0685dc70ffa5250298fe91a9ec5a273c7b62 Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Wed, 12 May 2021 10:49:09 +0200 Subject: [PATCH 05/13] Add license headers --- .../buffer/api/pool/BufferAllocatorMetric.java | 15 +++++++++++++++ .../pool/BufferAllocatorMetricProvider.java | 15 +++++++++++++++ .../netty/buffer/api/pool/LongLongHashMap.java | 15 +++++++++++++++ .../buffer/api/pool/LongPriorityQueue.java | 15 +++++++++++++++ .../io/netty/buffer/api/pool/PoolArena.java | 15 +++++++++++++++ .../netty/buffer/api/pool/PoolArenaMetric.java | 15 +++++++++++++++ .../io/netty/buffer/api/pool/PoolChunk.java | 15 +++++++++++++++ .../netty/buffer/api/pool/PoolChunkList.java | 15 +++++++++++++++ .../buffer/api/pool/PoolChunkListMetric.java | 15 +++++++++++++++ .../netty/buffer/api/pool/PoolChunkMetric.java | 15 +++++++++++++++ .../io/netty/buffer/api/pool/PoolSubpage.java | 15 +++++++++++++++ .../buffer/api/pool/PoolSubpageMetric.java | 15 +++++++++++++++ .../netty/buffer/api/pool/PoolThreadCache.java | 15 +++++++++++++++ .../api/pool/PooledAllocatorControl.java | 15 +++++++++++++++ .../buffer/api/pool/PooledBufferAllocator.java | 15 +++++++++++++++ .../api/pool/PooledBufferAllocatorMetric.java | 18 ++++++++++++++++-- .../io/netty/buffer/api/pool/SizeClasses.java | 15 +++++++++++++++ .../buffer/api/pool/SizeClassesMetric.java | 15 +++++++++++++++ 18 files changed, 271 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetric.java b/src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetric.java index 9e5ecb8..eeaa676 100644 --- a/src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetric.java +++ b/src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetric.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; import io.netty.buffer.api.BufferAllocator; diff --git a/src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetricProvider.java b/src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetricProvider.java index faac14d..1b19e73 100644 --- a/src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetricProvider.java +++ b/src/main/java/io/netty/buffer/api/pool/BufferAllocatorMetricProvider.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; import io.netty.buffer.api.BufferAllocator; diff --git a/src/main/java/io/netty/buffer/api/pool/LongLongHashMap.java b/src/main/java/io/netty/buffer/api/pool/LongLongHashMap.java index b272452..35e3a3b 100644 --- a/src/main/java/io/netty/buffer/api/pool/LongLongHashMap.java +++ b/src/main/java/io/netty/buffer/api/pool/LongLongHashMap.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; /** diff --git a/src/main/java/io/netty/buffer/api/pool/LongPriorityQueue.java b/src/main/java/io/netty/buffer/api/pool/LongPriorityQueue.java index 9329a60..2d9aa54 100644 --- a/src/main/java/io/netty/buffer/api/pool/LongPriorityQueue.java +++ b/src/main/java/io/netty/buffer/api/pool/LongPriorityQueue.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; import java.util.Arrays; diff --git a/src/main/java/io/netty/buffer/api/pool/PoolArena.java b/src/main/java/io/netty/buffer/api/pool/PoolArena.java index 743c5b9..9a60e74 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolArena.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolArena.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; import io.netty.buffer.api.AllocatorControl; diff --git a/src/main/java/io/netty/buffer/api/pool/PoolArenaMetric.java b/src/main/java/io/netty/buffer/api/pool/PoolArenaMetric.java index e119f1e..bc4d51e 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolArenaMetric.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolArenaMetric.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; import java.util.List; diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java index c9b65e9..37904ea 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; import io.netty.buffer.api.Buffer; diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java b/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java index 9211939..6fe5bae 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; import io.netty.buffer.api.Buffer; diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunkListMetric.java b/src/main/java/io/netty/buffer/api/pool/PoolChunkListMetric.java index 8c3d89f..9a60e1d 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolChunkListMetric.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunkListMetric.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; /** diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunkMetric.java b/src/main/java/io/netty/buffer/api/pool/PoolChunkMetric.java index 69868e7..8a90384 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolChunkMetric.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunkMetric.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; /** diff --git a/src/main/java/io/netty/buffer/api/pool/PoolSubpage.java b/src/main/java/io/netty/buffer/api/pool/PoolSubpage.java index a91c854..4dfc1ec 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolSubpage.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolSubpage.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; import static io.netty.buffer.api.pool.PoolChunk.RUN_OFFSET_SHIFT; diff --git a/src/main/java/io/netty/buffer/api/pool/PoolSubpageMetric.java b/src/main/java/io/netty/buffer/api/pool/PoolSubpageMetric.java index 75b2a53..5793a0a 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolSubpageMetric.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolSubpageMetric.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; /** diff --git a/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java b/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java index 8fb2e93..c74399d 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; import static io.netty.buffer.api.pool.PoolArena.SizeClass.*; diff --git a/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java b/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java index 1a33990..df5fd18 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; import io.netty.buffer.api.AllocatorControl; diff --git a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java index f8ecf7f..420c12f 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; import static io.netty.util.internal.ObjectUtil.checkPositiveOrZero; diff --git a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocatorMetric.java b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocatorMetric.java index f02b820..e4d4f04 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocatorMetric.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocatorMetric.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; import io.netty.util.internal.StringUtil; @@ -7,8 +22,7 @@ import java.util.List; /** * Exposed metric for {@link PooledBufferAllocator}. */ -@SuppressWarnings("deprecation") -public final class PooledBufferAllocatorMetric implements BufferAllocatorMetric { +final class PooledBufferAllocatorMetric implements BufferAllocatorMetric { private final PooledBufferAllocator allocator; diff --git a/src/main/java/io/netty/buffer/api/pool/SizeClasses.java b/src/main/java/io/netty/buffer/api/pool/SizeClasses.java index 6022cd7..8ad00ad 100644 --- a/src/main/java/io/netty/buffer/api/pool/SizeClasses.java +++ b/src/main/java/io/netty/buffer/api/pool/SizeClasses.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; import static io.netty.buffer.api.pool.PoolThreadCache.*; diff --git a/src/main/java/io/netty/buffer/api/pool/SizeClassesMetric.java b/src/main/java/io/netty/buffer/api/pool/SizeClassesMetric.java index 5fcb830..3f3ac3e 100644 --- a/src/main/java/io/netty/buffer/api/pool/SizeClassesMetric.java +++ b/src/main/java/io/netty/buffer/api/pool/SizeClassesMetric.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ package io.netty.buffer.api.pool; /** From 481b2ddd3d0c440203b95de99dd95784b9387a2b Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Wed, 12 May 2021 10:52:22 +0200 Subject: [PATCH 06/13] Switch to the new pooling allocator by default for heap buffers --- src/main/java/io/netty/buffer/api/BufferAllocator.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/netty/buffer/api/BufferAllocator.java b/src/main/java/io/netty/buffer/api/BufferAllocator.java index 2a43bcc..079dbf8 100644 --- a/src/main/java/io/netty/buffer/api/BufferAllocator.java +++ b/src/main/java/io/netty/buffer/api/BufferAllocator.java @@ -15,6 +15,8 @@ */ package io.netty.buffer.api; +import io.netty.buffer.api.pool.PooledBufferAllocator; + import java.nio.ByteOrder; import java.util.function.Supplier; @@ -123,7 +125,8 @@ public interface BufferAllocator extends AutoCloseable { } static BufferAllocator pooledHeap() { - return new SizeClassedMemoryPool(MemoryManagers.getManagers().getHeapMemoryManager()); + return new PooledBufferAllocator(MemoryManagers.getManagers().getHeapMemoryManager()); +// return new SizeClassedMemoryPool(MemoryManagers.getManagers().getHeapMemoryManager()); } static BufferAllocator pooledDirect() { From 6b62b3a6c79064ee72f23ad2208f992910f62eb4 Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Wed, 12 May 2021 10:52:51 +0200 Subject: [PATCH 07/13] The pooling buffer allocator must allocate buffers with native byte order by default --- .../java/io/netty/buffer/api/pool/PooledBufferAllocator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java index 420c12f..bebe1ba 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java @@ -32,6 +32,7 @@ import io.netty.util.internal.ThreadExecutorMap; import io.netty.util.internal.logging.InternalLogger; import io.netty.util.internal.logging.InternalLoggerFactory; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -283,7 +284,7 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe @Override public Buffer allocate(int size) { - return allocate(new PooledAllocatorControl(), size); + return allocate(new PooledAllocatorControl(), size).order(ByteOrder.nativeOrder()); } Buffer allocate(PooledAllocatorControl control, int size) { From 670cca2d4378b41fd78059f3156a41c5200a931f Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Wed, 12 May 2021 16:05:09 +0200 Subject: [PATCH 08/13] Fix more tests Fundamental design issues remain, though. Drops can end up being shared across instances with different memory allocations, and this means we can't currently attach the de-allocation information to the drop instance. We also cannot use the AllocationControl instance for this because it has the same problem. --- .../io/netty/buffer/api/pool/PoolArena.java | 18 ++---- .../io/netty/buffer/api/pool/PoolChunk.java | 29 +++------- .../io/netty/buffer/api/pool/PoolSubpage.java | 2 +- .../api/pool/PooledAllocatorControl.java | 4 +- .../api/pool/PooledBufferAllocator.java | 13 +++-- .../io/netty/buffer/api/pool/PooledDrop.java | 55 +++++++++++++++++++ 6 files changed, 79 insertions(+), 42 deletions(-) create mode 100644 src/main/java/io/netty/buffer/api/pool/PooledDrop.java diff --git a/src/main/java/io/netty/buffer/api/pool/PoolArena.java b/src/main/java/io/netty/buffer/api/pool/PoolArena.java index 9a60e74..7362887 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolArena.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolArena.java @@ -220,20 +220,12 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl } void free(PoolChunk chunk, long handle, int normCapacity, PoolThreadCache cache) { - if (chunk.unpooled) { - int size = chunk.chunkSize(); - chunk.destroy(); - activeBytesHuge.add(-size); - deallocationsHuge.increment(); - } else { - SizeClass sizeClass = sizeClass(handle); - if (cache != null && cache.add(this, chunk, handle, normCapacity, sizeClass)) { - // cached so not free it. - return; - } - - freeChunk(chunk, handle, normCapacity, sizeClass); + SizeClass sizeClass = sizeClass(handle); + if (cache != null && cache.add(this, chunk, handle, normCapacity, sizeClass)) { + // cached so not free it. + return; } + freeChunk(chunk, handle, normCapacity, sizeClass); } private static SizeClass sizeClass(long handle) { diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java index 37904ea..4e8753c 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java @@ -16,7 +16,6 @@ package io.netty.buffer.api.pool; import io.netty.buffer.api.Buffer; -import io.netty.buffer.api.Drop; import java.util.PriorityQueue; @@ -143,7 +142,6 @@ final class PoolChunk implements PoolChunkMetric { final PoolArena arena; final Buffer base; // The buffer that is the source of the memory. Closing it will free the memory. final Object memory; - final boolean unpooled; /** * store the first page and last page of each avail run @@ -171,7 +169,6 @@ final class PoolChunk implements PoolChunkMetric { PoolChunk next; PoolChunk(PoolArena arena, Buffer base, Object memory, int pageSize, int pageShifts, int chunkSize, int maxPageIdx) { - unpooled = false; this.arena = arena; this.base = base; this.memory = memory; @@ -520,13 +517,9 @@ final class PoolChunk implements PoolChunkMetric { int offset = runOffset(handle) << pageShifts; int maxLength = runSize(pageShifts, handle); PoolThreadCache poolThreadCache = arena.parent.threadCache(); - initAllocatorControl(control, poolThreadCache, handle, maxLength, offset, size); - return arena.manager.recoverMemory(control, memory, offset, size, new Drop() { - @Override - public void drop(Buffer obj) { - arena.free(PoolChunk.this, handle, maxLength, poolThreadCache); - } - }); + initAllocatorControl(control, poolThreadCache, handle, maxLength); + return arena.manager.recoverMemory(control, memory, offset, size, + new PooledDrop(control, arena, this, poolThreadCache, handle, maxLength)); } else { return allocateBufferWithSubpage(handle, size, threadCache, control); } @@ -542,25 +535,19 @@ final class PoolChunk implements PoolChunkMetric { assert size <= s.elemSize; int offset = (runOffset << pageShifts) + bitmapIdx * s.elemSize; - initAllocatorControl(control, threadCache, handle, s.elemSize, offset, size); - return arena.manager.recoverMemory(control, memory, offset, size, new Drop() { - @Override - public void drop(Buffer obj) { - arena.free(PoolChunk.this, handle, s.elemSize, threadCache); - } - }); + initAllocatorControl(control, threadCache, handle, s.elemSize); + return arena.manager.recoverMemory(control, memory, offset, size, + new PooledDrop(control, arena, this, threadCache, handle, s.elemSize)); } private void initAllocatorControl(PooledAllocatorControl control, PoolThreadCache threadCache, long handle, - int normSize, int offset, int size) { + int normSize) { control.arena = arena; control.chunk = this; control.threadCache = threadCache; control.handle = handle; control.normSize = normSize; - control.memory = memory; - control.offset = offset; - control.size = size; + control.updates++; } @Override diff --git a/src/main/java/io/netty/buffer/api/pool/PoolSubpage.java b/src/main/java/io/netty/buffer/api/pool/PoolSubpage.java index 4dfc1ec..c0b558d 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolSubpage.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolSubpage.java @@ -108,7 +108,7 @@ final class PoolSubpage implements PoolSubpageMetric { setNextAvail(bitmapIdx); - if (numAvail ++ == 0) { + if (numAvail++ == 0) { addToPool(head); // When maxNumElems == 1, the maximum numAvail is also 1. // Each of these PoolSubpages will go in here when they do free operation. diff --git a/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java b/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java index df5fd18..a4e53ca 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java @@ -24,9 +24,7 @@ class PooledAllocatorControl implements AllocatorControl { public PoolThreadCache threadCache; public long handle; public int normSize; - public Object memory; - public int offset; - public int size; + public int updates; @Override public Object allocateUntethered(Buffer originator, int size) { diff --git a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java index bebe1ba..29ffc9f 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java @@ -45,7 +45,7 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe private static final int DEFAULT_NUM_DIRECT_ARENA; private static final int DEFAULT_PAGE_SIZE; - private static final int DEFAULT_MAX_ORDER; // 8192 << 11 = 16 MiB per chunk + private static final int DEFAULT_MAX_ORDER; // 8192 << 9 = 4 MiB per chunk private static final int DEFAULT_SMALL_CACHE_SIZE; private static final int DEFAULT_NORMAL_CACHE_SIZE; static final int DEFAULT_MAX_CACHED_BUFFER_CAPACITY; @@ -75,7 +75,7 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe DEFAULT_PAGE_SIZE = defaultPageSize; DEFAULT_DIRECT_MEMORY_CACHE_ALIGNMENT = defaultAlignment; - int defaultMaxOrder = SystemPropertyUtil.getInt("io.netty.allocator.maxOrder", 11); + int defaultMaxOrder = SystemPropertyUtil.getInt("io.netty.allocator.maxOrder", 9); Throwable maxOrderFallbackCause = null; try { validateAndCalculateChunkSize(DEFAULT_PAGE_SIZE, defaultMaxOrder); @@ -284,7 +284,10 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe @Override public Buffer allocate(int size) { - return allocate(new PooledAllocatorControl(), size).order(ByteOrder.nativeOrder()); + if (size < 1) { + throw new IllegalArgumentException("Allocation size must be positive, but was " + size + '.'); + } + return allocate(new PooledAllocatorControl(), size); } Buffer allocate(PooledAllocatorControl control, int size) { @@ -292,7 +295,7 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe PoolArena arena = cache.arena; if (arena != null) { - return arena.allocate(control, cache, size); + return arena.allocate(control, cache, size).fill((byte) 0).order(ByteOrder.nativeOrder()); } BufferAllocator unpooled = manager.isNative()? BufferAllocator.direct() : BufferAllocator.heap(); return unpooled.allocate(size); @@ -300,9 +303,11 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe @Override public void close() { + trimCurrentThreadCache(); for (PoolArena arena : arenas) { arena.close(); } + threadCache.remove(); } /** diff --git a/src/main/java/io/netty/buffer/api/pool/PooledDrop.java b/src/main/java/io/netty/buffer/api/pool/PooledDrop.java new file mode 100644 index 0000000..efa50af --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/PooledDrop.java @@ -0,0 +1,55 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package io.netty.buffer.api.pool; + +import io.netty.buffer.api.Buffer; +import io.netty.buffer.api.Drop; + +class PooledDrop implements Drop { + private final PooledAllocatorControl control; + private PoolArena arena; + private PoolChunk chunk; + private PoolThreadCache threadCache; + private long handle; + private int normSize; + + PooledDrop(PooledAllocatorControl control, PoolArena arena, PoolChunk chunk, PoolThreadCache threadCache, + long handle, int normSize) { + this.control = control; + this.arena = arena; + this.chunk = chunk; + this.threadCache = threadCache; + this.handle = handle; + this.normSize = normSize; + } + + @Override + public void drop(Buffer obj) { + arena.free(chunk, handle, normSize, threadCache); + } + + @Override + public void attach(Buffer obj) { + if (control.updates > 0) { + arena = control.arena; + chunk = control.chunk; + threadCache = control.threadCache; + handle = control.handle; + normSize = control.normSize; + control.updates = 0; + } + } +} From fa75c81c6cd450a17515a4e40ca210dcdc7d5f3c Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Wed, 12 May 2021 16:39:27 +0200 Subject: [PATCH 09/13] Fix checkstyle issues --- .../io/netty/buffer/api/pool/PoolChunk.java | 3 ++- .../netty/buffer/api/pool/package-info.java | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 src/main/java/io/netty/buffer/api/pool/package-info.java diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java index 4e8753c..81aef2a 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java @@ -168,7 +168,8 @@ final class PoolChunk implements PoolChunkMetric { PoolChunk prev; PoolChunk next; - PoolChunk(PoolArena arena, Buffer base, Object memory, int pageSize, int pageShifts, int chunkSize, int maxPageIdx) { + PoolChunk(PoolArena arena, Buffer base, Object memory, int pageSize, int pageShifts, int chunkSize, + int maxPageIdx) { this.arena = arena; this.base = base; this.memory = memory; diff --git a/src/main/java/io/netty/buffer/api/pool/package-info.java b/src/main/java/io/netty/buffer/api/pool/package-info.java new file mode 100644 index 0000000..ce60c3b --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +/** + * A pooling {@link io.netty.buffer.api.BufferAllocator} implementation based on jemalloc. + */ +package io.netty.buffer.api.pool; From 12b38234e5265fd7869dd19f21a5d8bb5fe55a6d Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Mon, 17 May 2021 15:15:19 +0200 Subject: [PATCH 10/13] Make sure that every allocation get their own unique Drop instance. This allows the pooling allocator to precisely control how each allocation should be dropped. This is important to the pooling allocator, because it needs to know what arena, chunk, page, run, etc. is being freed, exactly. --- .../io/netty/buffer/api/AllocatorControl.java | 14 +++- .../buffer/api/ManagedBufferAllocator.java | 15 +++- .../io/netty/buffer/api/MemoryManager.java | 3 +- .../buffer/api/SizeClassedMemoryPool.java | 24 ++++-- .../bytebuffer/ByteBufferMemoryManager.java | 8 +- .../buffer/api/bytebuffer/NioBuffer.java | 12 +-- .../memseg/AbstractMemorySegmentManager.java | 8 +- .../netty/buffer/api/memseg/MemSegBuffer.java | 29 +++---- .../io/netty/buffer/api/pool/PoolArena.java | 75 +++++++++---------- .../io/netty/buffer/api/pool/PoolChunk.java | 52 +++++++++++-- .../netty/buffer/api/pool/PoolChunkList.java | 13 ++-- .../buffer/api/pool/PoolThreadCache.java | 40 +++++----- .../api/pool/PooledAllocatorControl.java | 5 +- .../api/pool/PooledBufferAllocator.java | 23 ++++-- .../api/pool/UnpooledUnthetheredMemory.java | 44 +++++++++++ .../netty/buffer/api/unsafe/UnsafeBuffer.java | 11 +-- .../api/unsafe/UnsafeMemoryManager.java | 7 +- 17 files changed, 245 insertions(+), 138 deletions(-) create mode 100644 src/main/java/io/netty/buffer/api/pool/UnpooledUnthetheredMemory.java diff --git a/src/main/java/io/netty/buffer/api/AllocatorControl.java b/src/main/java/io/netty/buffer/api/AllocatorControl.java index 35fa502..7010e12 100644 --- a/src/main/java/io/netty/buffer/api/AllocatorControl.java +++ b/src/main/java/io/netty/buffer/api/AllocatorControl.java @@ -22,7 +22,7 @@ package io.netty.buffer.api; */ public interface AllocatorControl { /** - * Allocate a buffer that is not tethered to any particular {@link Drop} implementation, + * Allocate a buffer that is not tethered to any particular {@link Buffer} object, * and return the recoverable memory object from it. *

* This allows a buffer to implement {@link Buffer#ensureWritable(int)} by having new memory allocated to it, @@ -30,9 +30,9 @@ public interface AllocatorControl { * * @param originator The buffer that originated the request for an untethered memory allocated. * @param size The size of the requested memory allocation, in bytes. - * @return A "recoverable memory" object that is the requested allocation. + * @return A {@link UntetheredMemory} object that is the requested allocation. */ - Object allocateUntethered(Buffer originator, int size); + UntetheredMemory allocateUntethered(Buffer originator, int size); /** * Return memory to the allocator, after it has been untethered from it's lifetime. @@ -42,4 +42,12 @@ public interface AllocatorControl { * @param memory The untethered memory to return to the allocator. */ void recoverMemory(Object memory); + + /** + * Memory that isn't attached to any particular buffer. + */ + interface UntetheredMemory { + Memory memory(); + Drop drop(); + } } diff --git a/src/main/java/io/netty/buffer/api/ManagedBufferAllocator.java b/src/main/java/io/netty/buffer/api/ManagedBufferAllocator.java index 09ba957..a132772 100644 --- a/src/main/java/io/netty/buffer/api/ManagedBufferAllocator.java +++ b/src/main/java/io/netty/buffer/api/ManagedBufferAllocator.java @@ -41,11 +41,22 @@ class ManagedBufferAllocator implements BufferAllocator, AllocatorControl { return () -> manager.allocateConstChild(constantBuffer); } + @SuppressWarnings("unchecked") @Override - public Object allocateUntethered(Buffer originator, int size) { + public UntetheredMemory allocateUntethered(Buffer originator, int size) { BufferAllocator.checkSize(size); var buf = manager.allocateShared(this, size, NO_OP_DROP, Statics.CLEANER); - return manager.unwrapRecoverableMemory(buf); + return new UntetheredMemory() { + @Override + public Memory memory() { + return (Memory) manager.unwrapRecoverableMemory(buf); + } + + @Override + public Drop drop() { + return (Drop) manager.drop(); + } + }; } @Override diff --git a/src/main/java/io/netty/buffer/api/MemoryManager.java b/src/main/java/io/netty/buffer/api/MemoryManager.java index 81e72cf..730b0ca 100644 --- a/src/main/java/io/netty/buffer/api/MemoryManager.java +++ b/src/main/java/io/netty/buffer/api/MemoryManager.java @@ -27,6 +27,5 @@ public interface MemoryManager { void discardRecoverableMemory(Object recoverableMemory); // todo should recoverMemory re-attach a cleaner? Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemory, Drop drop); - Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemoryBase, - int offset, int length, Drop drop); + Object sliceMemory(Object memory, int offset, int length); } diff --git a/src/main/java/io/netty/buffer/api/SizeClassedMemoryPool.java b/src/main/java/io/netty/buffer/api/SizeClassedMemoryPool.java index a759639..8a8095b 100644 --- a/src/main/java/io/netty/buffer/api/SizeClassedMemoryPool.java +++ b/src/main/java/io/netty/buffer/api/SizeClassedMemoryPool.java @@ -118,15 +118,29 @@ class SizeClassedMemoryPool implements BufferAllocator, AllocatorControl, Drop Memory memory() { + return (Memory) memory; + } + + @Override + public Drop drop() { + return (Drop) getDrop(); + } + }; } @Override diff --git a/src/main/java/io/netty/buffer/api/bytebuffer/ByteBufferMemoryManager.java b/src/main/java/io/netty/buffer/api/bytebuffer/ByteBufferMemoryManager.java index 26c3017..bd7bfea 100644 --- a/src/main/java/io/netty/buffer/api/bytebuffer/ByteBufferMemoryManager.java +++ b/src/main/java/io/netty/buffer/api/bytebuffer/ByteBufferMemoryManager.java @@ -82,10 +82,8 @@ public class ByteBufferMemoryManager implements MemoryManager { } @Override - public Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemoryBase, - int offset, int length, Drop drop) { - ByteBuffer memory = (ByteBuffer) recoverableMemoryBase; - memory = memory.slice(offset, length); - return new NioBuffer(memory, memory, allocatorControl, convert(drop)); + public Object sliceMemory(Object memory, int offset, int length) { + var buffer = (ByteBuffer) memory; + return buffer.slice(offset, length); } } diff --git a/src/main/java/io/netty/buffer/api/bytebuffer/NioBuffer.java b/src/main/java/io/netty/buffer/api/bytebuffer/NioBuffer.java index e96f2d0..4364958 100644 --- a/src/main/java/io/netty/buffer/api/bytebuffer/NioBuffer.java +++ b/src/main/java/io/netty/buffer/api/bytebuffer/NioBuffer.java @@ -408,27 +408,27 @@ class NioBuffer extends RcSupport implements Buffer, Readable // Allocate a bigger buffer. long newSize = capacity() + (long) Math.max(size - writableBytes(), minimumGrowth); BufferAllocator.checkSize(newSize); - ByteBuffer buffer = (ByteBuffer) control.allocateUntethered(this, (int) newSize); + var untethered = control.allocateUntethered(this, (int) newSize); + ByteBuffer buffer = untethered.memory(); buffer.order(order()); // Copy contents. copyInto(0, buffer, 0, capacity()); // Release the old memory and install the new: - Drop drop = disconnectDrop(); + Drop drop = untethered.drop(); + disconnectDrop(drop); attachNewBuffer(buffer, drop); } - private Drop disconnectDrop() { + private void disconnectDrop(Drop newDrop) { var drop = (Drop) unsafeGetDrop(); int roff = this.roff; int woff = this.woff; drop.drop(this); - drop = ArcDrop.unwrapAllArcs(drop); - unsafeSetDrop(new ArcDrop<>(drop)); + unsafeSetDrop(new ArcDrop<>(newDrop)); this.roff = roff; this.woff = woff; - return drop; } private void attachNewBuffer(ByteBuffer buffer, Drop drop) { diff --git a/src/main/java/io/netty/buffer/api/memseg/AbstractMemorySegmentManager.java b/src/main/java/io/netty/buffer/api/memseg/AbstractMemorySegmentManager.java index 2cc7c82..1aabd96 100644 --- a/src/main/java/io/netty/buffer/api/memseg/AbstractMemorySegmentManager.java +++ b/src/main/java/io/netty/buffer/api/memseg/AbstractMemorySegmentManager.java @@ -75,10 +75,8 @@ public abstract class AbstractMemorySegmentManager implements MemoryManager { } @Override - public Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemoryBase, - int offset, int length, Drop drop) { - var segment = (MemorySegment) recoverableMemoryBase; - segment = segment.asSlice(offset, length); - return new MemSegBuffer(segment, segment, convert(ArcDrop.acquire(drop)), allocatorControl); + public Object sliceMemory(Object memory, int offset, int length) { + var segment = (MemorySegment) memory; + return segment.asSlice(offset, length); } } diff --git a/src/main/java/io/netty/buffer/api/memseg/MemSegBuffer.java b/src/main/java/io/netty/buffer/api/memseg/MemSegBuffer.java index 6e31e04..426f99b 100644 --- a/src/main/java/io/netty/buffer/api/memseg/MemSegBuffer.java +++ b/src/main/java/io/netty/buffer/api/memseg/MemSegBuffer.java @@ -524,32 +524,27 @@ class MemSegBuffer extends RcSupport implements Buffer, Re // Allocate a bigger buffer. long newSize = capacity() + (long) Math.max(size - writableBytes(), minimumGrowth); BufferAllocator.checkSize(newSize); - MemorySegment newSegment = (MemorySegment) control.allocateUntethered(this, (int) newSize); + var untethered = control.allocateUntethered(this, (int) newSize); + MemorySegment newSegment = untethered.memory(); // Copy contents. newSegment.copyFrom(seg); // Release the old memory segment and install the new one: - Drop drop = disconnectDrop(); + Drop drop = untethered.drop(); + disconnectDrop(drop); attachNewMemorySegment(newSegment, drop); } - private Drop disconnectDrop() { + private void disconnectDrop(Drop newDrop) { var drop = unsafeGetDrop(); - if (drop instanceof ArcDrop) { - // Disconnect from the current arc drop, since we'll get our own fresh memory segment. - int roff = this.roff; - int woff = this.woff; - drop.drop(this); - drop = ArcDrop.unwrapAllArcs(drop); - unsafeSetDrop(new ArcDrop<>(drop)); - this.roff = roff; - this.woff = woff; - } else { - // TODO would we ever get here? - control.recoverMemory(recoverableMemory()); - } - return drop; + // Disconnect from the current arc drop, since we'll get our own fresh memory segment. + int roff = this.roff; + int woff = this.woff; + drop.drop(this); + unsafeSetDrop(new ArcDrop<>(newDrop)); + this.roff = roff; + this.woff = woff; } private void attachNewMemorySegment(MemorySegment newSegment, Drop drop) { diff --git a/src/main/java/io/netty/buffer/api/pool/PoolArena.java b/src/main/java/io/netty/buffer/api/pool/PoolArena.java index 7362887..c4c97eb 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolArena.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolArena.java @@ -114,7 +114,7 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl return manager.isNative(); } - Buffer allocate(PooledAllocatorControl control, PoolThreadCache cache, int size) { + UntetheredMemory allocate(PooledAllocatorControl control, PoolThreadCache cache, int size) { final int sizeIdx = size2SizeIdx(size); if (sizeIdx <= smallMaxSizeIdx) { @@ -129,12 +129,12 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl } } - private Buffer tcacheAllocateSmall(PooledAllocatorControl control, PoolThreadCache cache, final int size, - final int sizeIdx) { - Buffer buffer = cache.allocateSmall(control, size, sizeIdx); - if (buffer != null) { + private UntetheredMemory tcacheAllocateSmall(PooledAllocatorControl control, PoolThreadCache cache, final int size, + final int sizeIdx) { + UntetheredMemory memory = cache.allocateSmall(control, size, sizeIdx); + if (memory != null) { // was able to allocate out of the cache so move on - return buffer; + return memory; } /* @@ -150,73 +150,72 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl assert s.doNotDestroy && s.elemSize == sizeIdx2size(sizeIdx); long handle = s.allocate(); assert handle >= 0; - buffer = s.chunk.allocateBufferWithSubpage(handle, size, cache, control); + memory = s.chunk.allocateBufferWithSubpage(handle, size, cache, control); } } if (needsNormalAllocation) { synchronized (this) { - buffer = allocateNormal(size, sizeIdx, cache, control); + memory = allocateNormal(size, sizeIdx, cache, control); } } incSmallAllocation(); - return buffer; + return memory; } - private Buffer tcacheAllocateNormal(PooledAllocatorControl control, PoolThreadCache cache, int size, int sizeIdx) { - Buffer buffer = cache.allocateNormal(this, control, size, sizeIdx); - if (buffer != null) { + private UntetheredMemory tcacheAllocateNormal(PooledAllocatorControl control, PoolThreadCache cache, int size, int sizeIdx) { + UntetheredMemory memory = cache.allocateNormal(this, control, size, sizeIdx); + if (memory != null) { // was able to allocate out of the cache so move on - return buffer; + return memory; } synchronized (this) { - buffer = allocateNormal(size, sizeIdx, cache, control); + memory = allocateNormal(size, sizeIdx, cache, control); allocationsNormal++; } - return buffer; + return memory; } // Method must be called inside synchronized(this) { ... } block - private Buffer allocateNormal(int size, int sizeIdx, PoolThreadCache threadCache, PooledAllocatorControl control) { - Buffer buffer = q050.allocate(size, sizeIdx, threadCache, control); - if (buffer != null) { - return buffer; + private UntetheredMemory allocateNormal(int size, int sizeIdx, PoolThreadCache threadCache, PooledAllocatorControl control) { + UntetheredMemory memory = q050.allocate(size, sizeIdx, threadCache, control); + if (memory != null) { + return memory; } - buffer = q025.allocate(size, sizeIdx, threadCache, control); - if (buffer != null) { - return buffer; + memory = q025.allocate(size, sizeIdx, threadCache, control); + if (memory != null) { + return memory; } - buffer = q000.allocate(size, sizeIdx, threadCache, control); - if (buffer != null) { - return buffer; + memory = q000.allocate(size, sizeIdx, threadCache, control); + if (memory != null) { + return memory; } - buffer = qInit.allocate(size, sizeIdx, threadCache, control); - if (buffer != null) { - return buffer; + memory = qInit.allocate(size, sizeIdx, threadCache, control); + if (memory != null) { + return memory; } - buffer = q075.allocate(size, sizeIdx, threadCache, control); - if (buffer != null) { - return buffer; + memory = q075.allocate(size, sizeIdx, threadCache, control); + if (memory != null) { + return memory; } // Add a new chunk. PoolChunk c = newChunk(pageSize, nPSizes, pageShifts, chunkSize); - buffer = c.allocate(size, sizeIdx, threadCache, control); - assert buffer != null; + memory = c.allocate(size, sizeIdx, threadCache, control); + assert memory != null; qInit.add(c); - return buffer; + return memory; } private void incSmallAllocation() { allocationsSmall.increment(); } - private Buffer allocateHuge(int size) { + private UntetheredMemory allocateHuge(int size) { activeBytesHuge.add(size); allocationsHuge.increment(); - BufferAllocator allocator = isDirect()? BufferAllocator.direct() : BufferAllocator.heap(); - return allocator.allocate(size); + return new UnpooledUnthetheredMemory(manager, size); } void free(PoolChunk chunk, long handle, int normCapacity, PoolThreadCache cache) { @@ -258,7 +257,7 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl } @Override - public Object allocateUntethered(Buffer originator, int size) { + public UntetheredMemory allocateUntethered(Buffer originator, int size) { throw new AssertionError("PoolChunk base buffers should never need to reallocate."); } diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java index 81aef2a..e09292e 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java @@ -15,7 +15,9 @@ */ package io.netty.buffer.api.pool; +import io.netty.buffer.api.AllocatorControl.UntetheredMemory; import io.netty.buffer.api.Buffer; +import io.netty.buffer.api.Drop; import java.util.PriorityQueue; @@ -262,7 +264,7 @@ final class PoolChunk implements PoolChunkMetric { return 100 - freePercentage; } - Buffer allocate(int size, int sizeIdx, PoolThreadCache cache, PooledAllocatorControl control) { + UntetheredMemory allocate(int size, int sizeIdx, PoolThreadCache cache, PooledAllocatorControl control) { final long handle; if (sizeIdx <= arena.smallMaxSizeIdx) { // small @@ -513,21 +515,21 @@ final class PoolChunk implements PoolChunkMetric { | (long) inUsed << IS_USED_SHIFT; } - Buffer allocateBuffer(long handle, int size, PoolThreadCache threadCache, PooledAllocatorControl control) { + UntetheredMemory allocateBuffer(long handle, int size, PoolThreadCache threadCache, PooledAllocatorControl control) { if (isRun(handle)) { int offset = runOffset(handle) << pageShifts; int maxLength = runSize(pageShifts, handle); PoolThreadCache poolThreadCache = arena.parent.threadCache(); initAllocatorControl(control, poolThreadCache, handle, maxLength); - return arena.manager.recoverMemory(control, memory, offset, size, - new PooledDrop(control, arena, this, poolThreadCache, handle, maxLength)); + return new UntetheredChunkAllocation( + memory, control, this, poolThreadCache, handle, maxLength, offset, size); } else { return allocateBufferWithSubpage(handle, size, threadCache, control); } } - Buffer allocateBufferWithSubpage(long handle, int size, PoolThreadCache threadCache, - PooledAllocatorControl control) { + UntetheredMemory allocateBufferWithSubpage(long handle, int size, PoolThreadCache threadCache, + PooledAllocatorControl control) { int runOffset = runOffset(handle); int bitmapIdx = bitmapIdx(handle); @@ -537,8 +539,42 @@ final class PoolChunk implements PoolChunkMetric { int offset = (runOffset << pageShifts) + bitmapIdx * s.elemSize; initAllocatorControl(control, threadCache, handle, s.elemSize); - return arena.manager.recoverMemory(control, memory, offset, size, - new PooledDrop(control, arena, this, threadCache, handle, s.elemSize)); + return new UntetheredChunkAllocation(memory, control, this, threadCache, handle, s.elemSize, offset, size); + } + + @SuppressWarnings("unchecked") + private static class UntetheredChunkAllocation implements UntetheredMemory { + private final Object memory; + private final PooledAllocatorControl control; + private final PoolChunk chunk; + private final PoolThreadCache threadCache; + private final long handle; + private final int maxLength; + private final int offset; + private final int size; + + private UntetheredChunkAllocation( + Object memory, PooledAllocatorControl control, PoolChunk chunk, PoolThreadCache threadCache, + long handle, int maxLength, int offset, int size) { + this.memory = memory; + this.control = control; + this.chunk = chunk; + this.threadCache = threadCache; + this.handle = handle; + this.maxLength = maxLength; + this.offset = offset; + this.size = size; + } + + @Override + public Memory memory() { + return (Memory) chunk.arena.manager.sliceMemory(memory, offset, size); + } + + @Override + public Drop drop() { + return (Drop) new PooledDrop(control, chunk.arena, chunk, threadCache, handle, maxLength); + } } private void initAllocatorControl(PooledAllocatorControl control, PoolThreadCache threadCache, long handle, diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java b/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java index 6fe5bae..03347f6 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunkList.java @@ -15,7 +15,7 @@ */ package io.netty.buffer.api.pool; -import io.netty.buffer.api.Buffer; +import io.netty.buffer.api.AllocatorControl.UntetheredMemory; import io.netty.util.internal.StringUtil; import java.util.ArrayList; @@ -23,7 +23,8 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; -import static java.lang.Math.*; +import static java.lang.Math.max; +import static java.lang.Math.min; final class PoolChunkList implements PoolChunkListMetric { private static final Iterator EMPTY_METRICS = Collections.emptyIterator(); @@ -91,7 +92,7 @@ final class PoolChunkList implements PoolChunkListMetric { this.prevList = prevList; } - Buffer allocate(int size, int sizeIdx, PoolThreadCache threadCache, PooledAllocatorControl control) { + UntetheredMemory allocate(int size, int sizeIdx, PoolThreadCache threadCache, PooledAllocatorControl control) { int normCapacity = arena.sizeIdx2size(sizeIdx); if (normCapacity > maxCapacity) { // Either this PoolChunkList is empty, or the requested capacity is larger than the capacity which can @@ -100,13 +101,13 @@ final class PoolChunkList implements PoolChunkListMetric { } for (PoolChunk cur = head; cur != null; cur = cur.next) { - Buffer buffer = cur.allocate(size, sizeIdx, threadCache, control); - if (buffer != null) { + UntetheredMemory memory = cur.allocate(size, sizeIdx, threadCache, control); + if (memory != null) { if (cur.freeBytes <= freeMinThreshold) { remove(cur); nextList.add(cur); } - return buffer; + return memory; } } return null; diff --git a/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java b/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java index c74399d..14dc67d 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java @@ -15,10 +15,7 @@ */ package io.netty.buffer.api.pool; -import static io.netty.buffer.api.pool.PoolArena.SizeClass.*; -import static io.netty.util.internal.ObjectUtil.checkPositiveOrZero; - -import io.netty.buffer.api.Buffer; +import io.netty.buffer.api.AllocatorControl.UntetheredMemory; import io.netty.buffer.api.pool.PoolArena.SizeClass; import io.netty.util.internal.MathUtil; import io.netty.util.internal.ObjectPool; @@ -31,6 +28,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Queue; +import static io.netty.buffer.api.pool.PoolArena.SizeClass.Normal; +import static io.netty.buffer.api.pool.PoolArena.SizeClass.Small; +import static io.netty.util.internal.ObjectUtil.checkPositiveOrZero; + /** * Acts a Thread cache for allocations. This implementation is modelled after * jemalloc and the described @@ -121,23 +122,23 @@ final class PoolThreadCache { /** * Try to allocate a small buffer out of the cache. Returns {@code true} if successful {@code false} otherwise */ - Buffer allocateSmall(PooledAllocatorControl control, int size, int sizeIdx) { + UntetheredMemory allocateSmall(PooledAllocatorControl control, int size, int sizeIdx) { return allocate(cacheForSmall(sizeIdx), control, size); } /** * Try to allocate a normal buffer out of the cache. Returns {@code true} if successful {@code false} otherwise */ - Buffer allocateNormal(PoolArena area, PooledAllocatorControl control, int size, int sizeIdx) { + UntetheredMemory allocateNormal(PoolArena area, PooledAllocatorControl control, int size, int sizeIdx) { return allocate(cacheForNormal(area, sizeIdx), control, size); } - private Buffer allocate(MemoryRegionCache cache, PooledAllocatorControl control, int size) { + private UntetheredMemory allocate(MemoryRegionCache cache, PooledAllocatorControl control, int size) { if (cache == null) { // no cache found so just return false here return null; } - Buffer allocated = cache.allocate(size, this, control); + UntetheredMemory allocated = cache.allocate(size, this, control); if (++allocations >= freeSweepAllocationThreshold) { allocations = 0; trim(); @@ -160,14 +161,13 @@ final class PoolThreadCache { } private MemoryRegionCache cache(PoolArena area, int sizeIdx, SizeClass sizeClass) { - switch (sizeClass) { - case Normal: + if (sizeClass == Normal) { return cacheForNormal(area, sizeIdx); - case Small: - return cacheForSmall(sizeIdx); - default: - throw new Error(); } + if (sizeClass == Small) { + return cacheForSmall(sizeIdx); + } + throw new AssertionError("Unexpected size class: " + sizeClass); } /** @@ -252,7 +252,7 @@ final class PoolThreadCache { } @Override - protected Buffer allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache, + protected UntetheredMemory allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache, PooledAllocatorControl control) { return chunk.allocateBufferWithSubpage(handle, size, threadCache, control); } @@ -267,7 +267,7 @@ final class PoolThreadCache { } @Override - protected Buffer allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache, + protected UntetheredMemory allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache, PooledAllocatorControl control) { return chunk.allocateBuffer(handle, size, threadCache, control); } @@ -286,9 +286,9 @@ final class PoolThreadCache { } /** - * Allocate a new {@link Buffer} using the provided chunk and handle with the capacity restrictions. + * Allocate a new {@link UntetheredMemory} using the provided chunk and handle with the capacity restrictions. */ - protected abstract Buffer allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache, + protected abstract UntetheredMemory allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache, PooledAllocatorControl control); /** @@ -308,12 +308,12 @@ final class PoolThreadCache { /** * Allocate something out of the cache if possible and remove the entry from the cache. */ - public final Buffer allocate(int size, PoolThreadCache threadCache, PooledAllocatorControl control) { + public final UntetheredMemory allocate(int size, PoolThreadCache threadCache, PooledAllocatorControl control) { Entry entry = queue.poll(); if (entry == null) { return null; } - Buffer buffer = allocBuf(entry.chunk, entry.handle, size, threadCache, control); + UntetheredMemory buffer = allocBuf(entry.chunk, entry.handle, size, threadCache, control); entry.recycle(); // allocations are not thread-safe which is fine as this is only called from the same thread all time. diff --git a/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java b/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java index a4e53ca..87917da 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java @@ -27,9 +27,8 @@ class PooledAllocatorControl implements AllocatorControl { public int updates; @Override - public Object allocateUntethered(Buffer originator, int size) { - Buffer allocate = arena.parent.allocate(this, size); - return arena.manager.unwrapRecoverableMemory(allocate); + public UntetheredMemory allocateUntethered(Buffer originator, int size) { + return arena.parent.allocate(this, size); } @Override diff --git a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java index 29ffc9f..458318c 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java @@ -15,9 +15,7 @@ */ package io.netty.buffer.api.pool; -import static io.netty.util.internal.ObjectUtil.checkPositiveOrZero; -import static java.util.Objects.requireNonNull; - +import io.netty.buffer.api.AllocatorControl.UntetheredMemory; import io.netty.buffer.api.Buffer; import io.netty.buffer.api.BufferAllocator; import io.netty.buffer.api.MemoryManager; @@ -38,6 +36,9 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; +import static io.netty.util.internal.ObjectUtil.checkPositiveOrZero; +import static java.util.Objects.requireNonNull; + public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMetricProvider { private static final InternalLogger logger = InternalLoggerFactory.getInstance(PooledBufferAllocator.class); @@ -287,18 +288,24 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe if (size < 1) { throw new IllegalArgumentException("Allocation size must be positive, but was " + size + '.'); } - return allocate(new PooledAllocatorControl(), size); + PooledAllocatorControl control = new PooledAllocatorControl(); + UntetheredMemory memory = allocate(control, size); + Buffer buffer = manager.recoverMemory(control, memory.memory(), memory.drop()); + return buffer.fill((byte) 0).order(ByteOrder.nativeOrder()); } - Buffer allocate(PooledAllocatorControl control, int size) { + UntetheredMemory allocate(PooledAllocatorControl control, int size) { PoolThreadCache cache = threadCache.get(); PoolArena arena = cache.arena; if (arena != null) { - return arena.allocate(control, cache, size).fill((byte) 0).order(ByteOrder.nativeOrder()); + return arena.allocate(control, cache, size); } - BufferAllocator unpooled = manager.isNative()? BufferAllocator.direct() : BufferAllocator.heap(); - return unpooled.allocate(size); + return allocateUnpooled(size); + } + + private UntetheredMemory allocateUnpooled(int size) { + return new UnpooledUnthetheredMemory(manager, size); } @Override diff --git a/src/main/java/io/netty/buffer/api/pool/UnpooledUnthetheredMemory.java b/src/main/java/io/netty/buffer/api/pool/UnpooledUnthetheredMemory.java new file mode 100644 index 0000000..0f537c8 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/pool/UnpooledUnthetheredMemory.java @@ -0,0 +1,44 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package io.netty.buffer.api.pool; + +import io.netty.buffer.api.AllocatorControl; +import io.netty.buffer.api.Buffer; +import io.netty.buffer.api.Drop; +import io.netty.buffer.api.MemoryManager; +import io.netty.buffer.api.internal.Statics; + +@SuppressWarnings("unchecked") +class UnpooledUnthetheredMemory implements AllocatorControl.UntetheredMemory { + private final MemoryManager manager; + private final Buffer buffer; + + UnpooledUnthetheredMemory(MemoryManager manager, int size) { + this.manager = manager; + PooledAllocatorControl allocatorControl = new PooledAllocatorControl(); + buffer = manager.allocateShared(allocatorControl, size, manager.drop(), Statics.CLEANER); + } + + @Override + public Memory memory() { + return (Memory) manager.unwrapRecoverableMemory(buffer); + } + + @Override + public Drop drop() { + return (Drop) manager.drop(); + } +} diff --git a/src/main/java/io/netty/buffer/api/unsafe/UnsafeBuffer.java b/src/main/java/io/netty/buffer/api/unsafe/UnsafeBuffer.java index e4b72aa..bdbe06c 100644 --- a/src/main/java/io/netty/buffer/api/unsafe/UnsafeBuffer.java +++ b/src/main/java/io/netty/buffer/api/unsafe/UnsafeBuffer.java @@ -447,7 +447,8 @@ class UnsafeBuffer extends RcSupport implements Buffer, Re // Allocate a bigger buffer. long newSize = capacity() + (long) Math.max(size - writableBytes(), minimumGrowth); BufferAllocator.checkSize(newSize); - UnsafeMemory memory = (UnsafeMemory) control.allocateUntethered(this, (int) newSize); + var untethered = control.allocateUntethered(this, (int) newSize); + UnsafeMemory memory = untethered.memory(); // Copy contents. try { @@ -458,17 +459,17 @@ class UnsafeBuffer extends RcSupport implements Buffer, Re } // Release the old memory, and install the new memory: - Drop drop = disconnectDrop(); + Drop drop = untethered.drop(); + disconnectDrop(drop); attachNewMemory(memory, drop); } - private Drop disconnectDrop() { + private Drop disconnectDrop(Drop newDrop) { var drop = (Drop) unsafeGetDrop(); int roff = this.roff; int woff = this.woff; drop.drop(this); - drop = ArcDrop.unwrapAllArcs(drop); - unsafeSetDrop(new ArcDrop<>(drop)); + unsafeSetDrop(new ArcDrop<>(newDrop)); this.roff = roff; this.woff = woff; return drop; diff --git a/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemoryManager.java b/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemoryManager.java index 1298795..bb51202 100644 --- a/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemoryManager.java +++ b/src/main/java/io/netty/buffer/api/unsafe/UnsafeMemoryManager.java @@ -96,10 +96,7 @@ public class UnsafeMemoryManager implements MemoryManager { } @Override - public Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemoryBase, - int offset, int length, Drop drop) { - UnsafeMemory memory = (UnsafeMemory) recoverableMemoryBase; - memory = memory.slice(offset, length); - return new UnsafeBuffer(memory, 0, memory.size, allocatorControl, convert(drop)); + public Object sliceMemory(Object memory, int offset, int length) { + return ((UnsafeMemory) memory).slice(offset, length); } } From 03743fca0d7f4bc3f96463248b5144fbd94e0069 Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Mon, 17 May 2021 16:22:33 +0200 Subject: [PATCH 11/13] Pooling allocator cleanups and checkstyle fixes --- .../io/netty/buffer/api/pool/PoolArena.java | 28 ++++++++----------- .../io/netty/buffer/api/pool/PoolChunk.java | 16 +++++------ .../buffer/api/pool/PoolThreadCache.java | 4 +-- .../api/pool/PooledAllocatorControl.java | 4 +-- .../api/pool/PooledBufferAllocator.java | 5 ++-- .../io/netty/buffer/api/pool/PooledDrop.java | 27 ++++-------------- .../api/pool/UnpooledUnthetheredMemory.java | 3 +- 7 files changed, 33 insertions(+), 54 deletions(-) diff --git a/src/main/java/io/netty/buffer/api/pool/PoolArena.java b/src/main/java/io/netty/buffer/api/pool/PoolArena.java index c4c97eb..b3e87f5 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolArena.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolArena.java @@ -17,7 +17,6 @@ package io.netty.buffer.api.pool; import io.netty.buffer.api.AllocatorControl; import io.netty.buffer.api.Buffer; -import io.netty.buffer.api.BufferAllocator; import io.netty.buffer.api.MemoryManager; import io.netty.buffer.api.internal.Statics; import io.netty.util.internal.StringUtil; @@ -110,10 +109,6 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl return new PoolSubpage[size]; } - boolean isDirect() { - return manager.isNative(); - } - UntetheredMemory allocate(PooledAllocatorControl control, PoolThreadCache cache, int size) { final int sizeIdx = size2SizeIdx(size); @@ -164,7 +159,8 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl return memory; } - private UntetheredMemory tcacheAllocateNormal(PooledAllocatorControl control, PoolThreadCache cache, int size, int sizeIdx) { + private UntetheredMemory tcacheAllocateNormal( + PooledAllocatorControl control, PoolThreadCache cache, int size, int sizeIdx) { UntetheredMemory memory = cache.allocateNormal(this, control, size, sizeIdx); if (memory != null) { // was able to allocate out of the cache so move on @@ -178,7 +174,8 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl } // Method must be called inside synchronized(this) { ... } block - private UntetheredMemory allocateNormal(int size, int sizeIdx, PoolThreadCache threadCache, PooledAllocatorControl control) { + private UntetheredMemory allocateNormal( + int size, int sizeIdx, PoolThreadCache threadCache, PooledAllocatorControl control) { UntetheredMemory memory = q050.allocate(size, sizeIdx, threadCache, control); if (memory != null) { return memory; @@ -215,7 +212,7 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl private UntetheredMemory allocateHuge(int size) { activeBytesHuge.add(size); allocationsHuge.increment(); - return new UnpooledUnthetheredMemory(manager, size); + return new UnpooledUnthetheredMemory(parent, manager, size); } void free(PoolChunk chunk, long handle, int normCapacity, PoolThreadCache cache) { @@ -234,15 +231,12 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl void freeChunk(PoolChunk chunk, long handle, int normCapacity, SizeClass sizeClass) { final boolean destroyChunk; synchronized (this) { - switch (sizeClass) { - case Normal: - ++deallocationsNormal; - break; - case Small: - ++deallocationsSmall; - break; - default: - throw new Error(); + if (sizeClass == SizeClass.Normal) { + ++deallocationsNormal; + } else if (sizeClass == SizeClass.Small) { + ++deallocationsSmall; + } else { + throw new AssertionError("Unexpected size class: " + sizeClass); } destroyChunk = !chunk.parent.free(chunk, handle, normCapacity); } diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java index e09292e..0d0edc3 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java @@ -515,14 +515,15 @@ final class PoolChunk implements PoolChunkMetric { | (long) inUsed << IS_USED_SHIFT; } - UntetheredMemory allocateBuffer(long handle, int size, PoolThreadCache threadCache, PooledAllocatorControl control) { + UntetheredMemory allocateBuffer(long handle, int size, PoolThreadCache threadCache, + PooledAllocatorControl control) { if (isRun(handle)) { int offset = runOffset(handle) << pageShifts; int maxLength = runSize(pageShifts, handle); PoolThreadCache poolThreadCache = arena.parent.threadCache(); initAllocatorControl(control, poolThreadCache, handle, maxLength); return new UntetheredChunkAllocation( - memory, control, this, poolThreadCache, handle, maxLength, offset, size); + memory, this, poolThreadCache, handle, maxLength, offset, size); } else { return allocateBufferWithSubpage(handle, size, threadCache, control); } @@ -539,13 +540,12 @@ final class PoolChunk implements PoolChunkMetric { int offset = (runOffset << pageShifts) + bitmapIdx * s.elemSize; initAllocatorControl(control, threadCache, handle, s.elemSize); - return new UntetheredChunkAllocation(memory, control, this, threadCache, handle, s.elemSize, offset, size); + return new UntetheredChunkAllocation(memory, this, threadCache, handle, s.elemSize, offset, size); } @SuppressWarnings("unchecked") - private static class UntetheredChunkAllocation implements UntetheredMemory { + private static final class UntetheredChunkAllocation implements UntetheredMemory { private final Object memory; - private final PooledAllocatorControl control; private final PoolChunk chunk; private final PoolThreadCache threadCache; private final long handle; @@ -554,10 +554,9 @@ final class PoolChunk implements PoolChunkMetric { private final int size; private UntetheredChunkAllocation( - Object memory, PooledAllocatorControl control, PoolChunk chunk, PoolThreadCache threadCache, + Object memory, PoolChunk chunk, PoolThreadCache threadCache, long handle, int maxLength, int offset, int size) { this.memory = memory; - this.control = control; this.chunk = chunk; this.threadCache = threadCache; this.handle = handle; @@ -573,7 +572,7 @@ final class PoolChunk implements PoolChunkMetric { @Override public Drop drop() { - return (Drop) new PooledDrop(control, chunk.arena, chunk, threadCache, handle, maxLength); + return (Drop) new PooledDrop(chunk.arena, chunk, threadCache, handle, maxLength); } } @@ -584,7 +583,6 @@ final class PoolChunk implements PoolChunkMetric { control.threadCache = threadCache; control.handle = handle; control.normSize = normSize; - control.updates++; } @Override diff --git a/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java b/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java index 14dc67d..2771b7f 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolThreadCache.java @@ -288,8 +288,8 @@ final class PoolThreadCache { /** * Allocate a new {@link UntetheredMemory} using the provided chunk and handle with the capacity restrictions. */ - protected abstract UntetheredMemory allocBuf(PoolChunk chunk, long handle, int size, PoolThreadCache threadCache, - PooledAllocatorControl control); + protected abstract UntetheredMemory allocBuf( + PoolChunk chunk, long handle, int size, PoolThreadCache threadCache, PooledAllocatorControl control); /** * Add to cache if not already full. diff --git a/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java b/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java index 87917da..2e177c8 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledAllocatorControl.java @@ -19,16 +19,16 @@ import io.netty.buffer.api.AllocatorControl; import io.netty.buffer.api.Buffer; class PooledAllocatorControl implements AllocatorControl { + public PooledBufferAllocator parent; public PoolArena arena; public PoolChunk chunk; public PoolThreadCache threadCache; public long handle; public int normSize; - public int updates; @Override public UntetheredMemory allocateUntethered(Buffer originator, int size) { - return arena.parent.allocate(this, size); + return parent.allocate(this, size); } @Override diff --git a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java index 458318c..211dc2b 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java @@ -289,6 +289,7 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe throw new IllegalArgumentException("Allocation size must be positive, but was " + size + '.'); } PooledAllocatorControl control = new PooledAllocatorControl(); + control.parent = this; UntetheredMemory memory = allocate(control, size); Buffer buffer = manager.recoverMemory(control, memory.memory(), memory.drop()); return buffer.fill((byte) 0).order(ByteOrder.nativeOrder()); @@ -305,7 +306,7 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe } private UntetheredMemory allocateUnpooled(int size) { - return new UnpooledUnthetheredMemory(manager, size); + return new UnpooledUnthetheredMemory(this, manager, size); } @Override @@ -423,7 +424,7 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe threadCache.free(); } - private PoolArena leastUsedArena(PoolArena[] arenas) { + private static PoolArena leastUsedArena(PoolArena[] arenas) { if (arenas == null || arenas.length == 0) { return null; } diff --git a/src/main/java/io/netty/buffer/api/pool/PooledDrop.java b/src/main/java/io/netty/buffer/api/pool/PooledDrop.java index efa50af..a3191a2 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledDrop.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledDrop.java @@ -19,16 +19,13 @@ import io.netty.buffer.api.Buffer; import io.netty.buffer.api.Drop; class PooledDrop implements Drop { - private final PooledAllocatorControl control; - private PoolArena arena; - private PoolChunk chunk; - private PoolThreadCache threadCache; - private long handle; - private int normSize; + private final PoolArena arena; + private final PoolChunk chunk; + private final PoolThreadCache threadCache; + private final long handle; + private final int normSize; - PooledDrop(PooledAllocatorControl control, PoolArena arena, PoolChunk chunk, PoolThreadCache threadCache, - long handle, int normSize) { - this.control = control; + PooledDrop(PoolArena arena, PoolChunk chunk, PoolThreadCache threadCache, long handle, int normSize) { this.arena = arena; this.chunk = chunk; this.threadCache = threadCache; @@ -40,16 +37,4 @@ class PooledDrop implements Drop { public void drop(Buffer obj) { arena.free(chunk, handle, normSize, threadCache); } - - @Override - public void attach(Buffer obj) { - if (control.updates > 0) { - arena = control.arena; - chunk = control.chunk; - threadCache = control.threadCache; - handle = control.handle; - normSize = control.normSize; - control.updates = 0; - } - } } diff --git a/src/main/java/io/netty/buffer/api/pool/UnpooledUnthetheredMemory.java b/src/main/java/io/netty/buffer/api/pool/UnpooledUnthetheredMemory.java index 0f537c8..475087f 100644 --- a/src/main/java/io/netty/buffer/api/pool/UnpooledUnthetheredMemory.java +++ b/src/main/java/io/netty/buffer/api/pool/UnpooledUnthetheredMemory.java @@ -26,9 +26,10 @@ class UnpooledUnthetheredMemory implements AllocatorControl.UntetheredMemory { private final MemoryManager manager; private final Buffer buffer; - UnpooledUnthetheredMemory(MemoryManager manager, int size) { + UnpooledUnthetheredMemory(PooledBufferAllocator allocator, MemoryManager manager, int size) { this.manager = manager; PooledAllocatorControl allocatorControl = new PooledAllocatorControl(); + allocatorControl.parent = allocator; buffer = manager.allocateShared(allocatorControl, size, manager.drop(), Statics.CLEANER); } From dec3756e6df7af15ad318be9b7574f4cdf1aeddf Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Mon, 17 May 2021 18:08:16 +0200 Subject: [PATCH 12/13] Buffers from the pooling allocator must be able to return memory to the pool if the buffer objects are leaked. --- .../io/netty/buffer/api/AllocatorControl.java | 9 +++ .../io/netty/buffer/api/BufferAllocator.java | 3 +- .../buffer/api/internal/CleanerDrop.java | 81 +++++++++++++++++++ .../io/netty/buffer/api/pool/PoolChunk.java | 4 +- .../netty/buffer/api/BufferCleanerTest.java | 23 +++++- .../netty/buffer/api/BufferTestSupport.java | 6 +- 6 files changed, 118 insertions(+), 8 deletions(-) create mode 100644 src/main/java/io/netty/buffer/api/internal/CleanerDrop.java diff --git a/src/main/java/io/netty/buffer/api/AllocatorControl.java b/src/main/java/io/netty/buffer/api/AllocatorControl.java index 7010e12..7cf7ca8 100644 --- a/src/main/java/io/netty/buffer/api/AllocatorControl.java +++ b/src/main/java/io/netty/buffer/api/AllocatorControl.java @@ -47,7 +47,16 @@ public interface AllocatorControl { * Memory that isn't attached to any particular buffer. */ interface UntetheredMemory { + /** + * Produce the recoverable memory object associated with this piece of untethered memory. + * @implNote This method should only be called once, since it might be expensive. + */ Memory memory(); + + /** + * Produce the drop instance associated with this piece of untethered memory. + * @implNote This method should only be called once, since it might be expensive, or interact with Cleaners. + */ Drop drop(); } } diff --git a/src/main/java/io/netty/buffer/api/BufferAllocator.java b/src/main/java/io/netty/buffer/api/BufferAllocator.java index 079dbf8..9b24050 100644 --- a/src/main/java/io/netty/buffer/api/BufferAllocator.java +++ b/src/main/java/io/netty/buffer/api/BufferAllocator.java @@ -130,6 +130,7 @@ public interface BufferAllocator extends AutoCloseable { } static BufferAllocator pooledDirect() { - return new SizeClassedMemoryPool(MemoryManagers.getManagers().getNativeMemoryManager()); + return new PooledBufferAllocator(MemoryManagers.getManagers().getNativeMemoryManager()); +// return new SizeClassedMemoryPool(MemoryManagers.getManagers().getNativeMemoryManager()); } } diff --git a/src/main/java/io/netty/buffer/api/internal/CleanerDrop.java b/src/main/java/io/netty/buffer/api/internal/CleanerDrop.java new file mode 100644 index 0000000..225a067 --- /dev/null +++ b/src/main/java/io/netty/buffer/api/internal/CleanerDrop.java @@ -0,0 +1,81 @@ +/* + * Copyright 2021 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package io.netty.buffer.api.internal; + +import io.netty.buffer.api.Drop; + +import java.io.Serial; +import java.lang.ref.Cleaner; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A drop implementation that delegates to another drop instance, either when called directly, or when it becomes + * cleanable. This ensures that objects are dropped even if they leak. + */ +public final class CleanerDrop implements Drop { + private Cleaner.Cleanable cleanable; + private GatedRunner runner; + + /** + * Wrap the given drop instance, and produce a new drop instance that will also call the delegate drop instance if + * it becomes cleanable. + */ + public static Drop wrap(Drop drop) { + CleanerDrop cleanerDrop = new CleanerDrop<>(); + GatedRunner runner = new GatedRunner<>(drop); + cleanerDrop.cleanable = Statics.CLEANER.register(cleanerDrop, runner); + cleanerDrop.runner = runner; + return cleanerDrop; + } + + private CleanerDrop() { + } + + @Override + public void attach(T obj) { + runner.set(obj); + runner.drop.attach(obj); + } + + @Override + public void drop(T obj) { + attach(obj); + cleanable.clean(); + } + + @Override + public String toString() { + return "CleanerDrop(" + runner.drop + ')'; + } + + private static final class GatedRunner extends AtomicReference implements Runnable { + @Serial + private static final long serialVersionUID = 2685535951915798850L; + final Drop drop; + + private GatedRunner(Drop drop) { + this.drop = drop; + } + + @Override + public void run() { + T obj = getAndSet(null); // Make absolutely sure we only delegate once. + if (obj != null) { + drop.drop(obj); + } + } + } +} diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java index 0d0edc3..8e14ba0 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java @@ -18,6 +18,7 @@ package io.netty.buffer.api.pool; import io.netty.buffer.api.AllocatorControl.UntetheredMemory; import io.netty.buffer.api.Buffer; import io.netty.buffer.api.Drop; +import io.netty.buffer.api.internal.CleanerDrop; import java.util.PriorityQueue; @@ -572,7 +573,8 @@ final class PoolChunk implements PoolChunkMetric { @Override public Drop drop() { - return (Drop) new PooledDrop(chunk.arena, chunk, threadCache, handle, maxLength); + PooledDrop pooledDrop = new PooledDrop(chunk.arena, chunk, threadCache, handle, maxLength); + return (Drop) CleanerDrop.wrap(pooledDrop); } } diff --git a/src/test/java/io/netty/buffer/api/BufferCleanerTest.java b/src/test/java/io/netty/buffer/api/BufferCleanerTest.java index b83eec5..7785a89 100644 --- a/src/test/java/io/netty/buffer/api/BufferCleanerTest.java +++ b/src/test/java/io/netty/buffer/api/BufferCleanerTest.java @@ -19,11 +19,29 @@ import io.netty.buffer.api.memseg.NativeMemorySegmentManager; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import java.util.List; +import java.util.stream.Stream; + +import static io.netty.buffer.api.MemoryManagers.using; import static org.assertj.core.api.Assertions.assertThat; public class BufferCleanerTest extends BufferTestSupport { + @SuppressWarnings("OptionalGetWithoutIsPresent") + static Fixture[] memorySegmentAllocators() { + MemoryManagers managers = MemoryManagers.getAllManagers() + .map(p -> p.get()) + .filter(mm -> "MS".equals(mm.toString())) + .findFirst().get(); + List initFixtures = initialAllocators().stream().flatMap(f -> { + Stream.Builder builder = Stream.builder(); + builder.add(new Fixture(f + "/" + managers, () -> using(managers, f), f.getProperties())); + return builder.build(); + }).toList(); + return fixtureCombinations(initFixtures).filter(f -> f.isDirect()).toArray(Fixture[]::new); + } + @ParameterizedTest - @MethodSource("directAllocators") + @MethodSource("memorySegmentAllocators") public void bufferMustBeClosedByCleaner(Fixture fixture) throws InterruptedException { var initial = NativeMemorySegmentManager.MEM_USAGE_NATIVE.sum(); int allocationSize = 1024; @@ -34,6 +52,7 @@ public class BufferCleanerTest extends BufferTestSupport { System.runFinalization(); sum = NativeMemorySegmentManager.MEM_USAGE_NATIVE.sum() - initial; if (sum < allocationSize) { + // The memory must have been cleaned. return; } } @@ -42,7 +61,7 @@ public class BufferCleanerTest extends BufferTestSupport { private static void allocateAndForget(Fixture fixture, int size) { var allocator = fixture.createAllocator(); - allocator.close(); allocator.allocate(size); + allocator.close(); } } diff --git a/src/test/java/io/netty/buffer/api/BufferTestSupport.java b/src/test/java/io/netty/buffer/api/BufferTestSupport.java index 20a8671..184d698 100644 --- a/src/test/java/io/netty/buffer/api/BufferTestSupport.java +++ b/src/test/java/io/netty/buffer/api/BufferTestSupport.java @@ -59,7 +59,7 @@ public abstract class BufferTestSupport { private static final Memoize INITIAL_NO_CONST = new Memoize<>( () -> initialFixturesForEachImplementation().stream().filter(f -> !f.isConst()).toArray(Fixture[]::new)); private static final Memoize ALL_COMBINATIONS = new Memoize<>( - () -> fixtureCombinations().toArray(Fixture[]::new)); + () -> fixtureCombinations(initialFixturesForEachImplementation()).toArray(Fixture[]::new)); private static final Memoize ALL_ALLOCATORS = new Memoize<>( () -> Arrays.stream(ALL_COMBINATIONS.get()) .filter(sample()) @@ -160,9 +160,7 @@ public abstract class BufferTestSupport { return initFixtures; } - private static Stream fixtureCombinations() { - List initFixtures = initialFixturesForEachImplementation(); - + static Stream fixtureCombinations(List initFixtures) { Builder builder = Stream.builder(); initFixtures.forEach(builder); From 0105e5231da9b152661e170c680cdac2b9c1e2b3 Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Tue, 18 May 2021 18:20:32 +0200 Subject: [PATCH 13/13] Remove the SizeClassedMemoryPool implementation And fix the remaining test failures for the PooledBufferAllocator. The PooledBufferAllocator now also keeps its chunks alive, even after closing the pool, as long as there are allocated buffers that refer to the memory. The pool now clears all of its relevant internal references when closed, allowing the GC to reclaim all of the pooled memory, assuming no allocated buffers remain. --- .../io/netty/buffer/api/BufferAllocator.java | 2 - .../netty/buffer/api/CleanerPooledDrop.java | 115 ------------- .../buffer/api/SizeClassedMemoryPool.java | 162 ------------------ .../io/netty/buffer/api/internal/ArcDrop.java | 12 +- .../io/netty/buffer/api/pool/PoolArena.java | 12 +- .../buffer/api/pool/PoolArenaMetric.java | 8 - .../io/netty/buffer/api/pool/PoolChunk.java | 17 +- .../api/pool/PooledBufferAllocator.java | 22 ++- .../netty/buffer/api/BufferTestSupport.java | 64 +++++-- 9 files changed, 81 insertions(+), 333 deletions(-) delete mode 100644 src/main/java/io/netty/buffer/api/CleanerPooledDrop.java delete mode 100644 src/main/java/io/netty/buffer/api/SizeClassedMemoryPool.java diff --git a/src/main/java/io/netty/buffer/api/BufferAllocator.java b/src/main/java/io/netty/buffer/api/BufferAllocator.java index 9b24050..184c154 100644 --- a/src/main/java/io/netty/buffer/api/BufferAllocator.java +++ b/src/main/java/io/netty/buffer/api/BufferAllocator.java @@ -126,11 +126,9 @@ public interface BufferAllocator extends AutoCloseable { static BufferAllocator pooledHeap() { return new PooledBufferAllocator(MemoryManagers.getManagers().getHeapMemoryManager()); -// return new SizeClassedMemoryPool(MemoryManagers.getManagers().getHeapMemoryManager()); } static BufferAllocator pooledDirect() { return new PooledBufferAllocator(MemoryManagers.getManagers().getNativeMemoryManager()); -// return new SizeClassedMemoryPool(MemoryManagers.getManagers().getNativeMemoryManager()); } } diff --git a/src/main/java/io/netty/buffer/api/CleanerPooledDrop.java b/src/main/java/io/netty/buffer/api/CleanerPooledDrop.java deleted file mode 100644 index 35a0609..0000000 --- a/src/main/java/io/netty/buffer/api/CleanerPooledDrop.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2020 The Netty Project - * - * The Netty Project licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package io.netty.buffer.api; - -import java.lang.invoke.VarHandle; -import java.lang.ref.Cleaner.Cleanable; -import java.lang.ref.WeakReference; -import java.util.concurrent.atomic.AtomicBoolean; - -import static io.netty.buffer.api.internal.Statics.CLEANER; -import static io.netty.buffer.api.internal.Statics.findVarHandle; -import static java.lang.invoke.MethodHandles.lookup; - -class CleanerPooledDrop implements Drop { - private static final VarHandle CLEANABLE = - findVarHandle(lookup(), CleanerPooledDrop.class, "cleanable", GatedCleanable.class); - private final SizeClassedMemoryPool pool; - private final MemoryManager manager; - private final Drop delegate; - @SuppressWarnings("unused") - private volatile GatedCleanable cleanable; - - CleanerPooledDrop(SizeClassedMemoryPool pool, MemoryManager manager, - Drop delegate) { - this.pool = pool; - this.manager = manager; - this.delegate = delegate; - } - - @Override - public void drop(Buffer buf) { - GatedCleanable c = (GatedCleanable) CLEANABLE.getAndSet(this, null); - if (c != null) { - c.clean(); - delegate.drop(buf); - } - } - - @Override - public void attach(Buffer buf) { - // Unregister old cleanable, if any, to avoid uncontrolled build-up. - GatedCleanable c = (GatedCleanable) CLEANABLE.getAndSet(this, null); - if (c != null) { - c.disable(); - c.clean(); - } - - var mem = manager.unwrapRecoverableMemory(buf); - WeakReference ref = new WeakReference<>(this); - AtomicBoolean gate = new AtomicBoolean(true); - cleanable = new GatedCleanable(gate, CLEANER.register(this, new CleanAction(pool, mem, ref, gate))); - } - - @Override - public String toString() { - return "CleanerPooledDrop(" + delegate + ')'; - } - - private static final class CleanAction implements Runnable { - private final SizeClassedMemoryPool pool; - private final Object mem; - private final WeakReference ref; - private final AtomicBoolean gate; - - private CleanAction(SizeClassedMemoryPool pool, Object mem, WeakReference ref, - AtomicBoolean gate) { - this.pool = pool; - this.mem = mem; - this.ref = ref; - this.gate = gate; - } - - @Override - public void run() { - if (gate.getAndSet(false)) { - var monitored = ref.get(); - if (monitored == null) { - pool.recoverMemory(mem); - } - } - } - } - - private static final class GatedCleanable implements Cleanable { - private final AtomicBoolean gate; - private final Cleanable cleanable; - - GatedCleanable(AtomicBoolean gate, Cleanable cleanable) { - this.gate = gate; - this.cleanable = cleanable; - } - - public void disable() { - gate.set(false); - } - - @Override - public void clean() { - cleanable.clean(); - } - } -} diff --git a/src/main/java/io/netty/buffer/api/SizeClassedMemoryPool.java b/src/main/java/io/netty/buffer/api/SizeClassedMemoryPool.java deleted file mode 100644 index 8a8095b..0000000 --- a/src/main/java/io/netty/buffer/api/SizeClassedMemoryPool.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright 2020 The Netty Project - * - * The Netty Project licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package io.netty.buffer.api; - -import io.netty.buffer.api.internal.Statics; - -import java.lang.invoke.VarHandle; -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.function.Supplier; - -import static io.netty.buffer.api.internal.Statics.NO_OP_DROP; -import static java.lang.invoke.MethodHandles.lookup; - -class SizeClassedMemoryPool implements BufferAllocator, AllocatorControl, Drop { - private static final VarHandle CLOSE = Statics.findVarHandle( - lookup(), SizeClassedMemoryPool.class, "closed", boolean.class); - private final MemoryManager manager; - private final ConcurrentHashMap> pool; - @SuppressWarnings("unused") - private volatile boolean closed; - - protected SizeClassedMemoryPool(MemoryManager manager) { - this.manager = manager; - pool = new ConcurrentHashMap<>(); - } - - @Override - public Buffer allocate(int size) { - BufferAllocator.checkSize(size); - var sizeClassPool = getSizeClassPool(size); - Object memory = sizeClassPool.poll(); - if (memory != null) { - return recoverMemoryIntoBuffer(memory) - .fill((byte) 0) - .order(ByteOrder.nativeOrder()); - } - return createBuf(size, getDrop()); - } - - @Override - public Supplier constBufferSupplier(byte[] bytes) { - Buffer constantBuffer = manager.allocateShared(this, bytes.length, manager.drop(), Statics.CLEANER); - constantBuffer.writeBytes(bytes).makeReadOnly(); - return () -> manager.allocateConstChild(constantBuffer); - } - - protected MemoryManager getMemoryManager() { - return manager; - } - - protected Buffer createBuf(int size, Drop drop) { - var buf = manager.allocateShared(this, size, drop, null); - drop.attach(buf); - return buf; - } - - protected Drop getDrop() { - return new CleanerPooledDrop(this, getMemoryManager(), this); - } - - @Override - public void close() { - if (CLOSE.compareAndSet(this, false, true)) { - var capturedExceptions = new ArrayList(4); - pool.forEach((k, v) -> { - Object memory; - while ((memory = v.poll()) != null) { - try { - manager.discardRecoverableMemory(memory); - } catch (Exception e) { - capturedExceptions.add(e); - } - } - }); - if (!capturedExceptions.isEmpty()) { - var exception = new ResourceDisposeFailedException(); - capturedExceptions.forEach(exception::addSuppressed); - throw exception; - } - } - } - - @Override - public void drop(Buffer buf) { - if (closed) { - manager.drop().drop(buf); - return; - } - Object mem = manager.unwrapRecoverableMemory(buf); - var sizeClassPool = getSizeClassPool(manager.capacityOfRecoverableMemory(mem)); - sizeClassPool.offer(mem); - if (closed) { - Object memory; - while ((memory = sizeClassPool.poll()) != null) { - manager.discardRecoverableMemory(memory); - } - } - } - - @Override - public String toString() { - return "SizeClassedMemoryPool"; - } - - @SuppressWarnings("unchecked") - @Override - public UntetheredMemory allocateUntethered(Buffer originator, int size) { - var sizeClassPool = getSizeClassPool(size); - Object candidateMemory = sizeClassPool.poll(); - if (candidateMemory == null) { - Buffer untetheredBuf = createBuf(size, NO_OP_DROP); - candidateMemory = manager.unwrapRecoverableMemory(untetheredBuf); - } - Object memory = candidateMemory; - - return new UntetheredMemory() { - - @Override - public Memory memory() { - return (Memory) memory; - } - - @Override - public Drop drop() { - return (Drop) getDrop(); - } - }; - } - - @Override - public void recoverMemory(Object memory) { - Buffer buf = recoverMemoryIntoBuffer(memory); - buf.close(); - } - - private Buffer recoverMemoryIntoBuffer(Object memory) { - var drop = getDrop(); - var buf = manager.recoverMemory(this, memory, drop); - drop.attach(buf); - return buf; - } - - private ConcurrentLinkedQueue getSizeClassPool(int size) { - return pool.computeIfAbsent(size, k -> new ConcurrentLinkedQueue<>()); - } -} diff --git a/src/main/java/io/netty/buffer/api/internal/ArcDrop.java b/src/main/java/io/netty/buffer/api/internal/ArcDrop.java index 3f98b68..b2acd10 100644 --- a/src/main/java/io/netty/buffer/api/internal/ArcDrop.java +++ b/src/main/java/io/netty/buffer/api/internal/ArcDrop.java @@ -46,13 +46,6 @@ public final class ArcDrop implements Drop { return new ArcDrop(drop); } - public static Drop unwrapAllArcs(Drop drop) { - while (drop instanceof ArcDrop) { - drop = ((ArcDrop) drop).unwrap(); - } - return drop; - } - public static Drop acquire(Drop drop) { if (drop.getClass() == ArcDrop.class) { ((ArcDrop) drop).increment(); @@ -103,7 +96,10 @@ public final class ArcDrop implements Drop { @Override public String toString() { - StringBuilder builder = new StringBuilder().append("ArcDrop(").append(count).append(", "); + StringBuilder builder = new StringBuilder() + .append("ArcDrop@") + .append(Integer.toHexString(System.identityHashCode(this))) + .append('(').append(count).append(", "); Drop drop = this; while ((drop = ((ArcDrop) drop).unwrap()) instanceof ArcDrop) { builder.append(((ArcDrop) drop).count).append(", "); diff --git a/src/main/java/io/netty/buffer/api/pool/PoolArena.java b/src/main/java/io/netty/buffer/api/pool/PoolArena.java index b3e87f5..5d5c085 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolArena.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolArena.java @@ -18,7 +18,6 @@ package io.netty.buffer.api.pool; import io.netty.buffer.api.AllocatorControl; import io.netty.buffer.api.Buffer; import io.netty.buffer.api.MemoryManager; -import io.netty.buffer.api.internal.Statics; import io.netty.util.internal.StringUtil; import java.util.ArrayList; @@ -363,7 +362,6 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl @Override public long numActiveAllocations() { - long val = allocationsSmall.longValue() + allocationsHuge.longValue() - deallocationsHuge.longValue(); synchronized (this) { @@ -372,11 +370,6 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl return max(val, 0); } - @Override - public long numActiveTinyAllocations() { - return 0; - } - @Override public long numActiveSmallAllocations() { return max(numSmallAllocations() - numSmallDeallocations(), 0); @@ -410,10 +403,7 @@ class PoolArena extends SizeClasses implements PoolArenaMetric, AllocatorControl } protected final PoolChunk newChunk(int pageSize, int maxPageIdx, int pageShifts, int chunkSize) { - Buffer base = manager.allocateShared(this, chunkSize, manager.drop(), Statics.CLEANER); - Object memory = manager.unwrapRecoverableMemory(base); - return new PoolChunk( - this, base, memory, pageSize, pageShifts, chunkSize, maxPageIdx); + return new PoolChunk(this, pageSize, pageShifts, chunkSize, maxPageIdx); } @Override diff --git a/src/main/java/io/netty/buffer/api/pool/PoolArenaMetric.java b/src/main/java/io/netty/buffer/api/pool/PoolArenaMetric.java index bc4d51e..c5c2f26 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolArenaMetric.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolArenaMetric.java @@ -108,14 +108,6 @@ public interface PoolArenaMetric extends SizeClassesMetric { */ long numActiveAllocations(); - /** - * Return the number of currently active tiny allocations. - * - * @deprecated Tiny allocations have been merged into small allocations. - */ - @Deprecated - long numActiveTinyAllocations(); - /** * Return the number of currently active small allocations. */ diff --git a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java index 8e14ba0..78b07ed 100644 --- a/src/main/java/io/netty/buffer/api/pool/PoolChunk.java +++ b/src/main/java/io/netty/buffer/api/pool/PoolChunk.java @@ -18,7 +18,10 @@ package io.netty.buffer.api.pool; import io.netty.buffer.api.AllocatorControl.UntetheredMemory; import io.netty.buffer.api.Buffer; import io.netty.buffer.api.Drop; +import io.netty.buffer.api.MemoryManager; +import io.netty.buffer.api.internal.ArcDrop; import io.netty.buffer.api.internal.CleanerDrop; +import io.netty.buffer.api.internal.Statics; import java.util.PriorityQueue; @@ -145,6 +148,7 @@ final class PoolChunk implements PoolChunkMetric { final PoolArena arena; final Buffer base; // The buffer that is the source of the memory. Closing it will free the memory. final Object memory; + final Drop baseDrop; // An ArcDrop that manages references to the base Buffer. /** * store the first page and last page of each avail run @@ -171,11 +175,13 @@ final class PoolChunk implements PoolChunkMetric { PoolChunk prev; PoolChunk next; - PoolChunk(PoolArena arena, Buffer base, Object memory, int pageSize, int pageShifts, int chunkSize, + PoolChunk(PoolArena arena, int pageSize, int pageShifts, int chunkSize, int maxPageIdx) { this.arena = arena; - this.base = base; - this.memory = memory; + MemoryManager manager = arena.manager; + base = manager.allocateShared(arena, chunkSize, manager.drop(), Statics.CLEANER); + memory = manager.unwrapRecoverableMemory(base); + baseDrop = ArcDrop.wrap(Buffer::close); this.pageSize = pageSize; this.pageShifts = pageShifts; this.chunkSize = chunkSize; @@ -419,6 +425,7 @@ final class PoolChunk implements PoolChunkMetric { * @param handle handle to free */ void free(long handle, int normCapacity) { + baseDrop.drop(base); // Decrement reference count. if (isSubpage(handle)) { int sizeIdx = arena.size2SizeIdx(normCapacity); PoolSubpage head = arena.findSubpagePoolHead(sizeIdx); @@ -523,6 +530,7 @@ final class PoolChunk implements PoolChunkMetric { int maxLength = runSize(pageShifts, handle); PoolThreadCache poolThreadCache = arena.parent.threadCache(); initAllocatorControl(control, poolThreadCache, handle, maxLength); + ArcDrop.acquire(baseDrop); return new UntetheredChunkAllocation( memory, this, poolThreadCache, handle, maxLength, offset, size); } else { @@ -541,6 +549,7 @@ final class PoolChunk implements PoolChunkMetric { int offset = (runOffset << pageShifts) + bitmapIdx * s.elemSize; initAllocatorControl(control, threadCache, handle, s.elemSize); + ArcDrop.acquire(baseDrop); return new UntetheredChunkAllocation(memory, this, threadCache, handle, s.elemSize, offset, size); } @@ -620,7 +629,7 @@ final class PoolChunk implements PoolChunkMetric { } void destroy() { - base.close(); + baseDrop.drop(base); // Decrement reference count from the chunk (allocated buffers may keep the base alive) } static int runOffset(long handle) { diff --git a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java index 211dc2b..8052f28 100644 --- a/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java +++ b/src/main/java/io/netty/buffer/api/pool/PooledBufferAllocator.java @@ -166,6 +166,7 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe private final int smallCacheSize; private final int normalCacheSize; private final List arenaMetrics; + private final List arenaMetricsView; private final PoolThreadLocalCache threadCache; private final int chunkSize; private final PooledBufferAllocatorMetric metric; @@ -235,10 +236,12 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe arenas[i] = arena; metrics.add(arena); } - arenaMetrics = Collections.unmodifiableList(metrics); + arenaMetrics = metrics; + arenaMetricsView = Collections.unmodifiableList(metrics); } else { arenas = null; - arenaMetrics = Collections.emptyList(); + arenaMetrics = new ArrayList<>(1); + arenaMetricsView = Collections.emptyList(); } metric = new PooledBufferAllocatorMetric(this); @@ -312,10 +315,15 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe @Override public void close() { trimCurrentThreadCache(); - for (PoolArena arena : arenas) { - arena.close(); - } threadCache.remove(); + for (int i = 0, arenasLength = arenas.length; i < arenasLength; i++) { + PoolArena arena = arenas[i]; + if (arena != null) { + arena.close(); + arenas[i] = null; + } + } + arenaMetrics.clear(); } /** @@ -416,7 +424,7 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe return cache; } // No caching so just use 0 as sizes. - return new PoolThreadCache(arena, 0, 0, 0, 0); + return new PoolThreadCache(null, 0, 0, 0, 0); } @Override @@ -450,7 +458,7 @@ public class PooledBufferAllocator implements BufferAllocator, BufferAllocatorMe * Return a {@link List} of all heap {@link PoolArenaMetric}s that are provided by this pool. */ List arenaMetrics() { - return arenaMetrics; + return arenaMetricsView; } /** diff --git a/src/test/java/io/netty/buffer/api/BufferTestSupport.java b/src/test/java/io/netty/buffer/api/BufferTestSupport.java index 184d698..a5a131b 100644 --- a/src/test/java/io/netty/buffer/api/BufferTestSupport.java +++ b/src/test/java/io/netty/buffer/api/BufferTestSupport.java @@ -167,12 +167,14 @@ public abstract class BufferTestSupport { // Add 2-way composite buffers of all combinations. for (Fixture first : initFixtures) { for (Fixture second : initFixtures) { - var a = first.get(); - var b = second.get(); builder.add(new Fixture("compose(" + first + ", " + second + ')', () -> { return new BufferAllocator() { + BufferAllocator a; + BufferAllocator b; @Override public Buffer allocate(int size) { + a = first.get(); + b = second.get(); int half = size / 2; try (Buffer firstHalf = a.allocate(half); Buffer secondHalf = b.allocate(size - half)) { @@ -182,8 +184,14 @@ public abstract class BufferTestSupport { @Override public void close() { - a.close(); - b.close(); + if (a != null) { + a.close(); + a = null; + } + if (b != null) { + b.close(); + b = null; + } } }; }, COMPOSITE)); @@ -193,9 +201,10 @@ public abstract class BufferTestSupport { // Also add a 3-way composite buffer. builder.add(new Fixture("compose(heap,heap,heap)", () -> { return new BufferAllocator() { - final BufferAllocator alloc = BufferAllocator.heap(); + BufferAllocator alloc; @Override public Buffer allocate(int size) { + alloc = BufferAllocator.heap(); int part = size / 3; try (Buffer a = alloc.allocate(part); Buffer b = alloc.allocate(part); @@ -206,17 +215,21 @@ public abstract class BufferTestSupport { @Override public void close() { - alloc.close(); + if (alloc != null) { + alloc.close(); + alloc = null; + } } }; }, COMPOSITE)); for (Fixture fixture : initFixtures) { builder.add(new Fixture(fixture + ".ensureWritable", () -> { - var allocator = fixture.createAllocator(); return new BufferAllocator() { + BufferAllocator allocator; @Override public Buffer allocate(int size) { + allocator = fixture.createAllocator(); if (size < 2) { return allocator.allocate(size); } @@ -227,15 +240,19 @@ public abstract class BufferTestSupport { @Override public void close() { - allocator.close(); + if (allocator != null) { + allocator.close(); + allocator = null; + } } }; }, fixture.getProperties())); builder.add(new Fixture(fixture + ".compose.ensureWritable", () -> { - var allocator = fixture.createAllocator(); return new BufferAllocator() { + BufferAllocator allocator; @Override public Buffer allocate(int size) { + allocator = fixture.createAllocator(); if (size < 2) { return allocator.allocate(size); } @@ -246,7 +263,10 @@ public abstract class BufferTestSupport { @Override public void close() { - allocator.close(); + if (allocator != null) { + allocator.close(); + allocator = null; + } } }; }, COMPOSITE)); @@ -261,10 +281,11 @@ public abstract class BufferTestSupport { Builder builder = Stream.builder(); builder.add(f); builder.add(new Fixture(f + ".split", () -> { - var allocatorBase = f.get(); return new BufferAllocator() { + BufferAllocator allocatorBase; @Override public Buffer allocate(int size) { + allocatorBase = f.get(); try (Buffer buf = allocatorBase.allocate(size + 1)) { buf.writerOffset(size); return buf.split().writerOffset(0); @@ -273,7 +294,10 @@ public abstract class BufferTestSupport { @Override public void close() { - allocatorBase.close(); + if (allocatorBase != null) { + allocatorBase.close(); + allocatorBase = null; + } } }; }, f.getProperties())); @@ -285,10 +309,11 @@ public abstract class BufferTestSupport { builder.add(f); var props = concat(f.getProperties(), Properties.SLICE); builder.add(new Fixture(f + ".slice(0, capacity())", () -> { - var allocatorBase = f.get(); return new BufferAllocator() { + BufferAllocator allocatorBase; @Override public Buffer allocate(int size) { + allocatorBase = f.get(); try (Buffer base = allocatorBase.allocate(size)) { return base.slice(0, base.capacity()).writerOffset(0); } @@ -296,15 +321,19 @@ public abstract class BufferTestSupport { @Override public void close() { - allocatorBase.close(); + if (allocatorBase != null) { + allocatorBase.close(); + allocatorBase = null; + } } }; }, props)); builder.add(new Fixture(f + ".slice(1, capacity() - 2)", () -> { - var allocatorBase = f.get(); return new BufferAllocator() { + BufferAllocator allocatorBase; @Override public Buffer allocate(int size) { + allocatorBase = f.get(); try (Buffer base = allocatorBase.allocate(size + 2)) { return base.slice(1, size).writerOffset(0); } @@ -312,7 +341,10 @@ public abstract class BufferTestSupport { @Override public void close() { - allocatorBase.close(); + if (allocatorBase != null) { + allocatorBase.close(); + allocatorBase = null; + } } }; }, props));