From 7162d96ed502f601fcaf7f16fa543caf2456b37a Mon Sep 17 00:00:00 2001 From: Trustin Lee Date: Sat, 21 Jun 2014 19:18:45 +0900 Subject: [PATCH] Revert "Improve the allocation algorithm in PoolChunk" This reverts commit 36305d7dcee60bac9d353ba12e044c260435da57, which seems to cause an assertion failure on our CI machine. --- .../main/java/io/netty/buffer/PoolChunk.java | 425 ++++++++---------- 1 file changed, 193 insertions(+), 232 deletions(-) diff --git a/buffer/src/main/java/io/netty/buffer/PoolChunk.java b/buffer/src/main/java/io/netty/buffer/PoolChunk.java index 73aebda6d0..25505d1226 100644 --- a/buffer/src/main/java/io/netty/buffer/PoolChunk.java +++ b/buffer/src/main/java/io/netty/buffer/PoolChunk.java @@ -16,124 +16,25 @@ package io.netty.buffer; -import io.netty.util.collection.IntObjectHashMap; - -/** - * Description of algorithm for PageRun/PoolSubpage allocation from PoolChunk - * - * Notation: The following terms are important to understand the code - * > page - a page is the smallest unit of memory chunk that can be allocated - * > chunk - a chunk is a collection of pages - * > in this code chunkSize = 2^{maxOrder} * pageSize - * - * To begin we allocate a byte array of size = chunkSize - * Whenever a ByteBuf of given size needs to be created we search for the first position - * in the byte array that has enough empty space to accommodate the requested size and - * return a (long) handle that encodes this offset information, (this memory segment is then - * marked as reserved so it is always used by exactly one ByteBuf and no more) - * - * For simplicity all sizes are normalized according to PoolArena#normalizeCapacity method - * This ensures that when we request for memory segments of size >= pageSize the normalizedCapacity - * equals the next nearest power of 2 - * - * To search for the first offset in chunk that has at least requested size available we construct a - * complete balanced binary tree and store it in an array (just like heaps) - memoryMap - * - * The tree looks like this (the size of each node being mentioned in the parenthesis) - * - * depth=0 1 node (chunkSize) - * depth=1 2 nodes (chunkSize/2) - * .. - * .. - * depth=d 2^d nodes (chunkSize/2^d) - * .. - * depth=maxOrder 2^maxOrder nodes (chunkSize/2^{maxOrder} = pageSize) - * - * depth=maxOrder is the last level and the leafs consist of pages - * - * With this tree available searching in chunkArray translates like this: - * To allocate a memory segment of size chunkSize/2^k we search for the first node (from left) at height k - * which is unused - * - * Algorithm: - * ---------- - * Encode the tree in memoryMap with the notation - * memoryMap[id] = x => in the subtree rooted at id, the first node that is free to be allocated - * is at depth x (counted from depth=0) i.e., at depths [depth_of_id, x), there is no node that is free - * - * As we allocate & free nodes, we update values stored in memoryMap so that the property is maintained - * - * Initialization - - * In the beginning we construct the memoryMap array by storing the depth of a node at each node - * i.e., memoryMap[id] = depth_of_id - * - * Observations: - * ------------- - * 1) memoryMap[id] = depth_of_id => it is free / unallocated - * 2) memoryMap[id] > depth_of_id => at least one of its child nodes is allocated, so we cannot allocate it, but - * some of its children can still be allocated based on their availability - * 3) memoryMap[id] = maxOrder + 1 => the node is fully allocated & thus none of its children can be allocated, it - * is thus marked as unusable - * - * Algorithm: [allocateNode(d) => we want to find the first node (from left) at height h that can be allocated] - * ---------- - * 1) start at root (i.e., depth = 0 or id = 1) - * 2) if memoryMap[1] > d => cannot be allocated from this chunk - * 3) if left node value <= h; we can allocate from left subtree so move to left and repeat until found - * 4) else try in right subtree - * - * Algorithm: [allocateRun(size)] - * ---------- - * 1) Compute d = log_2(chunkSize/size) - * 2) Return allocateNode(d) - * - * Algorithm: [allocateSubpage(size)] - * ---------- - * All subpages allocated are stored in a map at key = elemSize - * 1) if subpage at elemSize != null: try allocating from it. - * if it fails: allocateSubpageSimple - * 2) else: just allocateSubpageSimple - * - * Algorithm: [allocateSubpageSimple(size)] - * ---------- - * 1) use allocateRun(maxOrder) to find an empty (i.e., unused) leaf (i.e., page) - * 2) use this handle to construct the poolsubpage object or if it already exists just initialize it - * with required normCapacity - * 3) store (insert/ overwrite) the subpage in elemSubpages map for easier access - * - * Note: - * ----- - * In the implementation for improving cache coherence, - * we store 2 pieces of information (i.e, 2 byte vals) as a short value in memoryMap - * - * memoryMap[id]= (depth_of_id, x) - * where as per convention defined above - * the second value (i.e, x) indicates that the first node which is free to be allocated is at depth x (from root) - */ - final class PoolChunk { - - private static final int BYTE_LENGTH = 8; - private static final int BYTE_MASK = 0xFF; - private static final int INV_BYTE_MASK = ~ BYTE_MASK; + private static final int ST_UNUSED = 0; + private static final int ST_BRANCH = 1; + private static final int ST_ALLOCATED = 2; + private static final int ST_ALLOCATED_SUBPAGE = 3; final PoolArena arena; final T memory; final boolean unpooled; - private final short[] memoryMap; + private final int[] memoryMap; private final PoolSubpage[] subpages; - private final IntObjectHashMap> elemSubpages; /** Used to determine if the requested capacity is equal to or greater than pageSize. */ private final int subpageOverflowMask; private final int pageSize; private final int pageShifts; - private final int maxOrder; + private final int chunkSize; - private final int log2ChunkSize; private final int maxSubpageAllocs; - /** Used to mark memory as unusable */ - private final byte unusable; private int freeBytes; @@ -150,31 +51,25 @@ final class PoolChunk { this.memory = memory; this.pageSize = pageSize; this.pageShifts = pageShifts; - this.maxOrder = maxOrder; this.chunkSize = chunkSize; - unusable = (byte) (maxOrder + 1); - log2ChunkSize = Integer.SIZE - 1 - Integer.numberOfLeadingZeros(chunkSize); subpageOverflowMask = ~(pageSize - 1); freeBytes = chunkSize; - assert maxOrder < 30 : "maxOrder should be < 30, but is : " + maxOrder; + int chunkSizeInPages = chunkSize >>> pageShifts; maxSubpageAllocs = 1 << maxOrder; // Generate the memory map. - memoryMap = new short[maxSubpageAllocs << 1]; + memoryMap = new int[maxSubpageAllocs << 1]; int memoryMapIndex = 1; - for (int d = 0; d <= maxOrder; ++d) { // move down the tree one level at a time - short dd = (short) ((d << BYTE_LENGTH) | d); - for (int p = 0; p < (1 << d); ++p) { - // in each level traverse left to right and set the depth of subtree - // that is completely free to be my depth since I am totally free to start with - memoryMap[memoryMapIndex] = dd; - memoryMapIndex += 1; + for (int i = 0; i <= maxOrder; i ++) { + int runSizeInPages = chunkSizeInPages >>> i; + for (int j = 0; j < chunkSizeInPages; j += runSizeInPages) { + //noinspection PointlessBitwiseExpression + memoryMap[memoryMapIndex ++] = j << 17 | runSizeInPages << 2 | ST_UNUSED; } } subpages = newSubpageArray(maxSubpageAllocs); - elemSubpages = new IntObjectHashMap>(pageShifts); } /** Creates a special chunk that is not pooled. */ @@ -184,14 +79,10 @@ final class PoolChunk { this.memory = memory; memoryMap = null; subpages = null; - elemSubpages = null; subpageOverflowMask = 0; pageSize = 0; pageShifts = 0; - maxOrder = 0; - unusable = (byte) (maxOrder + 1); chunkSize = size; - log2ChunkSize = Integer.SIZE - 1 - Integer.numberOfLeadingZeros(chunkSize); maxSubpageAllocs = 0; } @@ -213,141 +104,218 @@ final class PoolChunk { } long allocate(int normCapacity) { + int firstVal = memoryMap[1]; if ((normCapacity & subpageOverflowMask) != 0) { // >= pageSize - return allocateRun(normCapacity); + return allocateRun(normCapacity, 1, firstVal); } else { - return allocateSubpage(normCapacity); + return allocateSubpage(normCapacity, 1, firstVal); } } - /** - * Update method used by allocate - * This is triggered only when a successor is allocated and all its predecessors - * need to update their state - * The minimal depth at which subtree rooted at id has some free space - * @param id id - */ - private void updateParentsAlloc(int id) { - while (id > 1) { - int parentId = id >>> 1; - byte mem1 = value(id); - byte mem2 = value(id ^ 1); - byte mem = mem1 < mem2 ? mem1 : mem2; - setVal(parentId, mem); - id = parentId; + private long allocateRun(int normCapacity, int curIdx, int val) { + switch (val & 3) { + case ST_UNUSED: + return allocateRunSimple(normCapacity, curIdx, val); + case ST_BRANCH: + // Try the right node first because it is more likely to be ST_UNUSED. + // It is because allocateRunSimple() always chooses the left node. + final int nextIdxLeft = curIdx << 1; + final int nextIdxRight = nextIdxLeft ^ 1; + final int nextValRight = memoryMap[nextIdxRight]; + final boolean recurseRight; + + switch (nextValRight & 3) { + case ST_UNUSED: + return allocateRunSimple(normCapacity, nextIdxRight, nextValRight); + case ST_BRANCH: + recurseRight = true; + break; + default: + recurseRight = false; + } + + final int nextValLeft = memoryMap[nextIdxLeft]; + final boolean recurseLeft; + + switch (nextValLeft & 3) { + case ST_UNUSED: + return allocateRunSimple(normCapacity, nextIdxLeft, nextValLeft); + case ST_BRANCH: + recurseLeft = true; + break; + default: + recurseLeft = false; + } + + if (recurseRight) { + long res = branchRun(normCapacity, nextIdxRight); + if (res > 0) { + return res; + } + } + + if (recurseLeft) { + return branchRun(normCapacity, nextIdxLeft); + } } + + return -1; } - /** - * Update method used by free - * This needs to handle the special case when both children are completely free - * in which case parent be directly allocated on request of size = child-size * 2 - * @param id id - */ - private void updateParentsFree(int id) { - int logChild = depth(id) + 1; - while (id > 1) { - int parentId = id >>> 1; - byte mem1 = value(id); - byte mem2 = value(id ^ 1); - byte mem = mem1 < mem2 ? mem1 : mem2; - setVal(parentId, mem); - logChild -= 1; // in first iteration equals log, subsequently reduce 1 from logChild as we traverse up - - if (mem1 == logChild && mem2 == logChild) { - setVal(parentId, (byte) (logChild - 1)); - } - - id = parentId; + private long branchRun(int normCapacity, int nextIdx) { + int nextNextIdx = nextIdx << 1; + int nextNextVal = memoryMap[nextNextIdx]; + long res = allocateRun(normCapacity, nextNextIdx, nextNextVal); + if (res > 0) { + return res; } + + nextNextIdx ^= 1; + nextNextVal = memoryMap[nextNextIdx]; + return allocateRun(normCapacity, nextNextIdx, nextNextVal); } - private int allocateNode(int d) { - int id = 1; - byte mem = value(id); - if (mem > d) { // unusable + private long allocateRunSimple(int normCapacity, int curIdx, int val) { + int runLength = runLength(val); + if (normCapacity > runLength) { return -1; } - while (mem < d || (id & (1 << d)) == 0) { - id = id << 1; - mem = value(id); - if (mem > d) { - id = id ^ 1; - mem = value(id); + + for (;;) { + if (normCapacity == runLength) { + // Found the run that fits. + // Note that capacity has been normalized already, so we don't need to deal with + // the values that are not power of 2. + memoryMap[curIdx] = val & ~3 | ST_ALLOCATED; + freeBytes -= runLength; + return curIdx; } + + int nextIdx = curIdx << 1; + int unusedIdx = nextIdx ^ 1; + + memoryMap[curIdx] = val & ~3 | ST_BRANCH; + //noinspection PointlessBitwiseExpression + memoryMap[unusedIdx] = memoryMap[unusedIdx] & ~3 | ST_UNUSED; + + runLength >>>= 1; + curIdx = nextIdx; + val = memoryMap[curIdx]; } - setVal(id, unusable); // mark as unusable : because, maximum input d = maxOrder - updateParentsAlloc(id); - return id; } - private long allocateRun(int normCapacity) { - int numPages = normCapacity >>> pageShifts; - int d = maxOrder - (Integer.SIZE - 1 - Integer.numberOfLeadingZeros(numPages)); - int id = allocateNode(d); - if (id < 0) { - return id; + private long allocateSubpage(int normCapacity, int curIdx, int val) { + switch (val & 3) { + case ST_UNUSED: + return allocateSubpageSimple(normCapacity, curIdx, val); + case ST_BRANCH: + // Try the right node first because it is more likely to be ST_UNUSED. + // It is because allocateSubpageSimple() always chooses the left node. + final int nextIdxLeft = curIdx << 1; + final int nextIdxRight = nextIdxLeft ^ 1; + + long res = branchSubpage(normCapacity, nextIdxRight); + if (res > 0) { + return res; + } + + return branchSubpage(normCapacity, nextIdxLeft); + case ST_ALLOCATED_SUBPAGE: + PoolSubpage subpage = subpages[subpageIdx(curIdx)]; + int elemSize = subpage.elemSize; + if (normCapacity != elemSize) { + return -1; + } + + return subpage.allocate(); } - freeBytes -= runLength(id); - return id; + + return -1; } - private long allocateSubpage(int normCapacity) { - PoolSubpage subpage = elemSubpages.get(normCapacity); - if (subpage != null) { - long handle = subpage.allocate(); - if (handle >= 0) { - return handle; + private long allocateSubpageSimple(int normCapacity, int curIdx, int val) { + int runLength = runLength(val); + for (;;) { + if (runLength == pageSize) { + memoryMap[curIdx] = val & ~3 | ST_ALLOCATED_SUBPAGE; + freeBytes -= runLength; + + int subpageIdx = subpageIdx(curIdx); + PoolSubpage subpage = subpages[subpageIdx]; + if (subpage == null) { + subpage = new PoolSubpage(this, curIdx, runOffset(val), pageSize, normCapacity); + subpages[subpageIdx] = subpage; + } else { + subpage.init(normCapacity); + } + return subpage.allocate(); } - // if subpage full (i.e., handle < 0) then replace in elemSubpage with new subpage + + int nextIdx = curIdx << 1; + int unusedIdx = nextIdx ^ 1; + + memoryMap[curIdx] = val & ~3 | ST_BRANCH; + //noinspection PointlessBitwiseExpression + memoryMap[unusedIdx] = memoryMap[unusedIdx] & ~3 | ST_UNUSED; + + runLength >>>= 1; + curIdx = nextIdx; + val = memoryMap[curIdx]; } - return allocateSubpageSimple(normCapacity); } - private long allocateSubpageSimple(int normCapacity) { - int d = maxOrder; // subpages are only be allocated from pages i.e., leaves - int id = allocateNode(d); - if (id < 0) { - return id; + private long branchSubpage(int normCapacity, int nextIdx) { + int nextVal = memoryMap[nextIdx]; + if ((nextVal & 3) != ST_ALLOCATED) { + return allocateSubpage(normCapacity, nextIdx, nextVal); } - freeBytes -= pageSize; - - int subpageIdx = subpageIdx(id); - PoolSubpage subpage = subpages[subpageIdx]; - if (subpage == null) { - subpage = new PoolSubpage(this, id, runOffset(id), pageSize, normCapacity); - subpages[subpageIdx] = subpage; - } else { - subpage.init(normCapacity); - } - elemSubpages.put(normCapacity, subpage); // store subpage at proper elemSize pos - return subpage.allocate(); + return -1; } void free(long handle) { int memoryMapIdx = (int) handle; int bitmapIdx = (int) (handle >>> 32); - if (bitmapIdx != 0) { // free a subpage + int val = memoryMap[memoryMapIdx]; + int state = val & 3; + if (state == ST_ALLOCATED_SUBPAGE) { + assert bitmapIdx != 0; PoolSubpage subpage = subpages[subpageIdx(memoryMapIdx)]; assert subpage != null && subpage.doNotDestroy; if (subpage.free(bitmapIdx & 0x3FFFFFFF)) { return; } + } else { + assert state == ST_ALLOCATED : "state: " + state; + assert bitmapIdx == 0; } - freeBytes += runLength(memoryMapIdx); - setVal(memoryMapIdx, depth(memoryMapIdx)); - updateParentsFree(memoryMapIdx); + freeBytes += runLength(val); + + for (;;) { + //noinspection PointlessBitwiseExpression + memoryMap[memoryMapIdx] = val & ~3 | ST_UNUSED; + if (memoryMapIdx == 1) { + assert freeBytes == chunkSize; + return; + } + + if ((memoryMap[siblingIdx(memoryMapIdx)] & 3) != ST_UNUSED) { + break; + } + + memoryMapIdx = parentIdx(memoryMapIdx); + val = memoryMap[memoryMapIdx]; + } } void initBuf(PooledByteBuf buf, long handle, int reqCapacity) { int memoryMapIdx = (int) handle; int bitmapIdx = (int) (handle >>> 32); if (bitmapIdx == 0) { - byte val = value(memoryMapIdx); - assert val == (maxOrder + 1) : String.valueOf(val); - buf.init(this, handle, runOffset(memoryMapIdx), reqCapacity, runLength(memoryMapIdx)); + int val = memoryMap[memoryMapIdx]; + assert (val & 3) == ST_ALLOCATED : String.valueOf(val & 3); + buf.init(this, handle, runOffset(val), reqCapacity, runLength(val)); } else { initBufWithSubpage(buf, handle, bitmapIdx, reqCapacity); } @@ -361,45 +329,38 @@ final class PoolChunk { assert bitmapIdx != 0; int memoryMapIdx = (int) handle; + int val = memoryMap[memoryMapIdx]; + assert (val & 3) == ST_ALLOCATED_SUBPAGE; PoolSubpage subpage = subpages[subpageIdx(memoryMapIdx)]; assert subpage.doNotDestroy; assert reqCapacity <= subpage.elemSize; buf.init( - this, handle, - runOffset(memoryMapIdx) + (bitmapIdx & 0x3FFFFFFF) * subpage.elemSize, reqCapacity, subpage.elemSize); + this, handle, + runOffset(val) + (bitmapIdx & 0x3FFFFFFF) * subpage.elemSize, reqCapacity, subpage.elemSize); } - private byte value(int id) { - return (byte) (memoryMap[id] & BYTE_MASK); + private static int parentIdx(int memoryMapIdx) { + return memoryMapIdx >>> 1; } - private void setVal(int id, byte val) { - memoryMap[id] = (short) ((memoryMap[id] & INV_BYTE_MASK) | val); + private static int siblingIdx(int memoryMapIdx) { + return memoryMapIdx ^ 1; } - private byte depth(int id) { - short val = memoryMap[id]; - return (byte) (val >>> BYTE_LENGTH); + private int runLength(int val) { + return (val >>> 2 & 0x7FFF) << pageShifts; } - private int runLength(int id) { - // represents the size in #bytes supported by node 'id' in the tree - return 1 << (log2ChunkSize - depth(id)); - } - - private int runOffset(int id) { - // represents the 0-based offset in #bytes from start of the byte-array chunk - int shift = id - (1 << depth(id)); - return shift * runLength(id); + private int runOffset(int val) { + return val >>> 17 << pageShifts; } private int subpageIdx(int memoryMapIdx) { return memoryMapIdx - maxSubpageAllocs; } - @Override public String toString() { StringBuilder buf = new StringBuilder(); buf.append("Chunk(");