Add Buf.iterateReverse

Motivation:
We have the ability to iterate through the bytes in a buffer with the ByteIterator, but another important use case is being able to iterate through the bytes in reverse order.

Modification:
Add methods for iterating through the bytes in a buffer in reverse order. Also update the copyInto methods to make use of it. Also add a bit of missing javadocs, and argument checks.

Result:
We can also use ByteIterator for efficiently processing data within a buffer in reverse order.
This commit is contained in:
Chris Vest 2020-11-06 13:39:21 +01:00
parent 68795fb1a5
commit a63f3e609d
4 changed files with 529 additions and 34 deletions

View File

@ -266,7 +266,7 @@ public interface Buf extends Rc<Buf>, BufAccessors {
/**
* Iterate the readable bytes of this buffer. The {@linkplain #readerIndex() reader offset} and
* {@linkplain #writerIndex() witer offset} are not modified by the iterator.
*
* <p>
* Care should be taken to ensure that the buffers lifetime extends beyond the iteration, and the
* {@linkplain #readerIndex() reader offset} and {@linkplain #writerIndex() writer offset} are not modified while
* the iteration takes place. Otherwise unpredictable behaviour might result.
@ -277,5 +277,54 @@ public interface Buf extends Rc<Buf>, BufAccessors {
return iterate(readerIndex(), readableBytes());
}
/**
* Iterate the given number bytes of this buffer, starting at the given offset.
* The {@linkplain #readerIndex() reader offset} and {@linkplain #writerIndex() witer offset} are not modified by
* the iterator.
* <p>
* Care should be taken to ensure that the buffers lifetime extends beyond the iteration, and the
* {@linkplain #readerIndex() reader offset} and {@linkplain #writerIndex() writer offset} are not modified while
* the iteration takes place. Otherwise unpredictable behaviour might result.
*
* @param fromOffset The offset into the buffer where iteration should start.
* The first byte read from the iterator will be the byte at this offset.
* @param length The number of bytes to iterate.
* @return A {@link ByteIterator} for the given stretch of bytes of this buffer.
* @throws IllegalArgumentException if the length is negative, or if the region given by the {@code fromOffset} and
* the {@code length} reaches outside of the bounds of this buffer.
*/
ByteIterator iterate(int fromOffset, int length);
/**
* Iterate the readable bytes of this buffer, in reverse. The {@linkplain #readerIndex() reader offset} and
* {@linkplain #writerIndex() witer offset} are not modified by the iterator.
* <p>
* Care should be taken to ensure that the buffers lifetime extends beyond the iteration, and the
* {@linkplain #readerIndex() reader offset} and {@linkplain #writerIndex() writer offset} are not modified while
* the iteration takes place. Otherwise unpredictable behaviour might result.
*
* @return A {@link ByteIterator} for the readable bytes of this buffer.
*/
default ByteIterator iterateReverse() {
int woff = writerIndex();
return iterateReverse(woff == 0? 0 : woff - 1, readableBytes());
}
/**
* Iterate the given number bytes of this buffer, in reverse, starting at the given offset.
* The {@linkplain #readerIndex() reader offset} and {@linkplain #writerIndex() witer offset} are not modified by
* the iterator.
* <p>
* Care should be taken to ensure that the buffers lifetime extends beyond the iteration, and the
* {@linkplain #readerIndex() reader offset} and {@linkplain #writerIndex() writer offset} are not modified while
* the iteration takes place. Otherwise unpredictable behaviour might result.
*
* @param fromOffset The offset into the buffer where iteration should start.
* The first byte read from the iterator will be the byte at this offset.
* @param length The number of bytes to iterate.
* @return A {@link ByteIterator} for the given stretch of bytes of this buffer.
* @throws IllegalArgumentException if the length is negative, or if the region given by the {@code fromOffset} and
* the {@code length} reaches outside of the bounds of this buffer.
*/
ByteIterator iterateReverse(int fromOffset, int length);
}

View File

@ -248,6 +248,31 @@ final class CompositeBuf extends RcSupport<Buf, CompositeBuf> implements Buf {
copyInto(srcPos, (b, s, d, l) -> b.copyInto(s, dest, d, l), destPos, length);
}
private void copyInto(int srcPos, CopyInto dest, int destPos, int length) {
if (length < 0) {
throw new IndexOutOfBoundsException("Length cannot be negative: " + length + '.');
}
if (srcPos < 0) {
throw indexOutOfBounds(srcPos);
}
if (srcPos + length > capacity) {
throw indexOutOfBounds(srcPos + length);
}
while (length > 0) {
var buf = (Buf) chooseBuffer(srcPos, 0);
int toCopy = buf.capacity() - subOffset;
dest.copyInto(buf, subOffset, destPos, toCopy);
srcPos += toCopy;
destPos += toCopy;
length -= toCopy;
}
}
@FunctionalInterface
private interface CopyInto {
void copyInto(Buf src, int srcPos, int destPos, int length);
}
@Override
public void copyInto(int srcPos, Buf dest, int destPos, int length) {
if (length < 0) {
@ -259,14 +284,39 @@ final class CompositeBuf extends RcSupport<Buf, CompositeBuf> implements Buf {
if (srcPos + length > capacity) {
throw indexOutOfBounds(srcPos + length);
}
// todo optimise by doing bulk copy via consituent buffers
for (int i = length - 1; i >= 0; i--) { // Iterate in reverse to account for src and dest buffer overlap.
dest.writeByte(destPos + i, readByte(srcPos + i));
// Iterate in reverse to account for src and dest buffer overlap.
// todo optimise by delegating to constituent buffers.
var itr = iterateReverse(srcPos + length - 1, length);
ByteOrder prevOrder = dest.order();
// We read longs in BE, in reverse, so they need to be flipped for writing.
dest.order(ByteOrder.LITTLE_ENDIAN);
try {
while (itr.hasNextLong()) {
long val = itr.nextLong();
length -= Long.BYTES;
dest.writeLong(destPos + length, val);
}
while (itr.hasNextByte()) {
dest.writeByte(destPos + --length, itr.nextByte());
}
} finally {
dest.order(prevOrder);
}
}
@Override
public ByteIterator iterate(int fromOffset, int length) {
if (fromOffset < 0) {
throw new IllegalArgumentException("The fromOffset cannot be negative: " + fromOffset + '.');
}
if (length < 0) {
throw new IllegalArgumentException("The length cannot be negative: " + length + '.');
}
if (capacity < fromOffset + length) {
throw new IllegalArgumentException("The fromOffset+length is beyond the end of the buffer: " +
"fromOffset=" + fromOffset + ", length=" + length + '.');
}
int startBufferIndex = searchOffsets(fromOffset);
int off = fromOffset - offsets[startBufferIndex];
Buf startBuf = bufs[startBufferIndex];
@ -339,29 +389,87 @@ final class CompositeBuf extends RcSupport<Buf, CompositeBuf> implements Buf {
};
}
private void copyInto(int srcPos, CopyInto dest, int destPos, int length) {
@Override
public ByteIterator iterateReverse(int fromOffset, int length) {
if (fromOffset < 0) {
throw new IllegalArgumentException("The fromOffset cannot be negative: " + fromOffset + '.');
}
if (length < 0) {
throw new IndexOutOfBoundsException("Length cannot be negative: " + length + '.');
throw new IllegalArgumentException("The length cannot be negative: " + length + '.');
}
if (srcPos < 0) {
throw indexOutOfBounds(srcPos);
if (fromOffset - length < -1) {
throw new IllegalArgumentException("The fromOffset-length would underflow the buffer: " +
"fromOffset=" + fromOffset + ", length=" + length + '.');
}
if (srcPos + length > capacity) {
throw indexOutOfBounds(srcPos + length);
}
while (length > 0) {
var buf = (Buf) chooseBuffer(srcPos, 0);
int toCopy = buf.capacity() - subOffset;
dest.copyInto(buf, subOffset, destPos, toCopy);
srcPos += toCopy;
destPos += toCopy;
length -= toCopy;
}
}
int startBufferIndex = searchOffsets(fromOffset);
int off = fromOffset - offsets[startBufferIndex];
Buf startBuf = bufs[startBufferIndex];
ByteIterator startIterator = startBuf.iterateReverse(off, Math.min(off + 1, length));
return new ByteIterator() {
int index = fromOffset;
final int end = fromOffset - length;
int bufferIndex = startBufferIndex;
ByteIterator itr = startIterator;
@FunctionalInterface
private interface CopyInto {
void copyInto(Buf src, int srcPos, int destPos, int length);
@Override
public boolean hasNextLong() {
return bytesLeft() >= Long.BYTES;
}
@Override
public long nextLong() {
if (itr.hasNextLong()) {
index -= Long.BYTES;
return itr.nextLong();
}
if (!hasNextLong()) {
throw new NoSuchElementException();
}
return nextLongFromBytes(); // Leave index increments to 'nextByte'
}
private long nextLongFromBytes() {
long val = 0;
for (int i = 0; i < 8; i++) {
val <<= 8;
val |= nextByte();
}
return val;
}
@Override
public boolean hasNextByte() {
return index > end;
}
@Override
public byte nextByte() {
if (itr.hasNextByte()) {
byte val = itr.nextByte();
index--;
return val;
}
if (!hasNextByte()) {
throw new NoSuchElementException();
}
bufferIndex--;
Buf nextBuf = bufs[bufferIndex];
itr = nextBuf.iterateReverse(nextBuf.capacity() - 1, Math.min(nextBuf.capacity(), bytesLeft()));
byte val = itr.nextByte();
index--;
return val;
}
@Override
public int currentOffset() {
return index;
}
@Override
public int bytesLeft() {
return index - end;
}
};
}
// <editor-fold defaultstate="collapsed" desc="Primitive accessors.">

View File

@ -16,8 +16,6 @@
package io.netty.buffer.b2;
import io.netty.util.ByteIterator;
import io.netty.util.ByteProcessor;
import io.netty.util.internal.PlatformDependent;
import jdk.incubator.foreign.MemorySegment;
import java.nio.ByteBuffer;
@ -155,16 +153,44 @@ class MemSegBuf extends RcSupport<Buf, MemSegBuf> implements Buf {
}
}
private void copyInto(int srcPos, MemorySegment dest, int destPos, int length) {
dest.asSlice(destPos, length).copyFrom(seg.asSlice(srcPos, length));
}
@Override
public void copyInto(int srcPos, Buf dest, int destPos, int length) {
// todo optimise: specialise for MemSegBuf; use ByteIterator.
for (int i = length - 1; i >= 0; i--) { // Iterate in reverse to account for src and dest buffer overlap.
dest.writeByte(destPos + i, readByte(srcPos + i));
// todo optimise: specialise for MemSegBuf.
// Iterate in reverse to account for src and dest buffer overlap.
var itr = iterateReverse(srcPos + length - 1, length);
ByteOrder prevOrder = dest.order();
// We read longs in BE, in reverse, so they need to be flipped for writing.
dest.order(ByteOrder.LITTLE_ENDIAN);
try {
while (itr.hasNextLong()) {
long val = itr.nextLong();
length -= Long.BYTES;
dest.writeLong(destPos + length, val);
}
while (itr.hasNextByte()) {
dest.writeByte(destPos + --length, itr.nextByte());
}
} finally {
dest.order(prevOrder);
}
}
@Override
public ByteIterator iterate(int fromOffset, int length) {
if (fromOffset < 0) {
throw new IllegalArgumentException("The fromOffset cannot be negative: " + fromOffset + '.');
}
if (length < 0) {
throw new IllegalArgumentException("The length cannot be negative: " + length + '.');
}
if (seg.byteSize() < fromOffset + length) {
throw new IllegalArgumentException("The fromOffset+length is beyond the end of the buffer: " +
"fromOffset=" + fromOffset + ", length=" + length + '.');
}
return new ByteIterator() {
final MemorySegment segment = seg;
int index = fromOffset;
@ -178,7 +204,7 @@ class MemSegBuf extends RcSupport<Buf, MemSegBuf> implements Buf {
@Override
public long nextLong() {
if (!hasNextLong()) {
throw new NoSuchElementException();
throw new NoSuchElementException("No 'long' value at offet " + currentOffset() + '.');
}
long val = getLongAtOffset_BE(segment, index);
index += Long.BYTES;
@ -193,7 +219,7 @@ class MemSegBuf extends RcSupport<Buf, MemSegBuf> implements Buf {
@Override
public byte nextByte() {
if (!hasNextByte()) {
throw new NoSuchElementException();
throw new NoSuchElementException("No 'byte' value at offet " + currentOffset() + '.');
}
byte val = getByteAtOffset_BE(segment, index);
index++;
@ -212,8 +238,67 @@ class MemSegBuf extends RcSupport<Buf, MemSegBuf> implements Buf {
};
}
private void copyInto(int srcPos, MemorySegment dest, int destPos, int length) {
dest.asSlice(destPos, length).copyFrom(seg.asSlice(srcPos, length));
@Override
public ByteIterator iterateReverse(int fromOffset, int length) {
if (fromOffset < 0) {
throw new IllegalArgumentException("The fromOffset cannot be negative: " + fromOffset + '.');
}
if (length < 0) {
throw new IllegalArgumentException("The length cannot be negative: " + length + '.');
}
if (seg.byteSize() <= fromOffset) {
throw new IllegalArgumentException("The fromOffset is beyond the end of the buffer: " + fromOffset + '.');
}
if (fromOffset - length < -1) {
throw new IllegalArgumentException("The fromOffset-length would underflow the buffer: " +
"fromOffset=" + fromOffset + ", length=" + length + '.');
}
return new ByteIterator() {
final MemorySegment segment = seg;
int index = fromOffset;
final int end = index - length;
@Override
public boolean hasNextLong() {
return index - Long.BYTES >= end;
}
@Override
public long nextLong() {
if (!hasNextLong()) {
throw new NoSuchElementException("No 'long' value at offet " + currentOffset() + '.');
}
index -= 7;
long val = getLongAtOffset_LE(segment, index);
index--;
return val;
}
@Override
public boolean hasNextByte() {
return index > end;
}
@Override
public byte nextByte() {
if (!hasNextByte()) {
throw new NoSuchElementException("No 'byte' value at offet " + currentOffset() + '.');
}
byte val = getByteAtOffset_LE(segment, index);
index--;
return val;
}
@Override
public int currentOffset() {
return index;
}
@Override
public int bytesLeft() {
return index - end;
}
};
}
// ### CODEGEN START primitive accessors implementation

View File

@ -943,6 +943,37 @@ public abstract class BufTest {
}
private static void checkByteIterationOfRegion(Buf buf) {
try {
buf.iterate(-1, 1);
fail("Should throw on negative offset.");
} catch (IllegalArgumentException ignore) {
// Good.
}
try {
buf.iterate(1, -1);
fail("Should throw on negative length.");
} catch (IllegalArgumentException ignore) {
// Good.
}
try {
buf.iterate(buf.capacity(), 1);
fail("Should throw on offset overflow.");
} catch (IllegalArgumentException ignore) {
// Good.
}
try {
buf.iterate(buf.capacity() - 1, 2);
fail("Should throw on offset + length overflow.");
} catch (IllegalArgumentException ignore) {
// Good.
}
try {
buf.iterate(buf.capacity() - 2, 3);
fail("Should throw on offset + length overflow.");
} catch (IllegalArgumentException ignore) {
// Good.
}
var itr = buf.iterate(1, 0);
assertFalse(itr.hasNextByte());
assertFalse(itr.hasNextLong());
@ -1031,8 +1062,230 @@ public abstract class BufTest {
assertEquals(roff, buf.readerIndex());
assertEquals(woff, buf.writerIndex());
}
// todo reverse iteration
// todo reverse iteration of region
@Test
public void reverseByteIterationOfBigEndianBuffers() {
try (Buf buf = allocate(0x28)) {
buf.order(ByteOrder.BIG_ENDIAN); // The byte order should have no impact.
checkReverseByteIteration(buf);
buf.reset();
checkReverseByteIterationOfRegion(buf);
}
}
@Test
public void reverseByteIterationOfLittleEndianBuffers() {
try (Buf buf = allocate(0x28)) {
buf.order(ByteOrder.LITTLE_ENDIAN); // The byte order should have no impact.
checkReverseByteIteration(buf);
buf.reset();
checkReverseByteIterationOfRegion(buf);
}
}
private static void checkReverseByteIteration(Buf buf) {
var itr = buf.iterateReverse();
assertFalse(itr.hasNextByte());
assertFalse(itr.hasNextLong());
assertEquals(0, itr.bytesLeft());
try {
itr.nextLong();
fail("Expected a no such element exception.");
} catch (NoSuchElementException ignore) {
// Good.
}
try {
itr.nextByte();
fail("Expected a no such element exception.");
} catch (NoSuchElementException ignore) {
// Good.
}
for (int i = 0; i < 0x27; i++) {
buf.writeByte((byte) (i + 1));
}
int roff = buf.readerIndex();
int woff = buf.writerIndex();
itr = buf.iterateReverse();
assertEquals(0x27, itr.bytesLeft());
assertTrue(itr.hasNextByte());
assertTrue(itr.hasNextLong());
assertEquals(0x2726252423222120L, itr.nextLong());
assertEquals(0x1F, itr.bytesLeft());
assertTrue(itr.hasNextLong());
assertEquals(0x1F1E1D1C1B1A1918L, itr.nextLong());
assertTrue(itr.hasNextLong());
assertEquals(0x17, itr.bytesLeft());
assertEquals(0x1716151413121110L, itr.nextLong());
assertTrue(itr.hasNextLong());
assertEquals(0x0F, itr.bytesLeft());
assertEquals(0x0F0E0D0C0B0A0908L, itr.nextLong());
assertFalse(itr.hasNextLong());
assertEquals(7, itr.bytesLeft());
try {
itr.nextLong();
fail("Expected a no such element exception.");
} catch (NoSuchElementException ignore) {
// Good.
}
assertTrue(itr.hasNextByte());
assertEquals((byte) 0x07, itr.nextByte());
assertTrue(itr.hasNextByte());
assertEquals(6, itr.bytesLeft());
assertEquals((byte) 0x06, itr.nextByte());
assertTrue(itr.hasNextByte());
assertEquals(5, itr.bytesLeft());
assertEquals((byte) 0x05, itr.nextByte());
assertTrue(itr.hasNextByte());
assertEquals(4, itr.bytesLeft());
assertEquals((byte) 0x04, itr.nextByte());
assertTrue(itr.hasNextByte());
assertEquals(3, itr.bytesLeft());
assertEquals((byte) 0x03, itr.nextByte());
assertTrue(itr.hasNextByte());
assertEquals(2, itr.bytesLeft());
assertEquals((byte) 0x02, itr.nextByte());
assertTrue(itr.hasNextByte());
assertEquals(1, itr.bytesLeft());
assertEquals((byte) 0x01, itr.nextByte());
assertFalse(itr.hasNextByte());
assertEquals(0, itr.bytesLeft());
try {
itr.nextLong();
fail("Expected a no such element exception.");
} catch (NoSuchElementException ignore) {
// Good.
}
try {
itr.nextByte();
fail("Expected a no such element exception.");
} catch (NoSuchElementException ignore) {
// Good.
}
assertEquals(roff, buf.readerIndex());
assertEquals(woff, buf.writerIndex());
}
private static void checkReverseByteIterationOfRegion(Buf buf) {
try {
buf.iterateReverse(-1, 0);
fail("Should throw on negative offset.");
} catch (IllegalArgumentException ignore) {
// Good.
}
try {
buf.iterateReverse(0, -1);
fail("Should throw on negative length.");
} catch (IllegalArgumentException ignore) {
// Good.
}
try {
buf.iterateReverse(0, 2);
fail("Should throw on offset + length underflow.");
} catch (IllegalArgumentException ignore) {
// Good.
}
try {
buf.iterateReverse(1, 3);
fail("Should throw on offset + length underflow.");
} catch (IllegalArgumentException ignore) {
// Good.
}
try {
buf.iterateReverse(buf.capacity(), 0);
fail("Should throw on offset overflow.");
} catch (IllegalArgumentException ignore) {
// Good.
}
var itr = buf.iterateReverse(1, 0);
assertFalse(itr.hasNextByte());
assertFalse(itr.hasNextLong());
assertEquals(0, itr.bytesLeft());
try {
itr.nextLong();
fail("Expected a no such element exception.");
} catch (NoSuchElementException ignore) {
// Good.
}
try {
itr.nextByte();
fail("Expected a no such element exception.");
} catch (NoSuchElementException ignore) {
// Good.
}
for (int i = 0; i < 0x27; i++) {
buf.writeByte((byte) (i + 1));
}
int roff = buf.readerIndex();
int woff = buf.writerIndex();
itr = buf.iterateReverse(buf.writerIndex() - 2, buf.readableBytes() - 2);
assertEquals(0x25, itr.bytesLeft());
assertTrue(itr.hasNextByte());
assertTrue(itr.hasNextLong());
assertEquals(0x262524232221201FL, itr.nextLong());
assertEquals(0x1D, itr.bytesLeft());
assertTrue(itr.hasNextLong());
assertEquals(0x1E1D1C1B1A191817L, itr.nextLong());
assertTrue(itr.hasNextLong());
assertEquals(0x15, itr.bytesLeft());
assertEquals(0x161514131211100FL, itr.nextLong());
assertTrue(itr.hasNextLong());
assertEquals(0x0D, itr.bytesLeft());
assertEquals(0x0E0D0C0B0A090807L, itr.nextLong());
assertFalse(itr.hasNextLong());
assertEquals(5, itr.bytesLeft());
try {
itr.nextLong();
fail("Expected a no such element exception.");
} catch (NoSuchElementException ignore) {
// Good.
}
assertTrue(itr.hasNextByte());
assertEquals((byte) 0x06, itr.nextByte());
assertTrue(itr.hasNextByte());
assertEquals(4, itr.bytesLeft());
assertEquals((byte) 0x05, itr.nextByte());
assertTrue(itr.hasNextByte());
assertEquals(3, itr.bytesLeft());
assertEquals((byte) 0x04, itr.nextByte());
assertTrue(itr.hasNextByte());
assertEquals(2, itr.bytesLeft());
assertEquals((byte) 0x03, itr.nextByte());
assertTrue(itr.hasNextByte());
assertEquals(1, itr.bytesLeft());
assertEquals((byte) 0x02, itr.nextByte());
assertFalse(itr.hasNextByte());
assertEquals(0, itr.bytesLeft());
try {
itr.nextLong();
fail("Expected a no such element exception.");
} catch (NoSuchElementException ignore) {
// Good.
}
try {
itr.nextByte();
fail("Expected a no such element exception.");
} catch (NoSuchElementException ignore) {
// Good.
}
itr = buf.iterateReverse(buf.readerIndex() + 2, 2);
assertEquals(2, itr.bytesLeft());
assertTrue(itr.hasNextByte());
assertFalse(itr.hasNextLong());
assertEquals((byte) 0x03, itr.nextByte());
assertEquals(1, itr.bytesLeft());
assertTrue(itr.hasNextByte());
assertFalse(itr.hasNextLong());
assertEquals((byte) 0x02, itr.nextByte());
assertEquals(0, itr.bytesLeft());
assertFalse(itr.hasNextByte());
assertFalse(itr.hasNextLong());
assertEquals(roff, buf.readerIndex());
assertEquals(woff, buf.writerIndex());
}
// todo resize copying must preserve contents
// todo resize sharing