andreacavalli
/
CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Remove netty buffers, remove reactive streams

This commit is contained in:
Andrea Cavalli 2023-02-09 23:34:25 +01:00
parent 5c112484bd
commit a9857f7553
294 changed files with 9141 additions and 16277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
pom.xml
View File

@ -5,7 +5,7 @@
<groupId>it.cavallium</groupId>
<artifactId>dbengine</artifactId>
<version>3.0.${revision}</version>
<version>4.0.${revision}</version>
<packaging>jar</packaging>
<properties>
@ -13,7 +13,8 @@
<revision>0-SNAPSHOT</revision>
<dbengine.ci>false</dbengine.ci>
<micrometer.version>1.9.5</micrometer.version>
<lucene.version>9.4.2</lucene.version>
<lucene.version>9.5.0</lucene.version>
<rocksdb.version>7.9.2</rocksdb.version>
<junit.jupiter.version>5.9.0</junit.jupiter.version>
<data.generator.version>1.0.244</data.generator.version>
</properties>
@ -96,13 +97,6 @@
<artifactId>hamcrest-library</artifactId>
<version>2.2</version>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-bom</artifactId>
<version>2022.0.2</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
@ -113,33 +107,11 @@
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-tools</artifactId>
<classifier>original</classifier>
<scope>runtime</scope>
<version>3.5.1</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>31.1-jre</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty5-buffer</artifactId>
<version>5.0.0.Alpha5</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
@ -155,11 +127,6 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor.netty.incubator</groupId>
<artifactId>reactor-netty-incubator-quic</artifactId>
<version>0.1.2</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
@ -266,7 +233,7 @@
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
<version>7.9.2</version>
<version>${rocksdb.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
@ -356,40 +323,6 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty</artifactId>
<exclusions>
<exclusion>
<groupId>io.netty.incubator</groupId>
<artifactId>netty-incubator-codec-native-quic</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>reactor-netty-core</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
@ -457,11 +390,6 @@
<version>3.12.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<testSourceDirectory>src/test/java</testSourceDirectory>
@ -638,39 +566,4 @@
</plugins>
</pluginManagement>
</build>
<profiles>
<profile>
<id>reactor-agent</id>
<activation>
<activeByDefault>false</activeByDefault>
<property>
<name>reactor.agent.enable</name>
<value>true</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<groupId>net.bytebuddy</groupId>
<artifactId>byte-buddy-maven-plugin</artifactId>
<version>1.12.22</version>
<executions>
<execution>
<goals>
<goal>transform</goal>
</goals>
</execution>
</executions>
<configuration>
<transformations>
<transformation>
<plugin>reactor.tools.agent.ReactorDebugByteBuddyPlugin</plugin>
</transformation>
</transformations>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

17
src/main/data-generator/quic-rpc.yaml
View File

@ -77,8 +77,6 @@ superTypesData:
NIOFSDirectory,
RAFFSDirectory,
DirectIOFSDirectory,
RocksDBStandaloneDirectory,
RocksDBSharedDirectory,
NRTCachingDirectory
]
StandardFSDirectoryOptions: [
@ -90,7 +88,6 @@ superTypesData:
MemoryMappedFSDirectory,
NIOFSDirectory,
RAFFSDirectory,
RocksDBStandaloneDirectory,
StandardFSDirectoryOptions
]
Filter: [
@ -134,8 +131,8 @@ customTypesData:
serializer: it.cavallium.dbengine.database.remote.LLSnapshotSerializer
Bytes:
javaClass: it.unimi.dsi.fastutil.bytes.ByteList
serializer: it.cavallium.dbengine.database.remote.ByteListSerializer
javaClass: it.cavallium.dbengine.buffers.Buf
serializer: it.cavallium.dbengine.database.remote.BufSerializer
StringMap:
javaClass: java.util.Map<java.lang.String, java.lang.String>
serializer: it.cavallium.dbengine.database.remote.StringMapSerializer
@ -248,7 +245,6 @@ baseTypesData:
lowMemory: boolean
useDirectIO: boolean
allowMemoryMapping: boolean
allowNettyDirect: boolean
optimistic: boolean
maxOpenFiles: -int
blockCache: -long
@ -331,7 +327,6 @@ baseTypesData:
indexWriterMaxBufferedDocs: -int
applyAllDeletes: -boolean
writeAllDeletes: -boolean
allowNonVolatileCollection: boolean
maxInMemoryResultEntries: int
mergePolicy: TieredMergePolicy
TieredMergePolicy:
@ -359,14 +354,6 @@ baseTypesData:
delegate: StandardFSDirectoryOptions
mergeBufferSize: -int
minBytesDirect: -long
RocksDBStandaloneDirectory:
data:
managedPath: Path
blockSize: int
RocksDBSharedDirectory:
data:
managedPath: Path
blockSize: int
NRTCachingDirectory:
data:
delegate: LuceneDirectoryOptions

55
src/main/java/it/cavallium/dbengine/MetricUtils.java
View File

@ -1,55 +0,0 @@
package it.cavallium.dbengine;
import io.netty5.buffer.pool.PoolArenaMetric;
import io.netty5.buffer.pool.PooledBufferAllocator;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/**
* Netty5 hides some metrics. This utility class can read them.
*/
public class MetricUtils {
private static final Logger LOG = LogManager.getLogger(MetricUtils.class);
private static final MethodHandle GET_ARENA_METRICS;
static {
var lookup = MethodHandles.lookup();
// Get the method handle that returns the metrics of each pool arena
MethodHandle handle = null;
try {
// Find the class
var pooledBufferClass = Class.forName("io.netty5.buffer.pool.PooledBufferAllocatorMetric");
// Find the handle of the method
handle = lookup.findVirtual(pooledBufferClass, "arenaMetrics", MethodType.methodType(List.class));
} catch (NoSuchMethodException | IllegalAccessException | ClassNotFoundException ex) {
logMetricsNotAccessible(ex);
}
GET_ARENA_METRICS = handle;
}
private static void logMetricsNotAccessible(Throwable ex) {
LOG.debug("Failed to open pooled buffer allocator metrics", ex);
}
/**
* Get the metrics of each pool arena of a pooled allocator
* @param allocator Pooled allocator
* @return A list of {@link PoolArenaMetric}
*/
@SuppressWarnings("unchecked")
public static List<PoolArenaMetric> getPoolArenaMetrics(PooledBufferAllocator allocator) {
var metric = allocator.metric();
try {
// Invoke the method to get the metrics
return (List<PoolArenaMetric>) GET_ARENA_METRICS.invoke(metric);
} catch (Throwable e) {
return List.of();
}
}
}

182
src/main/java/it/cavallium/dbengine/buffers/Buf.java Normal file
View File

@ -0,0 +1,182 @@
package it.cavallium.dbengine.buffers;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.bytes.ByteList;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.RandomAccess;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.warp.commonutils.stream.SafeByteArrayInputStream;
import org.warp.commonutils.stream.SafeByteArrayOutputStream;
import org.warp.commonutils.stream.SafeDataOutput;
public interface Buf extends ByteList, RandomAccess {
static Buf wrap(ByteList bytes) {
if (bytes instanceof Buf buf) {
return buf;
} else if (bytes instanceof ByteArrayList byteArrayList) {
return ByteListBuf.wrap(byteArrayList.elements(), byteArrayList.size());
} else {
return ByteListBuf.wrap(bytes.toByteArray());
}
}
static Buf wrap(ByteList bytes, int from, int to) {
if (bytes instanceof Buf buf) {
return buf.subList(from, to);
} else if (bytes instanceof ByteArrayList byteArrayList) {
return ByteListBuf.wrap(byteArrayList.elements(), byteArrayList.size()).subList(from, to);
} else {
return ByteListBuf.wrap(bytes.toByteArray()).subList(from, to);
}
}
static Buf wrap(byte[] bytes) {
return ByteListBuf.wrap(bytes);
}
static Buf wrap(byte[] bytes, int from, int to) {
return ByteListBuf.wrap(bytes, to).subList(from, to);
}
static Buf create(int initialCapacity) {
return new ByteListBuf(initialCapacity);
}
static Buf copyOf(byte[] original) {
return new ByteListBuf(original);
}
static Buf create() {
return new ByteListBuf();
}
static Buf wrap(byte[] array, int length) {
return ByteListBuf.wrap(array, length);
}
static Buf createZeroes(int length) {
return ByteListBuf.wrap(new byte[length], length);
}
/**
* Get this element as an array, converting it if needed
*/
byte @NotNull[] asArray();
/**
* Get this element as an array, only if it's already an array, otherwise return null
*/
byte @Nullable[] asArrayStrict();
/**
* Get this element as an array with equal or bigger size, converting it if needed
* The returned array may be bigger than expected!
*/
byte @Nullable[] asUnboundedArray();
/**
* Get this element as an array with equal or bigger size, only if it's already an array, otherwise return null
* The returned array may be bigger than expected!
*/
byte @Nullable[] asUnboundedArrayStrict();
boolean isMutable();
void freeze();
@Override
Buf subList(int from, int to);
Buf copy();
SafeByteArrayInputStream binaryInputStream();
void writeTo(SafeDataOutput dataOutput);
default long getLong(int i) {
return Longs.fromBytes(getByte(i),
getByte(i + 1),
getByte(i + 2),
getByte(i + 3),
getByte(i + 4),
getByte(i + 5),
getByte(i + 6),
getByte(i + 7)
);
}
default int getInt(int i) {
return Ints.fromBytes(getByte(i),
getByte(i + 1),
getByte(i + 2),
getByte(i + 3)
);
}
default float getFloat(int i) {
return Float.intBitsToFloat(getInt(i));
}
default double getDouble(int i) {
return Double.longBitsToDouble(getLong(i));
}
default boolean getBoolean(int i) {
return getByte(i) != 0;
}
default void setBoolean(int i, boolean val) {
set(i, val ? (byte) 1 : 0);
}
default void setByte(int i, byte val) {
set(i, val);
}
default void setInt(int i, int val) {
set(i, (byte) (val >> 24));
set(i + 1, (byte) (val >> 16));
set(i + 2, (byte) (val >> 8));
set(i + 3, (byte) val);
}
default void setLong(int i, long val) {
set(i, (byte) (val >> 56));
set(i + 1, (byte) (val >> 48));
set(i + 2, (byte) (val >> 40));
set(i + 3, (byte) (val >> 32));
set(i + 4, (byte) (val >> 24));
set(i + 5, (byte) (val >> 16));
set(i + 6, (byte) (val >> 8));
set(i + 7, (byte) val);
}
default void setFloat(int i, float val) {
setInt(i, Float.floatToRawIntBits(val));
}
default void setDouble(int i, double val) {
setLong(i, Double.doubleToRawLongBits(val));
}
default SafeByteArrayOutputStream binaryOutputStream() {
return binaryOutputStream(0, size());
}
default SafeByteArrayOutputStream binaryOutputStream(int from) {
return binaryOutputStream(from, size());
}
SafeByteArrayOutputStream binaryOutputStream(int from, int to);
boolean equals(int aStartIndex, Buf b, int bStartIndex, int length);
boolean equals(int aStartIndex, byte[] b, int bStartIndex, int length);
default String toString(Charset charset) {
return new String(this.asArray(), charset);
}
}

42
src/main/java/it/cavallium/dbengine/buffers/BufDataInput.java Normal file
View File

@ -0,0 +1,42 @@
package it.cavallium.dbengine.buffers;
import org.jetbrains.annotations.NotNull;
import org.warp.commonutils.stream.SafeByteArrayInputStream;
import org.warp.commonutils.stream.SafeDataInputStream;
public class BufDataInput extends SafeDataInputStream {
/**
* Creates a DataInputStream that uses the specified underlying InputStream.
*
* @param in the specified input stream
*/
private BufDataInput(@NotNull SafeByteArrayInputStream in) {
super(in);
}
public static BufDataInput create(Buf byteList) {
return new BufDataInput(byteList.binaryInputStream());
}
@Deprecated
@Override
public void close() {
}
@Override
public void mark(int readlimit) {
throw new UnsupportedOperationException();
}
@Override
public void reset() {
throw new UnsupportedOperationException();
}
@Override
public boolean markSupported() {
return false;
}
}

218
src/main/java/it/cavallium/dbengine/buffers/BufDataOutput.java Normal file
View File

@ -0,0 +1,218 @@
package it.cavallium.dbengine.buffers;
import it.unimi.dsi.fastutil.Arrays;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Objects;
import org.jetbrains.annotations.NotNull;
import org.warp.commonutils.stream.SafeByteArrayOutputStream;
import org.warp.commonutils.stream.SafeDataOutputStream;
public class BufDataOutput implements DataOutput {
private final SafeByteArrayOutputStream buf;
private final SafeDataOutputStream dOut;
private final int limit;
private BufDataOutput(SafeByteArrayOutputStream buf) {
this.buf = buf;
this.dOut = new SafeDataOutputStream(buf);
limit = Integer.MAX_VALUE;
}
private BufDataOutput(SafeByteArrayOutputStream buf, int maxSize) {
this.buf = buf;
this.dOut = new SafeDataOutputStream(buf);
this.limit = maxSize;
}
public static BufDataOutput createLimited(int maxSize, int hint) {
if (hint >= 0) {
if (maxSize < 0 || maxSize == Integer.MAX_VALUE) {
return create(hint);
} else {
return new BufDataOutput(new SafeByteArrayOutputStream(Math.min(maxSize, hint)), maxSize);
}
} else {
return createLimited(maxSize);
}
}
public static BufDataOutput createLimited(int maxSize) {
if (maxSize < 0 || maxSize == Integer.MAX_VALUE) {
return create();
} else {
return new BufDataOutput(new SafeByteArrayOutputStream(maxSize), maxSize);
}
}
public static BufDataOutput create() {
return new BufDataOutput(new SafeByteArrayOutputStream());
}
public static BufDataOutput create(int hint) {
if (hint >= 0) {
return new BufDataOutput(new SafeByteArrayOutputStream(hint));
} else {
return create();
}
}
public static BufDataOutput wrap(Buf buf, int from, int to) {
Arrays.ensureFromTo(buf.size(), from, to);
if (buf.isEmpty()) {
return createLimited(0);
} else {
return new BufDataOutput(buf.binaryOutputStream(from), to - from);
}
}
public static BufDataOutput wrap(Buf buf) {
if (buf.isEmpty()) {
return createLimited(0);
} else {
return new BufDataOutput(buf.binaryOutputStream(), buf.size());
}
}
private IllegalStateException unreachable(IOException ex) {
return new IllegalStateException(ex);
}
@Override
public void write(int b) {
checkOutOfBounds(1);
dOut.write(b);
}
private void checkOutOfBounds(int delta) {
if (dOut.size() + delta > limit) {
throw new IndexOutOfBoundsException(limit);
}
}
@Override
public void write(byte @NotNull [] b) {
checkOutOfBounds(b.length);
dOut.write(b);
}
@Override
public void write(byte @NotNull [] b, int off, int len) {
checkOutOfBounds(Math.max(0, Math.min(b.length - off, len)));
dOut.write(b, off, len);
}
@Override
public void writeBoolean(boolean v) {
checkOutOfBounds(1);
dOut.writeBoolean(v);
}
@Override
public void writeByte(int v) {
checkOutOfBounds(Byte.BYTES);
dOut.writeByte(v);
}
@Override
public void writeShort(int v) {
checkOutOfBounds(Short.BYTES);
dOut.writeShort(v);
}
@Override
public void writeChar(int v) {
checkOutOfBounds(Character.BYTES);
dOut.writeChar(v);
}
@Override
public void writeInt(int v) {
checkOutOfBounds(Integer.BYTES);
dOut.writeInt(v);
}
@Override
public void writeLong(long v) {
checkOutOfBounds(Long.BYTES);
dOut.writeLong(v);
}
@Override
public void writeFloat(float v) {
checkOutOfBounds(Float.BYTES);
dOut.writeFloat(v);
}
@Override
public void writeDouble(double v) {
checkOutOfBounds(Double.BYTES);
dOut.writeDouble(v);
}
public void ensureWritable(int size) {
dOut.flush();
buf.ensureWritable(size);
}
@Override
public void writeBytes(@NotNull String s) {
checkOutOfBounds(s.length() * Byte.BYTES);
dOut.writeBytes(s);
}
// todo: check
public void writeBytes(Buf deserialized) {
checkOutOfBounds(deserialized.size());
deserialized.writeTo(dOut);
}
public void writeBytes(byte[] b, int off, int len) {
write(b, off, len);
}
@Override
public void writeChars(@NotNull String s) {
checkOutOfBounds(Character.BYTES * s.length());
dOut.writeChars(s);
}
@Override
public void writeUTF(@NotNull String s) {
throw new UnsupportedOperationException();
}
public Buf asList() {
dOut.flush();
return Buf.wrap(this.buf.array, this.buf.length);
}
@Override
public String toString() {
return dOut.toString();
}
@Override
public int hashCode() {
return dOut.hashCode();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
BufDataOutput that = (BufDataOutput) o;
return Objects.equals(dOut, that.dOut);
}
public int size() {
return dOut.size();
}
}

467
src/main/java/it/cavallium/dbengine/buffers/ByteListBuf.java Normal file
View File

@ -0,0 +1,467 @@
package it.cavallium.dbengine.buffers;
import it.unimi.dsi.fastutil.bytes.AbstractByteList;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.bytes.ByteCollection;
import it.unimi.dsi.fastutil.bytes.ByteConsumer;
import it.unimi.dsi.fastutil.bytes.ByteIterator;
import it.unimi.dsi.fastutil.bytes.ByteIterators;
import it.unimi.dsi.fastutil.bytes.ByteList;
import it.unimi.dsi.fastutil.bytes.ByteListIterator;
import it.unimi.dsi.fastutil.bytes.ByteSpliterator;
import it.unimi.dsi.fastutil.bytes.ByteSpliterators;
import java.io.Serial;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.NoSuchElementException;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.warp.commonutils.stream.SafeByteArrayInputStream;
import org.warp.commonutils.stream.SafeByteArrayOutputStream;
import org.warp.commonutils.stream.SafeDataOutput;
class ByteListBuf extends ByteArrayList implements Buf {
private boolean mutable = true;
protected ByteListBuf(byte[] a, boolean wrapped) {
super(a, wrapped);
}
public ByteListBuf(int capacity) {
super(capacity);
}
public ByteListBuf() {
}
public ByteListBuf(Collection<? extends Byte> c) {
super(c);
}
public ByteListBuf(ByteCollection c) {
super(c);
}
public ByteListBuf(ByteList l) {
super(l);
}
public ByteListBuf(byte[] a) {
super(a);
}
public ByteListBuf(byte[] a, int offset, int length) {
super(a, offset, length);
}
public ByteListBuf(Iterator<? extends Byte> i) {
super(i);
}
public ByteListBuf(ByteIterator i) {
super(i);
}
/**
* Wraps a given array into an array list of given size.
*
* <p>
* Note it is guaranteed that the type of the array returned by {@link #elements()} will be the same
* (see the comments in the class documentation).
*
* @param a an array to wrap.
* @param length the length of the resulting array list.
* @return a new array list of the given size, wrapping the given array.
*/
public static ByteListBuf wrap(final byte[] a, final int length) {
if (length > a.length) throw new IllegalArgumentException("The specified length (" + length + ") is greater than the array size (" + a.length + ")");
final ByteListBuf l = new ByteListBuf(a, true);
l.size = length;
return l;
}
/**
* Wraps a given array into an array list.
*
* <p>
* Note it is guaranteed that the type of the array returned by {@link #elements()} will be the same
* (see the comments in the class documentation).
*
* @param a an array to wrap.
* @return a new array list wrapping the given array.
*/
public static ByteListBuf wrap(final byte[] a) {
return wrap(a, a.length);
}
/**
* Creates a new empty array list.
*
* @return a new empty array list.
*/
public static ByteListBuf of() {
return new ByteListBuf();
}
/**
* Creates an array list using an array of elements.
*
* @param init a the array the will become the new backing array of the array list.
* @return a new array list backed by the given array.
* @see #wrap
*/
public static ByteListBuf of(final byte... init) {
return wrap(init);
}
@Override
public byte @NotNull [] asArray() {
if (this.size() == a.length) {
return this.a;
} else {
return this.toByteArray();
}
}
@Override
public byte @Nullable [] asArrayStrict() {
if (this.size() == a.length) {
return a;
} else {
return null;
}
}
@Override
public byte @Nullable [] asUnboundedArray() {
return a;
}
@Override
public byte @Nullable [] asUnboundedArrayStrict() {
return a;
}
@Override
public boolean isMutable() {
return mutable;
}
@Override
public void freeze() {
mutable = false;
}
@Override
public Buf subList(int from, int to) {
if (from == 0 && to == size()) return this;
ensureIndex(from);
ensureIndex(to);
if (from > to) throw new IndexOutOfBoundsException("Start index (" + from + ") is greater than end index (" + to + ")");
return new SubList(from, to);
}
@Override
public Buf copy() {
var copied = ByteListBuf.wrap(this.a.clone());
copied.size = this.size;
return copied;
}
@Override
public SafeByteArrayInputStream binaryInputStream() {
return new SafeByteArrayInputStream(this.a, 0, this.size);
}
@Override
public void writeTo(SafeDataOutput dataOutput) {
dataOutput.write(this.a, 0, this.size);
}
@Override
public SafeByteArrayOutputStream binaryOutputStream(int from, int to) {
it.unimi.dsi.fastutil.Arrays.ensureFromTo(size, from, to);
return new SafeByteArrayOutputStream(a, from, to);
}
@Override
public boolean equals(int aStartIndex, Buf b, int bStartIndex, int length) {
return b.equals(bStartIndex, this.a, aStartIndex, length);
}
@Override
public boolean equals(int aStartIndex, byte[] b, int bStartIndex, int length) {
if (aStartIndex < 0) return false;
if (aStartIndex + length > this.size) {
return false;
}
return Arrays.equals(a, aStartIndex, aStartIndex + length, b, bStartIndex, bStartIndex + length);
}
@Override
public String toString(Charset charset) {
return new String(a, 0, size, charset);
}
private class SubList extends AbstractByteList.ByteRandomAccessSubList implements Buf {
@Serial
private static final long serialVersionUID = -3185226345314976296L;
private boolean subMutable = true;
protected SubList(int from, int to) {
super(ByteListBuf.this, from, to);
}
// Most of the inherited methods should be fine, but we can override a few of them for performance.
// Needed because we can't access the parent class' instance variables directly in a different
// instance of SubList.
private byte[] getParentArray() {
return a;
}
@Override
public @NotNull Buf subList(int from, int to) {
it.unimi.dsi.fastutil.Arrays.ensureFromTo(a.length, from, to);
if (from > to) throw new IllegalArgumentException("Start index (" + from + ") is greater than end index (" + to + ")");
// Sadly we have to rewrap this, because if there is a sublist of a sublist, and the
// subsublist adds, both sublists need to update their "to" value.
return new SubList(from, to);
}
@Override
public Buf copy() {
return Buf.wrap(Arrays.copyOfRange(a, from, to));
}
@Override
public SafeByteArrayInputStream binaryInputStream() {
return new SafeByteArrayInputStream(a, from, size());
}
@Override
public void writeTo(SafeDataOutput dataOutput) {
dataOutput.write(a, from, size());
}
@Override
public SafeByteArrayOutputStream binaryOutputStream(int from, int to) {
it.unimi.dsi.fastutil.Arrays.ensureFromTo(size(), from, to);
return new SafeByteArrayOutputStream(a, from + this.from, to + this.from);
}
@Override
public boolean equals(int aStartIndex, Buf b, int bStartIndex, int length) {
return b.equals(bStartIndex, a, aStartIndex + from, length);
}
@Override
public boolean equals(int aStartIndex, byte[] b, int bStartIndex, int length) {
var aFrom = from + aStartIndex;
var aTo = from + aStartIndex + length;
if (aFrom < from) return false;
if (aTo > to) return false;
return Arrays.equals(a, aFrom, aTo, b, bStartIndex, bStartIndex + length);
}
@Override
public byte getByte(int i) {
ensureRestrictedIndex(i);
return a[i + from];
}
@Override
public byte @NotNull [] asArray() {
if (this.from == 0 && this.to == a.length) {
return a;
} else {
return toByteArray();
}
}
@Override
public byte @Nullable [] asArrayStrict() {
if (this.from == 0 && this.to == a.length) {
return a;
} else {
return null;
}
}
@Override
public byte @Nullable [] asUnboundedArray() {
if (from == 0) {
return a;
} else {
return toByteArray();
}
}
@Override
public byte @Nullable [] asUnboundedArrayStrict() {
if (from == 0) {
return a;
} else {
return null;
}
}
@Override
public boolean isMutable() {
return mutable && subMutable;
}
@Override
public void freeze() {
subMutable = false;
}
private final class SubListIterator extends ByteIterators.AbstractIndexBasedListIterator {
// We are using pos == 0 to be 0 relative to SubList.from (meaning you need to do a[from + i] when
// accessing array).
SubListIterator(int index) {
super(0, index);
}
@Override
protected byte get(int i) {
return a[from + i];
}
@Override
protected void add(int i, byte k) {
ByteListBuf.SubList.this.add(i, k);
}
@Override
protected void set(int i, byte k) {
ByteListBuf.SubList.this.set(i, k);
}
@Override
protected void remove(int i) {
ByteListBuf.SubList.this.removeByte(i);
}
@Override
protected int getMaxPos() {
return to - from;
}
@Override
public byte nextByte() {
if (!hasNext()) throw new NoSuchElementException();
return a[from + (lastReturned = pos++)];
}
@Override
public byte previousByte() {
if (!hasPrevious()) throw new NoSuchElementException();
return a[from + (lastReturned = --pos)];
}
@Override
public void forEachRemaining(final ByteConsumer action) {
final int max = to - from;
while (pos < max) {
action.accept(a[from + (lastReturned = pos++)]);
}
}
}
@Override
public @NotNull ByteListIterator listIterator(int index) {
return new ByteListBuf.SubList.SubListIterator(index);
}
private final class SubListSpliterator extends ByteSpliterators.LateBindingSizeIndexBasedSpliterator {
// We are using pos == 0 to be 0 relative to real array 0
SubListSpliterator() {
super(from);
}
private SubListSpliterator(int pos, int maxPos) {
super(pos, maxPos);
}
@Override
protected int getMaxPosFromBackingStore() {
return to;
}
@Override
protected byte get(int i) {
return a[i];
}
@Override
protected ByteListBuf.SubList.SubListSpliterator makeForSplit(int pos, int maxPos) {
return new ByteListBuf.SubList.SubListSpliterator(pos, maxPos);
}
@Override
public boolean tryAdvance(final ByteConsumer action) {
if (pos >= getMaxPos()) return false;
action.accept(a[pos++]);
return true;
}
@Override
public void forEachRemaining(final ByteConsumer action) {
final int max = getMaxPos();
while (pos < max) {
action.accept(a[pos++]);
}
}
}
@Override
public ByteSpliterator spliterator() {
return new ByteListBuf.SubList.SubListSpliterator();
}
boolean contentsEquals(byte[] otherA, int otherAFrom, int otherATo) {
if (a == otherA && from == otherAFrom && to == otherATo) return true;
return Arrays.equals(a, from, to, otherA, otherAFrom, otherATo);
}
@Override
public boolean equals(Object o) {
if (o == this) return true;
if (o == null) return false;
if (!(o instanceof java.util.List)) return false;
if (o instanceof ByteListBuf other) {
return contentsEquals(other.a, 0, other.size());
}
if (o instanceof SubList other) {
return contentsEquals(other.getParentArray(), other.from, other.to);
}
return super.equals(o);
}
int contentsCompareTo(byte[] otherA, int otherAFrom, int otherATo) {
if (a == otherA && from == otherAFrom && to == otherATo) return 0;
return Arrays.compareUnsigned(a, from, to, otherA, otherAFrom, otherATo);
}
@Override
public int compareTo(final java.util.@NotNull List<? extends Byte> l) {
if (l instanceof ByteListBuf other) {
return contentsCompareTo(other.a, 0, other.size());
}
if (l instanceof ByteListBuf.SubList other) {
return contentsCompareTo(other.getParentArray(), other.from, other.to);
}
return super.compareTo(l);
}
@Override
public String toString(Charset charset) {
return new String(a, from, to, charset);
}
}
}

43
src/main/java/it/cavallium/dbengine/client/Backuppable.java
View File

@ -1,8 +1,7 @@
package it.cavallium.dbengine.client;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
import reactor.core.publisher.Mono;
import reactor.core.publisher.SignalType;
public abstract class Backuppable implements IBackuppable {
@ -13,29 +12,29 @@ public abstract class Backuppable implements IBackuppable {
private final AtomicInteger state = new AtomicInteger();
@Override
public final Mono<Void> pauseForBackup() {
return Mono.defer(() -> {
if (state.compareAndSet(State.RUNNING.ordinal(), State.PAUSING.ordinal())) {
return onPauseForBackup().doFinally(type -> state.compareAndSet(State.PAUSING.ordinal(),
type == SignalType.ON_ERROR ? State.RUNNING.ordinal() : State.PAUSED.ordinal()
));
} else {
return Mono.empty();
public final void pauseForBackup() {
if (state.compareAndSet(State.RUNNING.ordinal(), State.PAUSING.ordinal())) {
try {
onPauseForBackup();
state.compareAndSet(State.PAUSING.ordinal(), State.PAUSED.ordinal());
} catch (Throwable ex) {
state.compareAndSet(State.PAUSING.ordinal(), State.RUNNING.ordinal());
throw ex;
}
});
}
}
@Override
public final Mono<Void> resumeAfterBackup() {
return Mono.defer(() -> {
if (state.compareAndSet(State.PAUSED.ordinal(), State.RESUMING.ordinal())) {
return onResumeAfterBackup().doFinally(type -> state.compareAndSet(State.RESUMING.ordinal(),
type == SignalType.ON_ERROR ? State.PAUSED.ordinal() : State.RUNNING.ordinal()
));
} else {
return Mono.empty();
public final void resumeAfterBackup() {
if (state.compareAndSet(State.PAUSED.ordinal(), State.RESUMING.ordinal())) {
try {
onResumeAfterBackup();
state.compareAndSet(State.RESUMING.ordinal(), State.RUNNING.ordinal());
} catch (Throwable ex) {
state.compareAndSet(State.RESUMING.ordinal(), State.PAUSED.ordinal());
throw ex;
}
});
}
}
@Override
@ -47,9 +46,9 @@ public abstract class Backuppable implements IBackuppable {
return State.values()[state.get()];
}
protected abstract Mono<Void> onPauseForBackup();
protected abstract void onPauseForBackup();
protected abstract Mono<Void> onResumeAfterBackup();
protected abstract void onResumeAfterBackup();
public final void setStopped() {
state.set(State.STOPPED.ordinal());

4
src/main/java/it/cavallium/dbengine/client/BadBlock.java
View File

@ -1,8 +1,8 @@
package it.cavallium.dbengine.client;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.rpc.current.data.Column;
import it.unimi.dsi.fastutil.bytes.ByteList;
import org.jetbrains.annotations.Nullable;
public record BadBlock(String databaseName, @Nullable Column column, @Nullable ByteList rawKey,
public record BadBlock(String databaseName, @Nullable Column column, @Nullable Buf rawKey,
@Nullable Throwable ex) {}

4
src/main/java/it/cavallium/dbengine/client/CastMapper.java
View File

@ -1,14 +1,14 @@
package it.cavallium.dbengine.client;
import it.cavallium.dbengine.client.Mapper;
public class CastMapper<T, U> implements Mapper<T, U> {
@SuppressWarnings("unchecked")
@Override
public U map(T key) {
return (U) key;
}
@SuppressWarnings("unchecked")
@Override
public T unmap(U key) {
return (T) key;

18
src/main/java/it/cavallium/dbengine/client/CompositeDatabase.java
View File

@ -1,36 +1,32 @@
package it.cavallium.dbengine.client;
import io.micrometer.core.instrument.MeterRegistry;
import io.netty5.buffer.BufferAllocator;
import it.cavallium.dbengine.database.DatabaseOperations;
import it.cavallium.dbengine.database.DatabaseProperties;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.util.stream.Stream;
public interface CompositeDatabase extends DatabaseProperties, DatabaseOperations {
Mono<Void> preClose();
void preClose();
Mono<Void> close();
void close();
/**
* Can return SnapshotException
*/
Mono<CompositeSnapshot> takeSnapshot();
CompositeSnapshot takeSnapshot();
/**
* Can return SnapshotException
*/
Mono<Void> releaseSnapshot(CompositeSnapshot snapshot);
BufferAllocator getAllocator();
void releaseSnapshot(CompositeSnapshot snapshot);
MeterRegistry getMeterRegistry();
/**
* Find corrupted items
*/
Flux<BadBlock> badBlocks();
Stream<BadBlock> badBlocks();
Mono<Void> verifyChecksum();
void verifyChecksum();
}

47
src/main/java/it/cavallium/dbengine/client/CountedStream.java
View File

@ -1,47 +0,0 @@
package it.cavallium.dbengine.client;
import java.util.Collection;
import java.util.List;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public class CountedStream<T> {
private final Flux<T> stream;
private final long count;
public CountedStream(Flux<T> stream, long count) {
this.stream = stream;
this.count = count;
}
public Flux<T> getStream() {
return stream;
}
public long getCount() {
return count;
}
@SafeVarargs
public static <T> CountedStream<T> merge(CountedStream<T>... stream) {
return merge(List.of(stream));
}
public static <T> CountedStream<T> merge(Collection<CountedStream<T>> stream) {
return stream
.stream()
.reduce((a, b) -> new CountedStream<>(Flux.merge(a.getStream(), b.getStream()), a.getCount() + b.getCount()))
.orElseGet(() -> new CountedStream<>(Flux.empty(), 0));
}
public static <T> Mono<CountedStream<T>> merge(Flux<CountedStream<T>> stream) {
return stream
.reduce((a, b) -> new CountedStream<>(Flux.merge(a.getStream(), b.getStream()), a.getCount() + b.getCount()))
.switchIfEmpty(Mono.fromSupplier(() -> new CountedStream<>(Flux.empty(), 0)));
}
public Mono<List<T>> collectList() {
return stream.collectList();
}
}

1
src/main/java/it/cavallium/dbengine/client/DefaultDatabaseOptions.java
View File

@ -58,7 +58,6 @@ public class DefaultDatabaseOptions {
false,
false,
true,
true,
Nullableint.empty(),
Nullablelong.empty(),
Nullablelong.empty(),

4
src/main/java/it/cavallium/dbengine/client/HitEntry.java
View File

@ -1,9 +1,9 @@
package it.cavallium.dbengine.client;
import org.jetbrains.annotations.NotNull;
import reactor.core.publisher.Mono;
import org.jetbrains.annotations.Nullable;
public record HitEntry<T, U>(T key, U value, float score)
public record HitEntry<T, U>(T key, @Nullable U value, float score)
implements Comparable<HitEntry<T, U>> {
@Override

7
src/main/java/it/cavallium/dbengine/client/HitKey.java
View File

@ -1,16 +1,13 @@
package it.cavallium.dbengine.client;
import it.cavallium.dbengine.database.collections.DatabaseEmpty.Nothing;
import java.util.Comparator;
import java.util.function.Function;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
public record HitKey<T>(T key, float score) implements Comparable<HitKey<T>> {
public <U> Mono<HitEntry<T, U>> withValue(Function<T, Mono<U>> valueGetter) {
return valueGetter.apply(key).map(value -> new HitEntry<>(key, value, score));
public <U> HitEntry<T, U> withValue(Function<T, U> valueGetter) {
return new HitEntry<>(key, valueGetter.apply(key), score);
}
public <U> HitEntry<T, U> withNullValue() {

54
src/main/java/it/cavallium/dbengine/client/Hits.java
View File

@ -4,29 +4,25 @@ import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.DiscardingCloseable;
import it.cavallium.dbengine.database.SafeCloseable;
import it.cavallium.dbengine.database.collections.ValueGetter;
import it.cavallium.dbengine.database.collections.ValueTransformer;
import it.cavallium.dbengine.lucene.LuceneCloseable;
import it.cavallium.dbengine.utils.SimpleResource;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public class Hits<T> extends SimpleResource implements DiscardingCloseable {
private static final Logger LOG = LogManager.getLogger(Hits.class);
private static final Hits<?> EMPTY_HITS = new Hits<>(Flux.empty(), TotalHitsCount.of(0, true), false);
private final Flux<T> results;
private static final Hits<?> EMPTY_HITS = new Hits<>(Stream.empty(), TotalHitsCount.of(0, true), false);
private final Stream<T> results;
private final TotalHitsCount totalHitsCount;
public Hits(Flux<T> results, TotalHitsCount totalHitsCount) {
public Hits(Stream<T> results, TotalHitsCount totalHitsCount) {
this(results, totalHitsCount, true);
}
private Hits(Flux<T> results, TotalHitsCount totalHitsCount, boolean canClose) {
private Hits(Stream<T> results, TotalHitsCount totalHitsCount, boolean canClose) {
super(canClose);
this.results = results;
this.totalHitsCount = totalHitsCount;
@ -37,44 +33,16 @@ public class Hits<T> extends SimpleResource implements DiscardingCloseable {
return (Hits<T>) EMPTY_HITS;
}
public static <T, U> Function<Hits<HitKey<T>>, Hits<LazyHitEntry<T, U>>> generateMapper(
public static <T, U> Function<Hits<HitKey<T>>, Hits<HitEntry<T, U>>> generateMapper(
ValueGetter<T, U> valueGetter) {
return result -> {
var hitsToTransform = result.results()
.map(hit -> new LazyHitEntry<>(Mono.just(hit.key()), valueGetter.get(hit.key()), hit.score()));
.map(hit -> new HitEntry<>(hit.key(), valueGetter.get(hit.key()), hit.score()));
return Hits.withResource(hitsToTransform, result.totalHitsCount(), result);
};
}
public static <T, U> Function<Hits<HitKey<T>>, Hits<LazyHitEntry<T, U>>> generateMapper(
ValueTransformer<T, U> valueTransformer) {
return result -> {
try {
var sharedHitsFlux = result.results().publish().refCount(3);
var scoresFlux = sharedHitsFlux.map(HitKey::score);
var keysFlux = sharedHitsFlux.map(HitKey::key);
var valuesFlux = valueTransformer.transform(keysFlux);
var transformedFlux = Flux.zip((Object[] data) -> {
//noinspection unchecked
var keyMono = Mono.just((T) data[0]);
//noinspection unchecked
var val = (Entry<T, Optional<U>>) data[1];
var valMono = Mono.justOrEmpty(val.getValue());
var score = (Float) data[2];
return new LazyHitEntry<>(keyMono, valMono, score);
}, keysFlux, valuesFlux, scoresFlux);
return Hits.withResource(transformedFlux, result.totalHitsCount(), result);
} catch (Throwable t) {
result.close();
throw t;
}
};
}
public static <T> Hits<T> withResource(Flux<T> hits, TotalHitsCount count, SafeCloseable resource) {
public static <T> Hits<T> withResource(Stream<T> hits, TotalHitsCount count, SafeCloseable resource) {
if (resource instanceof LuceneCloseable luceneCloseable) {
return new LuceneHits<>(hits, count, luceneCloseable);
} else {
@ -82,7 +50,7 @@ public class Hits<T> extends SimpleResource implements DiscardingCloseable {
}
}
public Flux<T> results() {
public Stream<T> results() {
ensureOpen();
return results;
}
@ -105,7 +73,7 @@ public class Hits<T> extends SimpleResource implements DiscardingCloseable {
private final LuceneCloseable resource;
public LuceneHits(Flux<U> hits, TotalHitsCount count, LuceneCloseable resource) {
public LuceneHits(Stream<U> hits, TotalHitsCount count, LuceneCloseable resource) {
super(hits, count);
this.resource = resource;
}
@ -125,7 +93,7 @@ public class Hits<T> extends SimpleResource implements DiscardingCloseable {
private final SafeCloseable resource;
public CloseableHits(Flux<U> hits, TotalHitsCount count, SafeCloseable resource) {
public CloseableHits(Stream<U> hits, TotalHitsCount count, SafeCloseable resource) {
super(hits, count);
this.resource = resource;
}

6
src/main/java/it/cavallium/dbengine/client/IBackuppable.java
View File

@ -1,12 +1,10 @@
package it.cavallium.dbengine.client;
import reactor.core.publisher.Mono;
public interface IBackuppable {
Mono<Void> pauseForBackup();
void pauseForBackup();
Mono<Void> resumeAfterBackup();
void resumeAfterBackup();
boolean isPaused();
}

128
src/main/java/it/cavallium/dbengine/client/IndexAction.java
View File

@ -1,128 +0,0 @@
package it.cavallium.dbengine.client;
import it.cavallium.dbengine.client.IndexAction.Add;
import it.cavallium.dbengine.client.IndexAction.AddMulti;
import it.cavallium.dbengine.client.IndexAction.Update;
import it.cavallium.dbengine.client.IndexAction.UpdateMulti;
import it.cavallium.dbengine.client.IndexAction.Delete;
import it.cavallium.dbengine.client.IndexAction.DeleteAll;
import it.cavallium.dbengine.client.IndexAction.TakeSnapshot;
import it.cavallium.dbengine.client.IndexAction.ReleaseSnapshot;
import it.cavallium.dbengine.client.IndexAction.Flush;
import it.cavallium.dbengine.client.IndexAction.Refresh;
import it.cavallium.dbengine.client.IndexAction.Close;
import it.cavallium.dbengine.database.LLUpdateDocument;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLTerm;
import java.util.Map;
import java.util.Map.Entry;
import reactor.core.publisher.Flux;
import reactor.core.publisher.MonoSink;
sealed interface IndexAction permits Add, AddMulti, Update, UpdateMulti, Delete, DeleteAll, TakeSnapshot,
ReleaseSnapshot, Flush, Refresh, Close {
IndexActionType getType();
final record Add(LLTerm key, LLUpdateDocument doc, MonoSink<Void> addedFuture) implements IndexAction {
@Override
public IndexActionType getType() {
return IndexActionType.ADD;
}
}
final record AddMulti(Flux<Entry<LLTerm, LLUpdateDocument>> docsFlux, MonoSink<Void> addedMultiFuture) implements IndexAction {
@Override
public IndexActionType getType() {
return IndexActionType.ADD_MULTI;
}
}
final record Update(LLTerm key, LLUpdateDocument doc, MonoSink<Void> updatedFuture) implements IndexAction {
@Override
public IndexActionType getType() {
return IndexActionType.UPDATE;
}
}
final record UpdateMulti(Map<LLTerm, LLUpdateDocument> docs, MonoSink<Void> updatedMultiFuture) implements IndexAction {
@Override
public IndexActionType getType() {
return IndexActionType.UPDATE_MULTI;
}
}
final record Delete(LLTerm key, MonoSink<Void> deletedFuture) implements IndexAction {
@Override
public IndexActionType getType() {
return IndexActionType.DELETE;
}
}
final record DeleteAll(MonoSink<Void> deletedAllFuture) implements IndexAction {
@Override
public IndexActionType getType() {
return IndexActionType.DELETE_ALL;
}
}
final record TakeSnapshot(MonoSink<LLSnapshot> snapshotFuture) implements IndexAction {
@Override
public IndexActionType getType() {
return IndexActionType.TAKE_SNAPSHOT;
}
}
final record ReleaseSnapshot(LLSnapshot snapshot, MonoSink<Void> releasedFuture) implements IndexAction {
@Override
public IndexActionType getType() {
return IndexActionType.RELEASE_SNAPSHOT;
}
}
final record Flush(MonoSink<Void> flushFuture) implements IndexAction {
@Override
public IndexActionType getType() {
return IndexActionType.FLUSH;
}
}
final record Refresh(boolean force, MonoSink<Void> refreshFuture) implements IndexAction {
@Override
public IndexActionType getType() {
return IndexActionType.REFRESH;
}
}
final record Close(MonoSink<Void> closeFuture) implements IndexAction {
@Override
public IndexActionType getType() {
return IndexActionType.CLOSE;
}
}
enum IndexActionType {
ADD,
ADD_MULTI,
UPDATE,
UPDATE_MULTI,
DELETE,
DELETE_ALL,
TAKE_SNAPSHOT,
RELEASE_SNAPSHOT,
FLUSH,
REFRESH,
CLOSE
}
}

33
src/main/java/it/cavallium/dbengine/client/Indicizer.java
View File

@ -4,40 +4,33 @@ import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import it.cavallium.dbengine.database.LLIndexRequest;
import it.cavallium.dbengine.database.LLSoftUpdateDocument;
import it.cavallium.dbengine.database.LLUpdateDocument;
import it.cavallium.dbengine.database.LLTerm;
import it.cavallium.dbengine.database.LLUpdateDocument;
import it.cavallium.dbengine.database.LLUpdateFields;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.rpc.current.data.IndicizerAnalyzers;
import it.cavallium.dbengine.rpc.current.data.IndicizerSimilarities;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.BytesRef;
import org.jetbrains.annotations.NotNull;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.util.function.Tuple2;
public abstract class Indicizer<T, U> {
/**
* Transform a value to an IndexRequest.
*/
public abstract @NotNull Mono<? extends LLIndexRequest> toIndexRequest(@NotNull T key, @NotNull U value);
public abstract @NotNull LLIndexRequest toIndexRequest(@NotNull T key, @NotNull U value);
public final @NotNull Mono<LLUpdateDocument> toDocument(@NotNull T key, @NotNull U value) {
return toIndexRequest(key, value).map(req -> {
if (req instanceof LLUpdateFields updateFields) {
return new LLUpdateDocument(updateFields.items());
} else if (req instanceof LLUpdateDocument updateDocument) {
return updateDocument;
} else if (req instanceof LLSoftUpdateDocument softUpdateDocument) {
return new LLUpdateDocument(softUpdateDocument.items());
} else {
throw new UnsupportedOperationException("Unexpected request type: " + req);
}
});
public final @NotNull LLUpdateDocument toDocument(@NotNull T key, @NotNull U value) {
var req = toIndexRequest(key, value);
if (req instanceof LLUpdateFields updateFields) {
return new LLUpdateDocument(updateFields.items());
} else if (req instanceof LLUpdateDocument updateDocument) {
return updateDocument;
} else if (req instanceof LLSoftUpdateDocument softUpdateDocument) {
return new LLUpdateDocument(softUpdateDocument.items());
} else {
throw new UnsupportedOperationException("Unexpected request type: " + req);
}
}
public abstract @NotNull LLTerm toIndex(@NotNull T key);

1
src/main/java/it/cavallium/dbengine/client/IndicizerAnalyzers.java
View File

@ -1,7 +1,6 @@
package it.cavallium.dbengine.client;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.rpc.current.serializers.IndicizerAnalyzersSerializer;
import java.util.Map;
public class IndicizerAnalyzers {

1
src/main/java/it/cavallium/dbengine/client/IntOpenHashSetJsonAdapter.java
View File

@ -2,7 +2,6 @@ package it.cavallium.dbengine.client;
import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
import it.cavallium.data.generator.nativedata.Int52;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import java.io.IOException;
import org.jetbrains.annotations.NotNull;

15
src/main/java/it/cavallium/dbengine/client/LazyHitEntry.java
View File

@ -1,15 +0,0 @@
package it.cavallium.dbengine.client;
import org.jetbrains.annotations.NotNull;
import reactor.core.publisher.Mono;
public record LazyHitEntry<T, U>(Mono<T> key, Mono<U> value, float score) {
public Mono<HitEntry<T, U>> resolve() {
return Mono.zip(key, value, (k, v) -> new HitEntry<>(k, v, score));
}
public Mono<HitKey<T>> resolveKey() {
return key.map(k -> new HitKey<>(k, score));
}
}

19
src/main/java/it/cavallium/dbengine/client/LazyHitKey.java
View File

@ -1,19 +0,0 @@
package it.cavallium.dbengine.client;
import java.util.function.Function;
import reactor.core.publisher.Mono;
public record LazyHitKey<T>(Mono<T> key, float score) {
public <U> LazyHitEntry<T, U> withValue(Function<T, Mono<U>> valueGetter) {
return new LazyHitEntry<>(key, key.flatMap(valueGetter), score);
}
public Mono<HitKey<T>> resolve() {
return key.map(k -> new HitKey<>(k, score));
}
public <U> Mono<HitEntry<T, U>> resolveWithValue(Function<T, Mono<U>> valueGetter) {
return resolve().flatMap(key -> key.withValue(valueGetter));
}
}

51
src/main/java/it/cavallium/dbengine/client/LuceneIndex.java
View File

@ -1,78 +1,71 @@
package it.cavallium.dbengine.client;
import io.netty5.util.Send;
import it.cavallium.dbengine.client.query.ClientQueryParams;
import it.cavallium.dbengine.client.query.current.data.Query;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.Delta;
import it.cavallium.dbengine.database.LLSnapshottable;
import it.cavallium.dbengine.database.collections.ValueGetter;
import it.cavallium.dbengine.database.collections.ValueTransformer;
import it.cavallium.dbengine.lucene.collector.Buckets;
import it.cavallium.dbengine.lucene.searcher.BucketParams;
import it.unimi.dsi.fastutil.doubles.DoubleArrayList;
import java.util.List;
import java.util.Map.Entry;
import java.util.stream.Stream;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public interface LuceneIndex<T, U> extends LLSnapshottable {
Mono<Void> addDocument(T key, U value);
void addDocument(T key, U value);
Mono<Long> addDocuments(boolean atomic, Flux<Entry<T, U>> entries);
long addDocuments(boolean atomic, Stream<Entry<T, U>> entries);
Mono<Void> deleteDocument(T key);
void deleteDocument(T key);
Mono<Void> updateDocument(T key, @NotNull U value);
void updateDocument(T key, @NotNull U value);
Mono<Long> updateDocuments(Flux<Entry<T, U>> entries);
long updateDocuments(Stream<Entry<T, U>> entries);
default Mono<Void> updateOrDeleteDocument(T key, @Nullable U value) {
default void updateOrDeleteDocument(T key, @Nullable U value) {
if (value == null) {
return deleteDocument(key);
deleteDocument(key);
} else {
return updateDocument(key, value);
updateDocument(key, value);
}
}
default Mono<Void> updateOrDeleteDocumentIfModified(T key, @NotNull Delta<U> delta) {
return updateOrDeleteDocumentIfModified(key, delta.current(), delta.isModified());
default void updateOrDeleteDocumentIfModified(T key, @NotNull Delta<U> delta) {
updateOrDeleteDocumentIfModified(key, delta.current(), delta.isModified());
}
default Mono<Void> updateOrDeleteDocumentIfModified(T key, @Nullable U currentValue, boolean modified) {
default void updateOrDeleteDocumentIfModified(T key, @Nullable U currentValue, boolean modified) {
if (modified) {
return updateOrDeleteDocument(key, currentValue);
} else {
return Mono.empty();
updateOrDeleteDocument(key, currentValue);
}
}
Mono<Void> deleteAll();
void deleteAll();
Mono<Hits<HitKey<T>>> moreLikeThis(ClientQueryParams queryParams, T key,
Hits<HitKey<T>> moreLikeThis(ClientQueryParams queryParams, T key,
U mltDocumentValue);
Mono<Hits<HitKey<T>>> search(ClientQueryParams queryParams);
Hits<HitKey<T>> search(ClientQueryParams queryParams);
Mono<Buckets> computeBuckets(@Nullable CompositeSnapshot snapshot,
Buckets computeBuckets(@Nullable CompositeSnapshot snapshot,
@NotNull List<Query> queries,
@Nullable Query normalizationQuery,
BucketParams bucketParams);
Mono<TotalHitsCount> count(@Nullable CompositeSnapshot snapshot, Query query);
TotalHitsCount count(@Nullable CompositeSnapshot snapshot, Query query);
boolean isLowMemoryMode();
void close();
Mono<Void> flush();
void flush();
Mono<Void> waitForMerges();
void waitForMerges();
Mono<Void> waitForLastMerges();
void waitForLastMerges();
Mono<Void> refresh(boolean force);
void refresh(boolean force);
}

128
src/main/java/it/cavallium/dbengine/client/LuceneIndexImpl.java
View File

@ -5,7 +5,6 @@ import it.cavallium.dbengine.client.Hits.LuceneHits;
import it.cavallium.dbengine.client.query.ClientQueryParams;
import it.cavallium.dbengine.client.query.current.data.Query;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.DiscardingCloseable;
import it.cavallium.dbengine.database.LLKeyScore;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLSearchResultShard;
@ -13,8 +12,6 @@ import it.cavallium.dbengine.database.LLSearchResultShard.LuceneLLSearchResultSh
import it.cavallium.dbengine.database.LLSearchResultShard.ResourcesLLSearchResultShard;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLTerm;
import it.cavallium.dbengine.database.LLUpdateDocument;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.SafeCloseable;
import it.cavallium.dbengine.lucene.LuceneCloseable;
import it.cavallium.dbengine.lucene.LuceneUtils;
@ -26,14 +23,12 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.logging.Level;
import java.util.function.Function;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.publisher.SignalType;
public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
@ -56,96 +51,87 @@ public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
}
@Override
public Mono<Void> addDocument(T key, U value) {
return indicizer
.toDocument(key, value)
.flatMap(doc -> luceneIndex.addDocument(indicizer.toIndex(key), doc));
public void addDocument(T key, U value) {
luceneIndex.addDocument(indicizer.toIndex(key), indicizer.toDocument(key, value));
}
@Override
public Mono<Long> addDocuments(boolean atomic, Flux<Entry<T, U>> entries) {
return luceneIndex.addDocuments(atomic, entries.flatMap(entry -> indicizer
.toDocument(entry.getKey(), entry.getValue())
.map(doc -> Map.entry(indicizer.toIndex(entry.getKey()), doc))));
public long addDocuments(boolean atomic, Stream<Entry<T, U>> entries) {
return luceneIndex.addDocuments(atomic, entries.map(entry ->
Map.entry(indicizer.toIndex(entry.getKey()), indicizer.toDocument(entry.getKey(), entry.getValue()))));
}
@Override
public Mono<Void> deleteDocument(T key) {
public void deleteDocument(T key) {
LLTerm id = indicizer.toIndex(key);
return luceneIndex.deleteDocument(id);
luceneIndex.deleteDocument(id);
}
@Override
public Mono<Void> updateDocument(T key, @NotNull U value) {
return indicizer
.toIndexRequest(key, value)
.flatMap(doc -> luceneIndex.update(indicizer.toIndex(key), doc));
public void updateDocument(T key, @NotNull U value) {
luceneIndex.update(indicizer.toIndex(key), indicizer.toIndexRequest(key, value));
}
@Override
public Mono<Long> updateDocuments(Flux<Entry<T, U>> entries) {
Flux<Entry<LLTerm, LLUpdateDocument>> mappedEntries = entries
.flatMap(entry -> Mono
.zip(Mono.just(indicizer.toIndex(entry.getKey())),
indicizer.toDocument(entry.getKey(), entry.getValue()).single(),
Map::entry
)
.single()
)
.log("impl-update-documents", Level.FINEST, false, SignalType.ON_NEXT, SignalType.ON_COMPLETE);
return luceneIndex.updateDocuments(mappedEntries);
public long updateDocuments(Stream<Entry<T, U>> entries) {
return luceneIndex.updateDocuments(entries.map(entry ->
Map.entry(indicizer.toIndex(entry.getKey()), indicizer.toDocument(entry.getKey(), entry.getValue()))));
}
@Override
public Mono<Void> deleteAll() {
return luceneIndex.deleteAll();
public void deleteAll() {
luceneIndex.deleteAll();
}
@Override
public Mono<Hits<HitKey<T>>> moreLikeThis(ClientQueryParams queryParams,
public Hits<HitKey<T>> moreLikeThis(ClientQueryParams queryParams,
T key,
U mltDocumentValue) {
var mltDocumentFields
= indicizer.getMoreLikeThisDocumentFields(key, mltDocumentValue);
return luceneIndex
var results = luceneIndex
.moreLikeThis(resolveSnapshot(queryParams.snapshot()),
queryParams.toQueryParams(),
indicizer.getKeyFieldName(),
mltDocumentFields
)
.collectList()
.mapNotNull(shards -> mergeResults(queryParams, shards))
.map(llSearchResult -> mapResults(llSearchResult))
.defaultIfEmpty(Hits.empty())
.doOnDiscard(DiscardingCloseable.class, LLUtils::onDiscard);
.toList();
LLSearchResultShard mergedResults = mergeResults(queryParams, results);
if (mergedResults != null) {
return mapResults(mergedResults);
} else {
return Hits.empty();
}
}
@Override
public Mono<Hits<HitKey<T>>> search(ClientQueryParams queryParams) {
return luceneIndex
public Hits<HitKey<T>> search(ClientQueryParams queryParams) {
var results = luceneIndex
.search(resolveSnapshot(queryParams.snapshot()),
queryParams.toQueryParams(),
indicizer.getKeyFieldName()
)
.collectList()
.mapNotNull(shards -> mergeResults(queryParams, shards))
.map(llSearchResult -> mapResults(llSearchResult))
.defaultIfEmpty(Hits.empty())
.doOnDiscard(DiscardingCloseable.class, LLUtils::onDiscard);
.toList();
var mergedResults = mergeResults(queryParams, results);
if (mergedResults != null) {
return mapResults(mergedResults);
} else {
return Hits.empty();
}
}
@Override
public Mono<Buckets> computeBuckets(@Nullable CompositeSnapshot snapshot,
public Buckets computeBuckets(@Nullable CompositeSnapshot snapshot,
@NotNull List<Query> query,
@Nullable Query normalizationQuery,
BucketParams bucketParams) {
return luceneIndex.computeBuckets(resolveSnapshot(snapshot), query,
normalizationQuery, bucketParams).single();
return luceneIndex.computeBuckets(resolveSnapshot(snapshot), query, normalizationQuery, bucketParams);
}
private Hits<HitKey<T>> mapResults(LLSearchResultShard llSearchResult) {
Flux<HitKey<T>> scoresWithKeysFlux = llSearchResult.results()
Stream<HitKey<T>> scoresWithKeysFlux = llSearchResult.results()
.map(hit -> new HitKey<>(indicizer.getKey(hit.key()), hit.score()));
if (llSearchResult instanceof LuceneCloseable luceneCloseable) {
@ -156,10 +142,8 @@ public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
}
@Override
public Mono<TotalHitsCount> count(@Nullable CompositeSnapshot snapshot, Query query) {
return luceneIndex
.count(resolveSnapshot(snapshot), query, MAX_COUNT_TIME)
.doOnDiscard(DiscardingCloseable.class, LLUtils::onDiscard);
public TotalHitsCount count(@Nullable CompositeSnapshot snapshot, Query query) {
return luceneIndex.count(resolveSnapshot(snapshot), query, MAX_COUNT_TIME);
}
@Override
@ -176,36 +160,36 @@ public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
* Flush writes to disk
*/
@Override
public Mono<Void> flush() {
return luceneIndex.flush();
public void flush() {
luceneIndex.flush();
}
@Override
public Mono<Void> waitForMerges() {
return luceneIndex.waitForMerges();
public void waitForMerges() {
luceneIndex.waitForMerges();
}
@Override
public Mono<Void> waitForLastMerges() {
return luceneIndex.waitForLastMerges();
public void waitForLastMerges() {
luceneIndex.waitForLastMerges();
}
/**
* Refresh index searcher
*/
@Override
public Mono<Void> refresh(boolean force) {
return luceneIndex.refresh(force);
public void refresh(boolean force) {
luceneIndex.refresh(force);
}
@Override
public Mono<LLSnapshot> takeSnapshot() {
public LLSnapshot takeSnapshot() {
return luceneIndex.takeSnapshot();
}
@Override
public Mono<Void> releaseSnapshot(LLSnapshot snapshot) {
return luceneIndex.releaseSnapshot(snapshot);
public void releaseSnapshot(LLSnapshot snapshot) {
luceneIndex.releaseSnapshot(snapshot);
}
@SuppressWarnings({"unchecked", "rawtypes"})
@ -217,7 +201,7 @@ public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
return shards.get(0);
}
TotalHitsCount count = null;
ObjectArrayList<Flux<LLKeyScore>> results = new ObjectArrayList<>(shards.size());
ObjectArrayList<Stream<LLKeyScore>> results = new ObjectArrayList<>(shards.size());
ObjectArrayList resources = new ObjectArrayList(shards.size());
boolean luceneResources = false;
for (LLSearchResultShard shard : shards) {
@ -230,17 +214,17 @@ public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
count = LuceneUtils.sum(count, shard.totalHitsCount());
}
var maxLimit = queryParams.offset() + queryParams.limit();
results.add(shard.results().take(maxLimit, true));
results.add(shard.results().limit(maxLimit));
resources.add(shard);
}
Objects.requireNonNull(count);
Flux<LLKeyScore> resultsFlux;
Stream<LLKeyScore> resultsFlux;
if (results.size() == 0) {
resultsFlux = Flux.empty();
resultsFlux = Stream.empty();
} else if (results.size() == 1) {
resultsFlux = results.get(0);
} else {
resultsFlux = Flux.merge(results);
resultsFlux = results.parallelStream().flatMap(Function.identity());
}
if (luceneResources) {
return new LuceneLLSearchResultShard(resultsFlux, count, (List<LuceneCloseable>) resources);

14
src/main/java/it/cavallium/dbengine/client/MappedSerializer.java
View File

@ -1,11 +1,11 @@
package it.cavallium.dbengine.client;
import io.netty5.buffer.Buffer;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.buffers.BufDataInput;
import it.cavallium.dbengine.buffers.BufDataOutput;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.Serializer;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public class MappedSerializer<A, B> implements Serializer<B> {
@ -19,13 +19,13 @@ public class MappedSerializer<A, B> implements Serializer<B> {
}
@Override
public @NotNull B deserialize(@NotNull Buffer serialized) throws SerializationException {
return keyMapper.map(serializer.deserialize(serialized));
public @NotNull B deserialize(@NotNull BufDataInput in) throws SerializationException {
return keyMapper.map(serializer.deserialize(in));
}
@Override
public void serialize(@NotNull B deserialized, Buffer output) throws SerializationException {
serializer.serialize(keyMapper.unmap(deserialized), output);
public void serialize(@NotNull B deserialized, BufDataOutput out) throws SerializationException {
serializer.serialize(keyMapper.unmap(deserialized), out);
}
@Override

14
src/main/java/it/cavallium/dbengine/client/MappedSerializerFixedLength.java
View File

@ -1,11 +1,11 @@
package it.cavallium.dbengine.client;
import io.netty5.buffer.Buffer;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.buffers.BufDataInput;
import it.cavallium.dbengine.buffers.BufDataOutput;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public class MappedSerializerFixedLength<A, B> implements SerializerFixedBinaryLength<B> {
@ -19,13 +19,13 @@ public class MappedSerializerFixedLength<A, B> implements SerializerFixedBinaryL
}
@Override
public @NotNull B deserialize(@NotNull Buffer serialized) throws SerializationException {
return keyMapper.map(fixedLengthSerializer.deserialize(serialized));
public @NotNull B deserialize(@NotNull BufDataInput in) throws SerializationException {
return keyMapper.map(fixedLengthSerializer.deserialize(in));
}
@Override
public void serialize(@NotNull B deserialized, Buffer output) throws SerializationException {
fixedLengthSerializer.serialize(keyMapper.unmap(deserialized), output);
public void serialize(@NotNull B deserialized, BufDataOutput out) throws SerializationException {
fixedLengthSerializer.serialize(keyMapper.unmap(deserialized), out);
}
@Override

2
src/main/java/it/cavallium/dbengine/client/NoMapper.java
View File

@ -1,7 +1,5 @@
package it.cavallium.dbengine.client;
import it.cavallium.dbengine.client.Mapper;
public class NoMapper<T> implements Mapper<T, T> {
@Override

90
src/main/java/it/cavallium/dbengine/client/UninterruptibleScheduler.java
View File

@ -1,90 +0,0 @@
package it.cavallium.dbengine.client;
import java.util.concurrent.TimeUnit;
import org.jetbrains.annotations.NotNull;
import reactor.core.Disposable;
import reactor.core.scheduler.Scheduler;
public class UninterruptibleScheduler {
public static Scheduler uninterruptibleScheduler(Scheduler scheduler) {
return new Scheduler() {
@Override
public @NotNull Disposable schedule(@NotNull Runnable task) {
scheduler.schedule(task);
return () -> {};
}
@Override
public @NotNull Disposable schedule(@NotNull Runnable task, long delay, @NotNull TimeUnit unit) {
scheduler.schedule(task, delay, unit);
return () -> {};
}
@Override
public @NotNull Disposable schedulePeriodically(@NotNull Runnable task,
long initialDelay,
long period,
@NotNull TimeUnit unit) {
scheduler.schedulePeriodically(task, initialDelay, period, unit);
return () -> {};
}
@Override
public boolean isDisposed() {
return scheduler.isDisposed();
}
@Override
public void dispose() {
scheduler.dispose();
}
@Override
public void start() {
scheduler.start();
}
@Override
public long now(@NotNull TimeUnit unit) {
return Scheduler.super.now(unit);
}
@Override
public @NotNull Worker createWorker() {
var worker = scheduler.createWorker();
return new Worker() {
@Override
public @NotNull Disposable schedule(@NotNull Runnable task) {
worker.schedule(task);
return () -> {};
}
@Override
public void dispose() {
}
@Override
public boolean isDisposed() {
return worker.isDisposed();
}
@Override
public @NotNull Disposable schedule(@NotNull Runnable task, long delay, @NotNull TimeUnit unit) {
worker.schedule(task, delay, unit);
return () -> {};
}
@Override
public @NotNull Disposable schedulePeriodically(@NotNull Runnable task,
long initialDelay,
long period,
@NotNull TimeUnit unit) {
worker.schedulePeriodically(task, initialDelay, period, unit);
return () -> {};
}
};
}
};
}
}

1
src/main/java/it/cavallium/dbengine/client/query/ClientQueryParams.java
View File

@ -1,7 +1,6 @@
package it.cavallium.dbengine.client.query;
import io.soabase.recordbuilder.core.RecordBuilder;
import it.cavallium.data.generator.nativedata.Nullablefloat;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.client.Sort;
import it.cavallium.dbengine.client.query.current.data.NoSort;

18
src/main/java/it/cavallium/dbengine/client/query/QueryMoshi.java
View File

@ -1,12 +1,19 @@
package it.cavallium.dbengine.client.query;
import com.squareup.moshi.JsonAdapter;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.client.IntOpenHashSetJsonAdapter;
import it.cavallium.dbengine.client.query.current.CurrentVersion;
import it.cavallium.dbengine.client.query.current.IBaseType;
import it.cavallium.dbengine.client.query.current.IType;
import it.cavallium.dbengine.utils.BooleanListJsonAdapter;
import it.cavallium.dbengine.utils.ByteListJsonAdapter;
import it.cavallium.dbengine.utils.CharListJsonAdapter;
import it.cavallium.dbengine.utils.IntListJsonAdapter;
import it.cavallium.dbengine.utils.LongListJsonAdapter;
import it.cavallium.dbengine.utils.MoshiPolymorphic;
import it.cavallium.dbengine.utils.ShortListJsonAdapter;
import it.unimi.dsi.fastutil.booleans.BooleanList;
import it.unimi.dsi.fastutil.bytes.ByteList;
import it.unimi.dsi.fastutil.chars.CharList;
import it.unimi.dsi.fastutil.ints.IntList;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
@ -18,13 +25,6 @@ import it.unimi.dsi.fastutil.shorts.ShortList;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import it.cavallium.dbengine.utils.BooleanListJsonAdapter;
import it.cavallium.dbengine.utils.ByteListJsonAdapter;
import it.cavallium.dbengine.utils.CharListJsonAdapter;
import it.cavallium.dbengine.utils.IntListJsonAdapter;
import it.cavallium.dbengine.utils.LongListJsonAdapter;
import it.cavallium.dbengine.utils.MoshiPolymorphic;
import it.cavallium.dbengine.utils.ShortListJsonAdapter;
public class QueryMoshi extends MoshiPolymorphic<IType> {
@ -57,7 +57,7 @@ public class QueryMoshi extends MoshiPolymorphic<IType> {
this.concreteClasses = concreteClasses;
Object2ObjectMap<Class<?>, JsonAdapter<?>> extraAdapters = new Object2ObjectOpenHashMap<>();
extraAdapters.put(BooleanList.class, new BooleanListJsonAdapter());
extraAdapters.put(ByteList.class, new ByteListJsonAdapter());
extraAdapters.put(Buf.class, new ByteListJsonAdapter());
extraAdapters.put(ShortList.class, new ShortListJsonAdapter());
extraAdapters.put(CharList.class, new CharListJsonAdapter());
extraAdapters.put(IntList.class, new IntListJsonAdapter());

158
src/main/java/it/cavallium/dbengine/client/query/QueryParser.java
View File

@ -51,16 +51,7 @@ import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.en.EnglishPossessiveFilter;
import org.apache.lucene.analysis.en.PorterStemFilter;
import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.IntPoint;
@ -89,7 +80,7 @@ public class QueryParser {
return null;
}
switch (query.getBaseType$()) {
case StandardQuery:
case StandardQuery -> {
var standardQuery = (it.cavallium.dbengine.client.query.current.data.StandardQuery) query;
// Fix the analyzer
@ -98,19 +89,12 @@ public class QueryParser {
.stream()
.collect(Collectors.toMap(Function.identity(), term -> new NoOpAnalyzer()));
analyzer = new PerFieldAnalyzerWrapper(analyzer, customAnalyzers);
var standardQueryParser = new StandardQueryParser(analyzer);
standardQueryParser.setPointsConfigMap(standardQuery
.pointsConfig()
.stream()
.collect(Collectors.toMap(
PointConfig::field,
pointConfig -> new PointsConfig(
toNumberFormat(pointConfig.data().numberFormat()),
toType(pointConfig.data().type())
)
)));
standardQueryParser.setPointsConfigMap(standardQuery.pointsConfig().stream().collect(
Collectors.toMap(PointConfig::field, pointConfig ->
new PointsConfig(toNumberFormat(pointConfig.data().numberFormat()), toType(pointConfig.data().type()))
))
);
var defaultFields = standardQuery.defaultFields();
try {
Query parsed;
@ -126,7 +110,8 @@ public class QueryParser {
} catch (QueryNodeException e) {
throw new IllegalStateException("Can't parse query expression \"" + standardQuery.query() + "\"", e);
}
case BooleanQuery:
}
case BooleanQuery -> {
var booleanQuery = (it.cavallium.dbengine.client.query.current.data.BooleanQuery) query;
var bq = new Builder();
for (BooleanQueryPart part : booleanQuery.parts()) {
@ -141,101 +126,127 @@ public class QueryParser {
}
bq.setMinimumNumberShouldMatch(booleanQuery.minShouldMatch());
return bq.build();
case IntPointExactQuery:
}
case IntPointExactQuery -> {
var intPointExactQuery = (IntPointExactQuery) query;
return IntPoint.newExactQuery(intPointExactQuery.field(), intPointExactQuery.value());
case IntNDPointExactQuery:
}
case IntNDPointExactQuery -> {
var intndPointExactQuery = (IntNDPointExactQuery) query;
var intndValues = intndPointExactQuery.value().toIntArray();
return IntPoint.newRangeQuery(intndPointExactQuery.field(), intndValues, intndValues);
case LongPointExactQuery:
}
case LongPointExactQuery -> {
var longPointExactQuery = (LongPointExactQuery) query;
return LongPoint.newExactQuery(longPointExactQuery.field(), longPointExactQuery.value());
case FloatPointExactQuery:
}
case FloatPointExactQuery -> {
var floatPointExactQuery = (FloatPointExactQuery) query;
return FloatPoint.newExactQuery(floatPointExactQuery.field(), floatPointExactQuery.value());
case DoublePointExactQuery:
}
case DoublePointExactQuery -> {
var doublePointExactQuery = (DoublePointExactQuery) query;
return DoublePoint.newExactQuery(doublePointExactQuery.field(), doublePointExactQuery.value());
case LongNDPointExactQuery:
}
case LongNDPointExactQuery -> {
var longndPointExactQuery = (LongNDPointExactQuery) query;
var longndValues = longndPointExactQuery.value().toLongArray();
return LongPoint.newRangeQuery(longndPointExactQuery.field(), longndValues, longndValues);
case FloatNDPointExactQuery:
}
case FloatNDPointExactQuery -> {
var floatndPointExactQuery = (FloatNDPointExactQuery) query;
var floatndValues = floatndPointExactQuery.value().toFloatArray();
return FloatPoint.newRangeQuery(floatndPointExactQuery.field(), floatndValues, floatndValues);
case DoubleNDPointExactQuery:
}
case DoubleNDPointExactQuery -> {
var doublendPointExactQuery = (DoubleNDPointExactQuery) query;
var doublendValues = doublendPointExactQuery.value().toDoubleArray();
return DoublePoint.newRangeQuery(doublendPointExactQuery.field(), doublendValues, doublendValues);
case IntPointSetQuery:
}
case IntPointSetQuery -> {
var intPointSetQuery = (IntPointSetQuery) query;
return IntPoint.newSetQuery(intPointSetQuery.field(), intPointSetQuery.values().toIntArray());
case LongPointSetQuery:
}
case LongPointSetQuery -> {
var longPointSetQuery = (LongPointSetQuery) query;
return LongPoint.newSetQuery(longPointSetQuery.field(), longPointSetQuery.values().toLongArray());
case FloatPointSetQuery:
}
case FloatPointSetQuery -> {
var floatPointSetQuery = (FloatPointSetQuery) query;
return FloatPoint.newSetQuery(floatPointSetQuery.field(), floatPointSetQuery.values().toFloatArray());
case DoublePointSetQuery:
}
case DoublePointSetQuery -> {
var doublePointSetQuery = (DoublePointSetQuery) query;
return DoublePoint.newSetQuery(doublePointSetQuery.field(), doublePointSetQuery.values().toDoubleArray());
case TermQuery:
}
case TermQuery -> {
var termQuery = (TermQuery) query;
return new org.apache.lucene.search.TermQuery(toTerm(termQuery.term()));
case IntTermQuery:
}
case IntTermQuery -> {
var intTermQuery = (IntTermQuery) query;
return new org.apache.lucene.search.TermQuery(new Term(intTermQuery.field(),
IntPoint.pack(intTermQuery.value())
));
case IntNDTermQuery:
}
case IntNDTermQuery -> {
var intNDTermQuery = (IntNDTermQuery) query;
return new org.apache.lucene.search.TermQuery(new Term(intNDTermQuery.field(),
IntPoint.pack(intNDTermQuery.value().toIntArray())
));
case LongTermQuery:
}
case LongTermQuery -> {
var longTermQuery = (LongTermQuery) query;
return new org.apache.lucene.search.TermQuery(new Term(longTermQuery.field(),
LongPoint.pack(longTermQuery.value())
));
case LongNDTermQuery:
}
case LongNDTermQuery -> {
var longNDTermQuery = (LongNDTermQuery) query;
return new org.apache.lucene.search.TermQuery(new Term(longNDTermQuery.field(),
LongPoint.pack(longNDTermQuery.value().toLongArray())
));
case FloatTermQuery:
}
case FloatTermQuery -> {
var floatTermQuery = (FloatTermQuery) query;
return new org.apache.lucene.search.TermQuery(new Term(floatTermQuery.field(),
FloatPoint.pack(floatTermQuery.value())
));
case FloatNDTermQuery:
}
case FloatNDTermQuery -> {
var floatNDTermQuery = (FloatNDTermQuery) query;
return new org.apache.lucene.search.TermQuery(new Term(floatNDTermQuery.field(),
FloatPoint.pack(floatNDTermQuery.value().toFloatArray())
));
case DoubleTermQuery:
}
case DoubleTermQuery -> {
var doubleTermQuery = (DoubleTermQuery) query;
return new org.apache.lucene.search.TermQuery(new Term(doubleTermQuery.field(),
DoublePoint.pack(doubleTermQuery.value())
));
case DoubleNDTermQuery:
}
case DoubleNDTermQuery -> {
var doubleNDTermQuery = (DoubleNDTermQuery) query;
return new org.apache.lucene.search.TermQuery(new Term(doubleNDTermQuery.field(),
DoublePoint.pack(doubleNDTermQuery.value().toDoubleArray())
));
case FieldExistsQuery:
}
case FieldExistsQuery -> {
var fieldExistQuery = (FieldExistsQuery) query;
return new org.apache.lucene.search.FieldExistsQuery(fieldExistQuery.field());
case BoostQuery:
}
case BoostQuery -> {
var boostQuery = (BoostQuery) query;
return new org.apache.lucene.search.BoostQuery(toQuery(boostQuery.query(), analyzer), boostQuery.scoreBoost());
case ConstantScoreQuery:
}
case ConstantScoreQuery -> {
var constantScoreQuery = (ConstantScoreQuery) query;
return new org.apache.lucene.search.ConstantScoreQuery(toQuery(constantScoreQuery.query(), analyzer));
case BoxedQuery:
}
case BoxedQuery -> {
return toQuery(((BoxedQuery) query).query(), analyzer);
case FuzzyQuery:
}
case FuzzyQuery -> {
var fuzzyQuery = (it.cavallium.dbengine.client.query.current.data.FuzzyQuery) query;
return new FuzzyQuery(toTerm(fuzzyQuery.term()),
fuzzyQuery.maxEdits(),
@ -243,56 +254,67 @@ public class QueryParser {
fuzzyQuery.maxExpansions(),
fuzzyQuery.transpositions()
);
case IntPointRangeQuery:
}
case IntPointRangeQuery -> {
var intPointRangeQuery = (IntPointRangeQuery) query;
return IntPoint.newRangeQuery(intPointRangeQuery.field(), intPointRangeQuery.min(), intPointRangeQuery.max());
case IntNDPointRangeQuery:
}
case IntNDPointRangeQuery -> {
var intndPointRangeQuery = (IntNDPointRangeQuery) query;
return IntPoint.newRangeQuery(intndPointRangeQuery.field(),
intndPointRangeQuery.min().toIntArray(),
intndPointRangeQuery.max().toIntArray()
);
case LongPointRangeQuery:
}
case LongPointRangeQuery -> {
var longPointRangeQuery = (LongPointRangeQuery) query;
return LongPoint.newRangeQuery(longPointRangeQuery.field(),
longPointRangeQuery.min(),
longPointRangeQuery.max()
);
case FloatPointRangeQuery:
}
case FloatPointRangeQuery -> {
var floatPointRangeQuery = (FloatPointRangeQuery) query;
return FloatPoint.newRangeQuery(floatPointRangeQuery.field(),
floatPointRangeQuery.min(),
floatPointRangeQuery.max()
);
case DoublePointRangeQuery:
}
case DoublePointRangeQuery -> {
var doublePointRangeQuery = (DoublePointRangeQuery) query;
return DoublePoint.newRangeQuery(doublePointRangeQuery.field(),
doublePointRangeQuery.min(),
doublePointRangeQuery.max()
);
case LongNDPointRangeQuery:
}
case LongNDPointRangeQuery -> {
var longndPointRangeQuery = (LongNDPointRangeQuery) query;
return LongPoint.newRangeQuery(longndPointRangeQuery.field(),
longndPointRangeQuery.min().toLongArray(),
longndPointRangeQuery.max().toLongArray()
);
case FloatNDPointRangeQuery:
}
case FloatNDPointRangeQuery -> {
var floatndPointRangeQuery = (FloatNDPointRangeQuery) query;
return FloatPoint.newRangeQuery(floatndPointRangeQuery.field(),
floatndPointRangeQuery.min().toFloatArray(),
floatndPointRangeQuery.max().toFloatArray()
);
case DoubleNDPointRangeQuery:
}
case DoubleNDPointRangeQuery -> {
var doublendPointRangeQuery = (DoubleNDPointRangeQuery) query;
return DoublePoint.newRangeQuery(doublendPointRangeQuery.field(),
doublendPointRangeQuery.min().toDoubleArray(),
doublendPointRangeQuery.max().toDoubleArray()
);
case MatchAllDocsQuery:
}
case MatchAllDocsQuery -> {
return new MatchAllDocsQuery();
case MatchNoDocsQuery:
}
case MatchNoDocsQuery -> {
return new MatchNoDocsQuery();
case PhraseQuery:
}
case PhraseQuery -> {
var phraseQuery = (PhraseQuery) query;
var pqb = new org.apache.lucene.search.PhraseQuery.Builder();
for (TermPosition phrase : phraseQuery.phrase()) {
@ -300,27 +322,31 @@ public class QueryParser {
}
pqb.setSlop(phraseQuery.slop());
return pqb.build();
case SortedDocFieldExistsQuery:
}
case SortedDocFieldExistsQuery -> {
var sortedDocFieldExistsQuery = (SortedDocFieldExistsQuery) query;
return new DocValuesFieldExistsQuery(sortedDocFieldExistsQuery.field());
case SynonymQuery:
}
case SynonymQuery -> {
var synonymQuery = (SynonymQuery) query;
var sqb = new org.apache.lucene.search.SynonymQuery.Builder(synonymQuery.field());
for (TermAndBoost part : synonymQuery.parts()) {
sqb.addTerm(toTerm(part.term()), part.boost());
}
return sqb.build();
case SortedNumericDocValuesFieldSlowRangeQuery:
}
case SortedNumericDocValuesFieldSlowRangeQuery -> {
var sortedNumericDocValuesFieldSlowRangeQuery = (SortedNumericDocValuesFieldSlowRangeQuery) query;
return SortedNumericDocValuesField.newSlowRangeQuery(sortedNumericDocValuesFieldSlowRangeQuery.field(),
sortedNumericDocValuesFieldSlowRangeQuery.min(),
sortedNumericDocValuesFieldSlowRangeQuery.max()
);
case WildcardQuery:
}
case WildcardQuery -> {
var wildcardQuery = (WildcardQuery) query;
return new org.apache.lucene.search.WildcardQuery(new Term(wildcardQuery.field(), wildcardQuery.pattern()));
default:
throw new IllegalStateException("Unexpected value: " + query.getBaseType$());
}
default -> throw new IllegalStateException("Unexpected value: " + query.getBaseType$());
}
}

27
src/main/java/it/cavallium/dbengine/client/query/QueryUtils.java
View File

@ -17,7 +17,6 @@ import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.util.QueryBuilder;
import org.jetbrains.annotations.NotNull;
@ -63,29 +62,17 @@ public class QueryUtils {
for (BooleanClause booleanClause : booleanQuery) {
org.apache.lucene.search.Query queryPartQuery = booleanClause.getQuery();
Occur occur;
switch (booleanClause.getOccur()) {
case MUST:
occur = OccurMust.of();
break;
case FILTER:
occur = OccurFilter.of();
break;
case SHOULD:
occur = OccurShould.of();
break;
case MUST_NOT:
occur = OccurMustNot.of();
break;
default:
throw new IllegalArgumentException();
}
Occur occur = switch (booleanClause.getOccur()) {
case MUST -> OccurMust.of();
case FILTER -> OccurFilter.of();
case SHOULD -> OccurShould.of();
case MUST_NOT -> OccurMustNot.of();
};
queryParts.add(BooleanQueryPart.of(transformQuery(field, queryPartQuery), occur));
}
return BooleanQuery.of(List.copyOf(queryParts), booleanQuery.getMinimumNumberShouldMatch());
}
if (luceneQuery instanceof org.apache.lucene.search.PhraseQuery) {
var phraseQuery = (org.apache.lucene.search.PhraseQuery) luceneQuery;
if (luceneQuery instanceof org.apache.lucene.search.PhraseQuery phraseQuery) {
int slop = phraseQuery.getSlop();
var terms = phraseQuery.getTerms();
var positions = phraseQuery.getPositions();

62
src/main/java/it/cavallium/dbengine/database/BufSupplier.java
View File

@ -1,62 +0,0 @@
package it.cavallium.dbengine.database;
import io.netty5.buffer.Buffer;
import io.netty5.util.Send;
import java.util.function.Supplier;
public abstract class BufSupplier implements SafeCloseable, DiscardingCloseable, Supplier<Buffer> {
public static BufSupplier of(Supplier<Buffer> supplier) {
return new SimpleBufSupplier(supplier);
}
public static BufSupplier of(Send<Buffer> supplier) {
return new CopyBufSupplier(supplier.receive());
}
public static BufSupplier ofOwned(Buffer supplier) {
return new CopyBufSupplier(supplier);
}
public static BufSupplier ofShared(Buffer supplier) {
return new SimpleBufSupplier(() -> supplier.copy());
}
private static final class SimpleBufSupplier extends BufSupplier {
private final Supplier<Buffer> supplier;
public SimpleBufSupplier(Supplier<Buffer> supplier) {
this.supplier = supplier;
}
@Override
public Buffer get() {
return supplier.get();
}
@Override
public void close() {
}
}
private static final class CopyBufSupplier extends BufSupplier {
private final Buffer supplier;
public CopyBufSupplier(Buffer supplier) {
this.supplier = supplier;
}
@Override
public Buffer get() {
return supplier.copy();
}
@Override
public void close() {
supplier.close();
}
}
}

5
src/main/java/it/cavallium/dbengine/database/DatabaseOperations.java
View File

@ -2,10 +2,9 @@ package it.cavallium.dbengine.database;
import it.cavallium.dbengine.rpc.current.data.Column;
import java.nio.file.Path;
import org.reactivestreams.Publisher;
import reactor.core.publisher.Mono;
import java.util.stream.Stream;
public interface DatabaseOperations {
Mono<Void> ingestSST(Column column, Publisher<Path> files, boolean replaceExisting);
void ingestSST(Column column, Stream<Path> files, boolean replaceExisting);
}

24
src/main/java/it/cavallium/dbengine/database/DatabaseProperties.java
View File

@ -2,30 +2,30 @@ package it.cavallium.dbengine.database;
import it.cavallium.dbengine.client.MemoryStats;
import it.cavallium.dbengine.rpc.current.data.Column;
import java.io.IOException;
import java.util.Map;
import java.util.stream.Stream;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public interface DatabaseProperties {
Mono<MemoryStats> getMemoryStats();
MemoryStats getMemoryStats();
Mono<String> getRocksDBStats();
String getRocksDBStats();
Mono<Map<String, String>> getMapProperty(@Nullable Column column, RocksDBMapProperty property);
Map<String, String> getMapProperty(@Nullable Column column, RocksDBMapProperty property);
Flux<ColumnProperty<Map<String, String>>> getMapColumnProperties(RocksDBMapProperty property);
Stream<ColumnProperty<Map<String, String>>> getMapColumnProperties(RocksDBMapProperty property);
Mono<String> getStringProperty(@Nullable Column column, RocksDBStringProperty property);
String getStringProperty(@Nullable Column column, RocksDBStringProperty property);
Flux<ColumnProperty<String>> getStringColumnProperties(RocksDBStringProperty property);
Stream<ColumnProperty<String>> getStringColumnProperties(RocksDBStringProperty property);
Mono<Long> getLongProperty(@Nullable Column column, RocksDBLongProperty property);
Long getLongProperty(@Nullable Column column, RocksDBLongProperty property);
Flux<ColumnProperty<Long>> getLongColumnProperties(RocksDBLongProperty property);
Stream<ColumnProperty<Long>> getLongColumnProperties(RocksDBLongProperty property);
Mono<Long> getAggregatedLongProperty(RocksDBLongProperty property);
Long getAggregatedLongProperty(RocksDBLongProperty property);
Flux<TableWithProperties> getTableProperties();
Stream<TableWithProperties> getTableProperties();
}

15
src/main/java/it/cavallium/dbengine/database/LLDatabaseConnection.java
View File

@ -1,39 +1,34 @@
package it.cavallium.dbengine.database;
import io.micrometer.core.instrument.MeterRegistry;
import io.netty5.buffer.BufferAllocator;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneRocksDBManager;
import it.cavallium.dbengine.rpc.current.data.Column;
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
import it.cavallium.dbengine.rpc.current.data.IndicizerAnalyzers;
import it.cavallium.dbengine.rpc.current.data.IndicizerSimilarities;
import it.cavallium.dbengine.rpc.current.data.LuceneIndexStructure;
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
import java.io.IOException;
import java.util.List;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
@SuppressWarnings("UnusedReturnValue")
public interface LLDatabaseConnection {
BufferAllocator getAllocator();
MeterRegistry getMeterRegistry();
Mono<? extends LLDatabaseConnection> connect();
LLDatabaseConnection connect();
Mono<? extends LLKeyValueDatabase> getDatabase(String name,
LLKeyValueDatabase getDatabase(String name,
List<Column> columns,
DatabaseOptions databaseOptions);
Mono<? extends LLLuceneIndex> getLuceneIndex(String clusterName,
LLLuceneIndex getLuceneIndex(String clusterName,
LuceneIndexStructure indexStructure,
IndicizerAnalyzers indicizerAnalyzers,
IndicizerSimilarities indicizerSimilarities,
LuceneOptions luceneOptions,
@Nullable LuceneHacks luceneHacks);
Mono<Void> disconnect();
void disconnect();
}

64
src/main/java/it/cavallium/dbengine/database/LLDelta.java
View File

@ -1,71 +1,37 @@
package it.cavallium.dbengine.database;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.Drop;
import io.netty5.buffer.Owned;
import io.netty5.util.Send;
import io.netty5.buffer.internal.ResourceSupport;
import it.cavallium.dbengine.utils.SimpleResource;
import static it.cavallium.dbengine.database.LLUtils.unmodifiableBytes;
import it.cavallium.dbengine.buffers.Buf;
import java.util.StringJoiner;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
public class LLDelta extends SimpleResource implements DiscardingCloseable {
public class LLDelta {
@Nullable
private final Buffer previous;
private final Buf previous;
@Nullable
private final Buffer current;
private final Buf current;
private LLDelta(@Nullable Buffer previous, @Nullable Buffer current) {
private LLDelta(@Nullable Buf previous, @Nullable Buf current) {
super();
this.previous = previous != null ? previous.makeReadOnly() : null;
this.current = current != null ? current.makeReadOnly() : null;
this.previous = unmodifiableBytes(previous);
this.current = unmodifiableBytes(current);
}
@Override
protected void ensureOpen() {
super.ensureOpen();
assert previous == null || previous.isAccessible();
assert current == null || current.isAccessible();
}
@Override
protected void onClose() {
if (previous != null && previous.isAccessible()) {
previous.close();
}
if (current != null && current.isAccessible()) {
current.close();
}
}
public static LLDelta of(Buffer previous, Buffer current) {
public static LLDelta of(Buf previous, Buf current) {
assert (previous == null && current == null) || (previous != current);
return new LLDelta(previous, current);
}
public Send<Buffer> previous() {
ensureOpen();
return previous != null ? previous.copy().send() : null;
}
public Send<Buffer> current() {
ensureOpen();
return current != null ? current.copy().send() : null;
}
public Buffer currentUnsafe() {
ensureOpen();
return current;
}
public Buffer previousUnsafe() {
ensureOpen();
public Buf previous() {
return previous;
}
public Buf current() {
return current;
}
public boolean isModified() {
return !LLUtils.equals(previous, current);
}

96
src/main/java/it/cavallium/dbengine/database/LLDictionary.java
View File

@ -1,105 +1,93 @@
package it.cavallium.dbengine.database;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.BufferAllocator;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.client.BadBlock;
import it.cavallium.dbengine.database.disk.BinarySerializationFunction;
import it.cavallium.dbengine.database.serialization.KVSerializationFunction;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Stream;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import org.rocksdb.RocksDBException;
@SuppressWarnings("unused")
public interface LLDictionary extends LLKeyValueDatabaseStructure {
String getColumnName();
BufferAllocator getAllocator();
Buf get(@Nullable LLSnapshot snapshot, Buf key);
Mono<Buffer> get(@Nullable LLSnapshot snapshot, Mono<Buffer> key);
Mono<Buffer> put(Mono<Buffer> key, Mono<Buffer> value, LLDictionaryResultType resultType);
Buf put(Buf key, Buf value, LLDictionaryResultType resultType);
UpdateMode getUpdateMode();
default Mono<Buffer> update(Mono<Buffer> key,
BinarySerializationFunction updater,
UpdateReturnMode updateReturnMode) {
return this
.updateAndGetDelta(key, updater)
.transform(prev -> LLUtils.resolveLLDelta(prev, updateReturnMode));
default Buf update(Buf key, BinarySerializationFunction updater, UpdateReturnMode updateReturnMode) {
LLDelta prev = this.updateAndGetDelta(key, updater);
return LLUtils.resolveLLDelta(prev, updateReturnMode);
}
Mono<LLDelta> updateAndGetDelta(Mono<Buffer> key, BinarySerializationFunction updater);
LLDelta updateAndGetDelta(Buf key, BinarySerializationFunction updater);
Mono<Void> clear();
void clear();
Mono<Buffer> remove(Mono<Buffer> key, LLDictionaryResultType resultType);
Buf remove(Buf key, LLDictionaryResultType resultType);
Flux<OptionalBuf> getMulti(@Nullable LLSnapshot snapshot, Flux<Buffer> keys);
Stream<OptionalBuf> getMulti(@Nullable LLSnapshot snapshot, Stream<Buf> keys);
Mono<Void> putMulti(Flux<LLEntry> entries);
void putMulti(Stream<LLEntry> entries);
<K> Flux<Boolean> updateMulti(Flux<K> keys, Flux<Buffer> serializedKeys,
KVSerializationFunction<K, @Nullable Buffer, @Nullable Buffer> updateFunction);
<K> Stream<Boolean> updateMulti(Stream<K> keys, Stream<Buf> serializedKeys,
KVSerializationFunction<K, @Nullable Buf, @Nullable Buf> updateFunction);
Flux<LLEntry> getRange(@Nullable LLSnapshot snapshot,
Mono<LLRange> range,
Stream<LLEntry> getRange(@Nullable LLSnapshot snapshot,
LLRange range,
boolean reverse,
boolean smallRange);
Flux<List<LLEntry>> getRangeGrouped(@Nullable LLSnapshot snapshot,
Mono<LLRange> range,
Stream<List<LLEntry>> getRangeGrouped(@Nullable LLSnapshot snapshot,
LLRange range,
int prefixLength,
boolean smallRange);
Flux<Buffer> getRangeKeys(@Nullable LLSnapshot snapshot,
Mono<LLRange> range,
Stream<Buf> getRangeKeys(@Nullable LLSnapshot snapshot,
LLRange range,
boolean reverse,
boolean smallRange);
boolean smallRange) throws RocksDBException, IOException;
Flux<List<Buffer>> getRangeKeysGrouped(@Nullable LLSnapshot snapshot,
Mono<LLRange> range,
Stream<List<Buf>> getRangeKeysGrouped(@Nullable LLSnapshot snapshot,
LLRange range,
int prefixLength,
boolean smallRange);
Flux<Buffer> getRangeKeyPrefixes(@Nullable LLSnapshot snapshot,
Mono<LLRange> range,
Stream<Buf> getRangeKeyPrefixes(@Nullable LLSnapshot snapshot,
LLRange range,
int prefixLength,
boolean smallRange);
Flux<BadBlock> badBlocks(Mono<LLRange> range);
Stream<BadBlock> badBlocks(LLRange range);
Mono<Void> setRange(Mono<LLRange> range, Flux<LLEntry> entries, boolean smallRange);
void setRange(LLRange range, Stream<LLEntry> entries, boolean smallRange);
default Mono<Void> replaceRange(Mono<LLRange> range,
default void replaceRange(LLRange range,
boolean canKeysChange,
Function<LLEntry, Mono<LLEntry>> entriesReplacer,
Function<@NotNull LLEntry, @NotNull LLEntry> entriesReplacer,
boolean smallRange) {
return Mono.defer(() -> {
if (canKeysChange) {
return this
.setRange(range, this
.getRange(null, range, false, smallRange)
.flatMap(entriesReplacer), smallRange);
} else {
return this.putMulti(this.getRange(null, range, false, smallRange).flatMap(entriesReplacer));
}
});
if (canKeysChange) {
this.setRange(range, this.getRange(null, range, false, smallRange).map(entriesReplacer), smallRange);
} else {
this.putMulti(this.getRange(null, range, false, smallRange).map(entriesReplacer));
}
}
Mono<Boolean> isRangeEmpty(@Nullable LLSnapshot snapshot, Mono<LLRange> range, boolean fillCache);
boolean isRangeEmpty(@Nullable LLSnapshot snapshot, LLRange range, boolean fillCache);
Mono<Long> sizeRange(@Nullable LLSnapshot snapshot, Mono<LLRange> range, boolean fast);
long sizeRange(@Nullable LLSnapshot snapshot, LLRange range, boolean fast);
Mono<LLEntry> getOne(@Nullable LLSnapshot snapshot, Mono<LLRange> range);
LLEntry getOne(@Nullable LLSnapshot snapshot, LLRange range);
Mono<Buffer> getOneKey(@Nullable LLSnapshot snapshot, Mono<LLRange> range);
Buf getOneKey(@Nullable LLSnapshot snapshot, LLRange range);
Mono<LLEntry> removeOne(Mono<LLRange> range);
LLEntry removeOne(LLRange range);
}

74
src/main/java/it/cavallium/dbengine/database/LLEntry.java
View File

@ -1,67 +1,37 @@
package it.cavallium.dbengine.database;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.Drop;
import io.netty5.buffer.Owned;
import io.netty5.util.Resource;
import io.netty5.util.Send;
import io.netty5.buffer.internal.ResourceSupport;
import it.cavallium.dbengine.utils.SimpleResource;
import it.cavallium.dbengine.buffers.Buf;
import java.util.Objects;
import java.util.StringJoiner;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public class LLEntry extends SimpleResource implements DiscardingCloseable {
public class LLEntry {
private static final Logger logger = LogManager.getLogger(LLEntry.class);
private Buffer key;
private Buffer value;
private final Buf key;
private final Buf value;
private LLEntry(@NotNull Send<Buffer> key, @NotNull Send<Buffer> value) {
this.key = key.receive();
this.value = value.receive();
assert isAllAccessible();
}
private LLEntry(@NotNull Buffer key, @NotNull Buffer value) {
private LLEntry(@NotNull Buf key, @NotNull Buf value) {
this.key = key;
this.value = value;
assert isAllAccessible();
}
private boolean isAllAccessible() {
assert key != null && key.isAccessible();
assert value != null && value.isAccessible();
return true;
}
public static LLEntry of(@NotNull Buffer key, @NotNull Buffer value) {
public static LLEntry of(@NotNull Buf key, @NotNull Buf value) {
return new LLEntry(key, value);
}
public Send<Buffer> getKey() {
ensureOwned();
return Objects.requireNonNull(key).copy().send();
public static LLEntry copyOf(Buf keyView, Buf valueView) {
return new LLEntry(keyView.copy(), valueView.copy());
}
public Buffer getKeyUnsafe() {
return key;
public Buf getKey() {
return Objects.requireNonNull(key);
}
public Send<Buffer> getValue() {
ensureOwned();
return Objects.requireNonNull(value).copy().send();
}
public Buffer getValueUnsafe() {
return value;
}
private void ensureOwned() {
assert isAllAccessible();
public Buf getValue() {
return Objects.requireNonNull(value);
}
@Override
@ -90,24 +60,4 @@ public class LLEntry extends SimpleResource implements DiscardingCloseable {
.add("value=" + LLUtils.toString(value))
.toString();
}
@Override
protected void onClose() {
try {
if (key != null && key.isAccessible()) {
key.close();
}
} catch (Throwable ex) {
logger.error("Failed to close key", ex);
}
try {
if (value != null && value.isAccessible()) {
value.close();
}
} catch (Throwable ex) {
logger.error("Failed to close value", ex);
}
key = null;
value = null;
}
}

7
src/main/java/it/cavallium/dbengine/database/LLItem.java
View File

@ -1,13 +1,6 @@
package it.cavallium.dbengine.database;
import com.google.common.primitives.Floats;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.FloatBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Objects;
import java.util.StringJoiner;
import org.apache.lucene.document.Field;

4
src/main/java/it/cavallium/dbengine/database/LLKeyScore.java
View File

@ -1,10 +1,6 @@
package it.cavallium.dbengine.database;
import java.util.Objects;
import java.util.StringJoiner;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.BytesRef;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
public record LLKeyScore(int docId, int shardId, float score, @Nullable IndexableField key) {}

62
src/main/java/it/cavallium/dbengine/database/LLKeyValueDatabase.java
View File

@ -3,69 +3,67 @@ package it.cavallium.dbengine.database;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import io.micrometer.core.instrument.MeterRegistry;
import io.netty5.buffer.BufferAllocator;
import it.cavallium.dbengine.client.IBackuppable;
import it.cavallium.dbengine.client.MemoryStats;
import it.cavallium.dbengine.database.collections.DatabaseInt;
import it.cavallium.dbengine.database.collections.DatabaseLong;
import it.cavallium.dbengine.rpc.current.data.Column;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import org.rocksdb.RocksDBException;
public interface LLKeyValueDatabase extends LLSnapshottable, LLKeyValueDatabaseStructure, DatabaseProperties,
IBackuppable, DatabaseOperations {
Mono<? extends LLSingleton> getSingleton(byte[] singletonListColumnName, byte[] name, byte @Nullable[] defaultValue);
LLSingleton getSingleton(byte[] singletonListColumnName, byte[] name, byte @Nullable [] defaultValue)
throws IOException;
Mono<? extends LLDictionary> getDictionary(byte[] columnName, UpdateMode updateMode);
LLDictionary getDictionary(byte[] columnName, UpdateMode updateMode);
@Deprecated
default Mono<? extends LLDictionary> getDeprecatedSet(String name, UpdateMode updateMode) {
default LLDictionary getDeprecatedSet(String name, UpdateMode updateMode) {
return getDictionary(ColumnUtils.deprecatedSet(name).name().getBytes(StandardCharsets.US_ASCII), updateMode);
}
default Mono<? extends LLDictionary> getDictionary(String name, UpdateMode updateMode) {
default LLDictionary getDictionary(String name, UpdateMode updateMode) {
return getDictionary(ColumnUtils.dictionary(name).name().getBytes(StandardCharsets.US_ASCII), updateMode);
}
default Mono<? extends LLSingleton> getSingleton(String singletonListName, String name) {
default LLSingleton getSingleton(String singletonListName, String name) {
return getSingleton(ColumnUtils.special(singletonListName).name().getBytes(StandardCharsets.US_ASCII),
name.getBytes(StandardCharsets.US_ASCII),
null
);
}
default Mono<DatabaseInt> getInteger(String singletonListName, String name, int defaultValue) {
return this
.getSingleton(ColumnUtils.special(singletonListName).name().getBytes(StandardCharsets.US_ASCII),
name.getBytes(StandardCharsets.US_ASCII),
Ints.toByteArray(defaultValue)
)
.map(DatabaseInt::new);
default DatabaseInt getInteger(String singletonListName, String name, int defaultValue) {
return new DatabaseInt(this.getSingleton(ColumnUtils
.special(singletonListName)
.name()
.getBytes(StandardCharsets.US_ASCII),
name.getBytes(StandardCharsets.US_ASCII),
Ints.toByteArray(defaultValue)
));
}
default Mono<DatabaseLong> getLong(String singletonListName, String name, long defaultValue) {
return this
.getSingleton(ColumnUtils.special(singletonListName).name().getBytes(StandardCharsets.US_ASCII),
name.getBytes(StandardCharsets.US_ASCII),
Longs.toByteArray(defaultValue)
)
.map(DatabaseLong::new);
default DatabaseLong getLong(String singletonListName, String name, long defaultValue) {
return new DatabaseLong(this.getSingleton(ColumnUtils
.special(singletonListName)
.name()
.getBytes(StandardCharsets.US_ASCII),
name.getBytes(StandardCharsets.US_ASCII),
Longs.toByteArray(defaultValue)
));
}
Mono<Void> verifyChecksum();
void verifyChecksum();
Mono<Void> compact();
void compact() throws RocksDBException;
Mono<Void> flush();
BufferAllocator getAllocator();
void flush();
MeterRegistry getMeterRegistry();
Mono<Void> preClose();
Mono<Void> close();
void preClose();
void close();
}

44
src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java
View File

@ -8,30 +8,29 @@ import it.cavallium.dbengine.client.query.current.data.QueryParams;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.lucene.collector.Buckets;
import it.cavallium.dbengine.lucene.searcher.BucketParams;
import java.io.IOException;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.stream.Stream;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public interface LLLuceneIndex extends LLSnapshottable, IBackuppable, SafeCloseable {
String getLuceneIndexName();
Mono<Void> addDocument(LLTerm id, LLUpdateDocument doc);
void addDocument(LLTerm id, LLUpdateDocument doc);
Mono<Long> addDocuments(boolean atomic, Flux<Entry<LLTerm, LLUpdateDocument>> documents);
long addDocuments(boolean atomic, Stream<Entry<LLTerm, LLUpdateDocument>> documents);
Mono<Void> deleteDocument(LLTerm id);
void deleteDocument(LLTerm id);
Mono<Void> update(LLTerm id, LLIndexRequest request);
void update(LLTerm id, LLIndexRequest request);
Mono<Long> updateDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents);
long updateDocuments(Stream<Entry<LLTerm, LLUpdateDocument>> documents);
Mono<Void> deleteAll();
void deleteAll();
/**
* @param queryParams the limit is valid for each lucene instance. If you have 15 instances, the number of elements
@ -40,7 +39,7 @@ public interface LLLuceneIndex extends LLSnapshottable, IBackuppable, SafeClosea
* The additional query will be used with the moreLikeThis query: "mltQuery AND additionalQuery"
* @return the collection has one or more flux
*/
Flux<LLSearchResultShard> moreLikeThis(@Nullable LLSnapshot snapshot,
Stream<LLSearchResultShard> moreLikeThis(@Nullable LLSnapshot snapshot,
QueryParams queryParams,
@Nullable String keyFieldName,
Multimap<String, String> mltDocumentFields);
@ -50,19 +49,19 @@ public interface LLLuceneIndex extends LLSnapshottable, IBackuppable, SafeClosea
* returned can be at most <code>limit * 15</code>
* @return the collection has one or more flux
*/
Flux<LLSearchResultShard> search(@Nullable LLSnapshot snapshot,
Stream<LLSearchResultShard> search(@Nullable LLSnapshot snapshot,
QueryParams queryParams,
@Nullable String keyFieldName);
/**
* @return buckets with each value collected into one of the buckets
*/
Mono<Buckets> computeBuckets(@Nullable LLSnapshot snapshot,
Buckets computeBuckets(@Nullable LLSnapshot snapshot,
@NotNull List<Query> queries,
@Nullable Query normalizationQuery,
BucketParams bucketParams);
default Mono<TotalHitsCount> count(@Nullable LLSnapshot snapshot, Query query, @Nullable Duration timeout) {
default TotalHitsCount count(@Nullable LLSnapshot snapshot, Query query, @Nullable Duration timeout) {
QueryParams params = QueryParams.of(query,
0,
0,
@ -70,12 +69,11 @@ public interface LLLuceneIndex extends LLSnapshottable, IBackuppable, SafeClosea
false,
timeout == null ? Long.MAX_VALUE : timeout.toMillis()
);
return Mono
.usingWhen(this.search(snapshot, params, null).singleOrEmpty(),
llSearchResultShard -> Mono.just(llSearchResultShard.totalHitsCount()),
LLUtils::finalizeResource
)
.defaultIfEmpty(TotalHitsCount.of(0, true));
return this
.search(snapshot, params, null)
.parallel()
.map(LLSearchResultShard::totalHitsCount)
.reduce(TotalHitsCount.of(0, true), (a, b) -> TotalHitsCount.of(a.value() + b.value(), a.exact() && b.exact()));
}
boolean isLowMemoryMode();
@ -84,18 +82,18 @@ public interface LLLuceneIndex extends LLSnapshottable, IBackuppable, SafeClosea
* Flush writes to disk.
* This does not commit, it syncs the data to the disk
*/
Mono<Void> flush();
void flush();
Mono<Void> waitForMerges();
void waitForMerges();
/**
* Wait for the latest pending merge
* This disables future merges until shutdown!
*/
Mono<Void> waitForLastMerges();
void waitForLastMerges();
/**
* Refresh index searcher
*/
Mono<Void> refresh(boolean force);
void refresh(boolean force);
}

112
src/main/java/it/cavallium/dbengine/database/LLMultiDatabaseConnection.java
View File

@ -2,14 +2,10 @@ package it.cavallium.dbengine.database;
import com.google.common.collect.Multimap;
import io.micrometer.core.instrument.MeterRegistry;
import io.netty5.buffer.BufferAllocator;
import it.cavallium.dbengine.client.ConnectionSettings.ConnectionPart;
import it.cavallium.dbengine.client.ConnectionSettings.ConnectionPart.ConnectionPartLucene;
import it.cavallium.dbengine.client.ConnectionSettings.ConnectionPart.ConnectionPartRocksDB;
import it.cavallium.dbengine.client.IndicizerAnalyzers;
import it.cavallium.dbengine.client.IndicizerSimilarities;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneRocksDBManager;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.rpc.current.data.Column;
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
@ -18,6 +14,7 @@ import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntSet;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -25,12 +22,10 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletionException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.util.function.Tuple2;
public class LLMultiDatabaseConnection implements LLDatabaseConnection {
@ -83,30 +78,26 @@ public class LLMultiDatabaseConnection implements LLDatabaseConnection {
allConnections.addAll(databaseShardConnections.values());
}
@Override
public BufferAllocator getAllocator() {
return anyConnection.getAllocator();
}
@Override
public MeterRegistry getMeterRegistry() {
return anyConnection.getMeterRegistry();
}
@Override
public Mono<? extends LLDatabaseConnection> connect() {
return Flux
.fromIterable(allConnections)
.flatMap((LLDatabaseConnection databaseConnection) -> databaseConnection
.connect()
.doOnError(ex -> LOG.error("Failed to open connection", ex))
)
.then()
.thenReturn(this);
public LLDatabaseConnection connect() {
// todo: parallelize?
for (LLDatabaseConnection connection : allConnections) {
try {
connection.connect();
} catch (Exception ex) {
LOG.error("Failed to open connection", ex);
}
}
return this;
}
@Override
public Mono<? extends LLKeyValueDatabase> getDatabase(String name,
public LLKeyValueDatabase getDatabase(String name,
List<Column> columns,
DatabaseOptions databaseOptions) {
var conn = databaseShardConnections.getOrDefault(name, defaultDatabaseConnection);
@ -115,7 +106,7 @@ public class LLMultiDatabaseConnection implements LLDatabaseConnection {
}
@Override
public Mono<? extends LLLuceneIndex> getLuceneIndex(String clusterName,
public LLLuceneIndex getLuceneIndex(String clusterName,
LuceneIndexStructure indexStructure,
it.cavallium.dbengine.rpc.current.data.IndicizerAnalyzers indicizerAnalyzers,
it.cavallium.dbengine.rpc.current.data.IndicizerSimilarities indicizerSimilarities,
@ -150,51 +141,44 @@ public class LLMultiDatabaseConnection implements LLDatabaseConnection {
luceneHacks
);
} else {
return Flux
.fromIterable(connectionToShardMap.entrySet())
.flatMap(entry -> {
var connectionIndexStructure = indexStructure
.setActiveShards(new IntArrayList(entry.getValue()));
record ShardToIndex(int shard, LLLuceneIndex connIndex) {}
var indices = connectionToShardMap.entrySet().stream().flatMap(entry -> {
var connectionIndexStructure = indexStructure.setActiveShards(new IntArrayList(entry.getValue()));
Flux<LLLuceneIndex> connIndex = entry.getKey()
.getLuceneIndex(clusterName,
connectionIndexStructure,
indicizerAnalyzers,
indicizerSimilarities,
luceneOptions,
luceneHacks
).cast(LLLuceneIndex.class).cache().repeat();
return Flux
.fromIterable(entry.getValue())
.zipWith(connIndex);
})
.collectList()
.map(indices -> {
var luceneIndices = new LLLuceneIndex[indexStructure.totalShards()];
for (var index : indices) {
luceneIndices[index.getT1()] = index.getT2();
}
return new LLMultiLuceneIndex(clusterName,
indexStructure,
indicizerAnalyzers,
indicizerSimilarities,
luceneOptions,
luceneHacks,
luceneIndices
);
});
LLLuceneIndex connIndex;
try {
connIndex = entry.getKey().getLuceneIndex(clusterName, connectionIndexStructure,
indicizerAnalyzers, indicizerSimilarities, luceneOptions, luceneHacks);
} catch (IOException e) {
throw new CompletionException(e);
}
return entry.getValue().intStream().mapToObj(shard -> new ShardToIndex(shard, connIndex));
}).toList();
var luceneIndices = new LLLuceneIndex[indexStructure.totalShards()];
for (var index : indices) {
luceneIndices[index.shard] = index.connIndex;
}
return new LLMultiLuceneIndex(clusterName,
indexStructure,
indicizerAnalyzers,
indicizerSimilarities,
luceneOptions,
luceneHacks,
luceneIndices
);
}
}
@Override
public Mono<Void> disconnect() {
return Flux
.fromIterable(allConnections)
.flatMap(databaseConnection -> databaseConnection
.disconnect()
.doOnError(ex -> LOG.error("Failed to close connection", ex))
.onErrorResume(ex -> Mono.empty())
)
.then();
public void disconnect() {
// todo: parallelize?
for (LLDatabaseConnection connection : allConnections) {
try {
connection.disconnect();
} catch (Exception ex) {
LOG.error("Failed to close connection", ex);
}
}
}
}

203
src/main/java/it/cavallium/dbengine/database/LLMultiLuceneIndex.java
View File

@ -1,35 +1,31 @@
package it.cavallium.dbengine.database;
import com.google.common.collect.Iterables;
import static it.cavallium.dbengine.lucene.LuceneUtils.getLuceneIndexId;
import static java.util.stream.Collectors.groupingBy;
import com.google.common.collect.Multimap;
import it.cavallium.dbengine.client.IBackuppable;
import it.cavallium.dbengine.rpc.current.data.IndicizerAnalyzers;
import it.cavallium.dbengine.rpc.current.data.IndicizerSimilarities;
import it.cavallium.dbengine.client.query.current.data.Query;
import it.cavallium.dbengine.client.query.current.data.QueryParams;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.collector.Buckets;
import it.cavallium.dbengine.lucene.searcher.BucketParams;
import it.cavallium.dbengine.rpc.current.data.IndicizerAnalyzers;
import it.cavallium.dbengine.rpc.current.data.IndicizerSimilarities;
import it.cavallium.dbengine.rpc.current.data.LuceneIndexStructure;
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
import it.unimi.dsi.fastutil.doubles.DoubleArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.reactivestreams.Publisher;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.publisher.SignalType;
public class LLMultiLuceneIndex implements LLLuceneIndex {
@ -46,7 +42,6 @@ public class LLMultiLuceneIndex implements LLLuceneIndex {
private final LLLuceneIndex[] luceneIndicesById;
private final List<LLLuceneIndex> luceneIndicesSet;
private final int totalShards;
private final Flux<LLLuceneIndex> luceneIndicesFlux;
public LLMultiLuceneIndex(String clusterName,
LuceneIndexStructure indexStructure,
@ -70,7 +65,6 @@ public class LLMultiLuceneIndex implements LLLuceneIndex {
}
}
this.luceneIndicesSet = new ArrayList<>(luceneIndicesSet);
this.luceneIndicesFlux = Flux.fromIterable(luceneIndicesSet);
}
@Override
@ -79,108 +73,115 @@ public class LLMultiLuceneIndex implements LLLuceneIndex {
}
private LLLuceneIndex getLuceneIndex(LLTerm id) {
return luceneIndicesById[LuceneUtils.getLuceneIndexId(id, totalShards)];
return luceneIndicesById[getLuceneIndexId(id, totalShards)];
}
@Override
public Mono<Void> addDocument(LLTerm id, LLUpdateDocument doc) {
return getLuceneIndex(id).addDocument(id, doc);
public void addDocument(LLTerm id, LLUpdateDocument doc) {
getLuceneIndex(id).addDocument(id, doc);
}
@Override
public Mono<Long> addDocuments(boolean atomic, Flux<Entry<LLTerm, LLUpdateDocument>> documents) {
return documents
.groupBy(term -> LuceneUtils.getLuceneIndexId(term.getKey(), totalShards))
.flatMap(group -> {
var index = luceneIndicesById[group.key()];
return index.addDocuments(atomic, group);
})
public long addDocuments(boolean atomic, Stream<Entry<LLTerm, LLUpdateDocument>> documents) {
var groupedRequests = documents
.collect(groupingBy(term -> getLuceneIndexId(term.getKey(), totalShards),
Int2ObjectOpenHashMap::new,
Collectors.toList()
));
return groupedRequests
.int2ObjectEntrySet()
.stream()
.map(entry -> luceneIndicesById[entry.getIntKey()].addDocuments(atomic, entry.getValue().stream()))
.reduce(0L, Long::sum);
}
@Override
public Mono<Void> deleteDocument(LLTerm id) {
return getLuceneIndex(id).deleteDocument(id);
public void deleteDocument(LLTerm id) {
getLuceneIndex(id).deleteDocument(id);
}
@Override
public Mono<Void> update(LLTerm id, LLIndexRequest request) {
return getLuceneIndex(id).update(id, request);
public void update(LLTerm id, LLIndexRequest request) {
getLuceneIndex(id).update(id, request);
}
@Override
public Mono<Long> updateDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents) {
return documents
.log("multi-update-documents", Level.FINEST, false, SignalType.ON_NEXT, SignalType.ON_COMPLETE)
.groupBy(term -> getLuceneIndex(term.getKey()))
.flatMap(groupFlux -> groupFlux.key().updateDocuments(groupFlux))
public long updateDocuments(Stream<Entry<LLTerm, LLUpdateDocument>> documents) {
var groupedRequests = documents
.collect(groupingBy(term -> getLuceneIndexId(term.getKey(), totalShards),
Int2ObjectOpenHashMap::new,
Collectors.toList()
));
return groupedRequests
.int2ObjectEntrySet()
.stream()
.map(entry -> luceneIndicesById[entry.getIntKey()].updateDocuments(entry.getValue().stream()))
.reduce(0L, Long::sum);
}
@Override
public Mono<Void> deleteAll() {
Iterable<Mono<Void>> it = () -> luceneIndicesSet.stream().map(llLuceneIndex -> llLuceneIndex.deleteAll()).iterator();
return Mono.whenDelayError(it);
public void deleteAll() {
luceneIndicesSet.forEach(LLLuceneIndex::deleteAll);
}
@Override
public Flux<LLSearchResultShard> moreLikeThis(@Nullable LLSnapshot snapshot,
public Stream<LLSearchResultShard> moreLikeThis(@Nullable LLSnapshot snapshot,
QueryParams queryParams,
@Nullable String keyFieldName,
Multimap<String, String> mltDocumentFields) {
return luceneIndicesFlux.flatMap(luceneIndex -> luceneIndex.moreLikeThis(snapshot,
return luceneIndicesSet.parallelStream().flatMap(luceneIndex -> luceneIndex.moreLikeThis(snapshot,
queryParams,
keyFieldName,
mltDocumentFields
)).doOnDiscard(DiscardingCloseable.class, LLUtils::onDiscard);
));
}
private Mono<Buckets> mergeShards(List<Buckets> shards) {
return Mono.fromCallable(() -> {
List<DoubleArrayList> seriesValues = new ArrayList<>();
DoubleArrayList totals = new DoubleArrayList(shards.get(0).totals());
private Buckets mergeShards(List<Buckets> shards) {
List<DoubleArrayList> seriesValues = new ArrayList<>();
DoubleArrayList totals = new DoubleArrayList(shards.get(0).totals());
for (Buckets shard : shards) {
if (seriesValues.isEmpty()) {
seriesValues.addAll(shard.seriesValues());
} else {
for (int serieIndex = 0; serieIndex < seriesValues.size(); serieIndex++) {
DoubleArrayList mergedSerieValues = seriesValues.get(serieIndex);
for (int dataIndex = 0; dataIndex < mergedSerieValues.size(); dataIndex++) {
mergedSerieValues.set(dataIndex, mergedSerieValues.getDouble(dataIndex)
+ shard.seriesValues().get(serieIndex).getDouble(dataIndex)
);
}
for (Buckets shard : shards) {
if (seriesValues.isEmpty()) {
seriesValues.addAll(shard.seriesValues());
} else {
for (int serieIndex = 0; serieIndex < seriesValues.size(); serieIndex++) {
DoubleArrayList mergedSerieValues = seriesValues.get(serieIndex);
for (int dataIndex = 0; dataIndex < mergedSerieValues.size(); dataIndex++) {
mergedSerieValues.set(dataIndex, mergedSerieValues.getDouble(dataIndex)
+ shard.seriesValues().get(serieIndex).getDouble(dataIndex)
);
}
}
for (int i = 0; i < totals.size(); i++) {
totals.set(i, totals.getDouble(i) + shard.totals().getDouble(i));
}
}
return new Buckets(seriesValues, totals);
});
for (int i = 0; i < totals.size(); i++) {
totals.set(i, totals.getDouble(i) + shard.totals().getDouble(i));
}
}
return new Buckets(seriesValues, totals);
}
@Override
public Flux<LLSearchResultShard> search(@Nullable LLSnapshot snapshot,
public Stream<LLSearchResultShard> search(@Nullable LLSnapshot snapshot,
QueryParams queryParams,
@Nullable String keyFieldName) {
return luceneIndicesFlux.flatMap(luceneIndex -> luceneIndex.search(snapshot,
return luceneIndicesSet.parallelStream().flatMap(luceneIndex -> luceneIndex.search(snapshot,
queryParams,
keyFieldName
)).doOnDiscard(DiscardingCloseable.class, LLUtils::onDiscard);
));
}
@Override
public Mono<Buckets> computeBuckets(@Nullable LLSnapshot snapshot,
public Buckets computeBuckets(@Nullable LLSnapshot snapshot,
@NotNull List<Query> queries,
@Nullable Query normalizationQuery,
BucketParams bucketParams) {
return luceneIndicesFlux.flatMap(luceneIndex -> luceneIndex.computeBuckets(snapshot,
return mergeShards(luceneIndicesSet.parallelStream().map(luceneIndex -> luceneIndex.computeBuckets(snapshot,
queries,
normalizationQuery,
bucketParams
)).collectList().flatMap(this::mergeShards).doOnDiscard(DiscardingCloseable.class, LLUtils::onDiscard);
)).toList());
}
@Override
@ -190,78 +191,60 @@ public class LLMultiLuceneIndex implements LLLuceneIndex {
@Override
public void close() {
Iterable<Mono<Void>> it = () -> luceneIndicesSet.stream().map(e -> Mono.<Void>fromRunnable(e::close)).iterator();
Mono.whenDelayError(it).transform(LLUtils::handleDiscard).block();
luceneIndicesSet.parallelStream().forEach(SafeCloseable::close);
}
@Override
public Mono<Void> flush() {
Iterable<Mono<Void>> it = () -> luceneIndicesSet.stream().map(LLLuceneIndex::flush).iterator();
return Mono.whenDelayError(it);
public void flush() {
luceneIndicesSet.parallelStream().forEach(LLLuceneIndex::flush);
}
@Override
public Mono<Void> waitForMerges() {
Iterable<Mono<Void>> it = () -> luceneIndicesSet.stream().map(LLLuceneIndex::waitForMerges).iterator();
return Mono.whenDelayError(it);
public void waitForMerges() {
luceneIndicesSet.parallelStream().forEach(LLLuceneIndex::waitForMerges);
}
@Override
public Mono<Void> waitForLastMerges() {
Iterable<Mono<Void>> it = () -> luceneIndicesSet.stream().map(LLLuceneIndex::waitForLastMerges).iterator();
return Mono.whenDelayError(it);
public void waitForLastMerges() {
luceneIndicesSet.parallelStream().forEach(LLLuceneIndex::waitForLastMerges);
}
@Override
public Mono<Void> refresh(boolean force) {
Iterable<Mono<Void>> it = () -> luceneIndicesSet.stream().map(index -> index.refresh(force)).iterator();
return Mono.whenDelayError(it);
public void refresh(boolean force) {
luceneIndicesSet.parallelStream().forEach(index -> index.refresh(force));
}
@Override
public Mono<LLSnapshot> takeSnapshot() {
return Mono
// Generate next snapshot index
.fromCallable(nextSnapshotNumber::getAndIncrement)
.flatMap(snapshotIndex -> luceneIndicesFlux
.flatMapSequential(llLuceneIndex -> llLuceneIndex.takeSnapshot())
.collectList()
.doOnNext(instancesSnapshotsArray -> registeredSnapshots.put(snapshotIndex, instancesSnapshotsArray))
.thenReturn(new LLSnapshot(snapshotIndex))
);
public LLSnapshot takeSnapshot() {
// Generate next snapshot index
var snapshotIndex = nextSnapshotNumber.getAndIncrement();
var snapshot = luceneIndicesSet.parallelStream().map(LLSnapshottable::takeSnapshot).toList();
registeredSnapshots.put(snapshotIndex, snapshot);
return new LLSnapshot(snapshotIndex);
}
@Override
public Mono<Void> releaseSnapshot(LLSnapshot snapshot) {
return Mono
.fromCallable(() -> registeredSnapshots.remove(snapshot.getSequenceNumber()))
.flatMapIterable(list -> list)
.index()
.flatMap(tuple -> {
int index = (int) (long) tuple.getT1();
LLSnapshot instanceSnapshot = tuple.getT2();
return luceneIndicesSet.get(index).releaseSnapshot(instanceSnapshot);
})
.then();
public void releaseSnapshot(LLSnapshot snapshot) {
var list = registeredSnapshots.remove(snapshot.getSequenceNumber());
for (int shardIndex = 0; shardIndex < list.size(); shardIndex++) {
var luceneIndex = luceneIndicesSet.get(shardIndex);
LLSnapshot instanceSnapshot = list.get(shardIndex);
luceneIndex.releaseSnapshot(instanceSnapshot);
}
}
@Override
public Mono<Void> pauseForBackup() {
return Mono.whenDelayError(Iterables.transform(this.luceneIndicesSet, IBackuppable::pauseForBackup));
public void pauseForBackup() {
this.luceneIndicesSet.forEach(IBackuppable::pauseForBackup);
}
@Override
public Mono<Void> resumeAfterBackup() {
return Mono.whenDelayError(Iterables.transform(this.luceneIndicesSet, IBackuppable::resumeAfterBackup));
public void resumeAfterBackup() {
this.luceneIndicesSet.forEach(IBackuppable::resumeAfterBackup);
}
@Override
public boolean isPaused() {
for (LLLuceneIndex llLuceneIndex : this.luceneIndicesSet) {
if (llLuceneIndex.isPaused()) {
return true;
}
}
return false;
return this.luceneIndicesSet.stream().anyMatch(IBackuppable::isPaused);
}
}

166
src/main/java/it/cavallium/dbengine/database/LLRange.java
View File

@ -1,195 +1,92 @@
package it.cavallium.dbengine.database;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.Drop;
import io.netty5.buffer.Owned;
import io.netty5.util.Send;
import io.netty5.buffer.internal.ResourceSupport;
import it.cavallium.dbengine.utils.SimpleResource;
import it.cavallium.dbengine.buffers.Buf;
import java.util.StringJoiner;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
/**
* Range of data, from min (inclusive), to max (exclusive)
*/
public class LLRange extends SimpleResource {
public class LLRange {
private static final LLRange RANGE_ALL = new LLRange( null, null, (Buffer) null, false);
private static final LLRange RANGE_ALL = new LLRange( null, null, (Buf) null);
@Nullable
private final Buffer min;
private final Buf min;
@Nullable
private final Buffer max;
private final Buf max;
@Nullable
private final Buffer single;
private final Buf single;
private LLRange(Send<Buffer> min, Send<Buffer> max, Send<Buffer> single, boolean closeable) {
super(closeable);
private LLRange(@Nullable Buf min, @Nullable Buf max, @Nullable Buf single) {
assert single == null || (min == null && max == null);
this.min = min != null ? min.receive().makeReadOnly() : null;
this.max = max != null ? max.receive().makeReadOnly() : null;
this.single = single != null ? single.receive().makeReadOnly() : null;
}
private LLRange(Buffer min, Buffer max, Buffer single, boolean closeable) {
super(closeable);
assert single == null || (min == null && max == null);
this.min = min != null ? min.makeReadOnly() : null;
this.max = max != null ? max.makeReadOnly() : null;
this.single = single != null ? single.makeReadOnly() : null;
this.min = min;
this.max = max;
this.single = single;
}
public static LLRange all() {
return RANGE_ALL;
}
public static LLRange from(Send<Buffer> min) {
return new LLRange(min, null, null, true);
public static LLRange from(Buf min) {
return new LLRange(min, null, null);
}
public static LLRange to(Send<Buffer> max) {
return new LLRange(null, max, null, true);
public static LLRange to(Buf max) {
return new LLRange(null, max, null);
}
public static LLRange single(Send<Buffer> single) {
return new LLRange(null, null, single, true);
public static LLRange single(Buf single) {
return new LLRange(null, null, single);
}
public static LLRange singleUnsafe(Buffer single) {
return new LLRange(null, null, single, true);
}
public static LLRange of(Send<Buffer> min, Send<Buffer> max) {
return new LLRange(min, max, null, true);
}
public static LLRange ofUnsafe(Buffer min, Buffer max) {
return new LLRange(min, max, null, true);
public static LLRange of(Buf min, Buf max) {
return new LLRange(min, max, null);
}
public boolean isAll() {
ensureOpen();
return min == null && max == null && single == null;
}
public boolean isSingle() {
ensureOpen();
return single != null;
}
public boolean hasMin() {
ensureOpen();
return min != null || single != null;
}
public Send<Buffer> getMin() {
ensureOpen();
if (min != null) {
// todo: use a read-only copy
return min.copy().send();
} else if (single != null) {
// todo: use a read-only copy
return single.copy().send();
} else {
return null;
}
}
public Buffer getMinUnsafe() {
ensureOpen();
public Buf getMin() {
// todo: use a read-only copy
if (min != null) {
return min;
} else if (single != null) {
} else {
return single;
} else {
return null;
}
}
public Buffer getMinCopy() {
ensureOpen();
if (min != null) {
return min.copy();
} else if (single != null) {
return single.copy();
} else {
return null;
}
}
public boolean hasMax() {
ensureOpen();
return max != null || single != null;
}
public Send<Buffer> getMax() {
ensureOpen();
if (max != null) {
// todo: use a read-only copy
return max.copy().send();
} else if (single != null) {
// todo: use a read-only copy
return single.copy().send();
} else {
return null;
}
}
public Buffer getMaxUnsafe() {
ensureOpen();
public Buf getMax() {
// todo: use a read-only copy
if (max != null) {
return max;
} else if (single != null) {
} else {
return single;
} else {
return null;
}
}
public Buffer getMaxCopy() {
ensureOpen();
if (max != null) {
return max.copy();
} else if (single != null) {
return single.copy();
} else {
return null;
}
}
public Send<Buffer> getSingle() {
ensureOpen();
public Buf getSingle() {
assert isSingle();
// todo: use a read-only copy
return single != null ? single.copy().send() : null;
}
public Buffer getSingleUnsafe() {
ensureOpen();
assert isSingle();
return single;
}
@Override
protected void ensureOpen() {
super.ensureOpen();
assert min == null || min.isAccessible() : "Range min not owned";
assert max == null || max.isAccessible() : "Range max not owned";
assert single == null || single.isAccessible() : "Range single not owned";
}
@Override
protected void onClose() {
if (min != null && min.isAccessible()) {
min.close();
}
if (max != null && max.isAccessible()) {
max.close();
}
if (single != null && single.isAccessible()) {
single.close();
}
public Buf getSingleUnsafe() {
assert isSingle();
return single;
}
@Override
@ -220,12 +117,7 @@ public class LLRange extends SimpleResource {
}
public LLRange copy() {
ensureOpen();
// todo: use a read-only copy
return new LLRange(min != null ? min.copy().send() : null,
max != null ? max.copy().send() : null,
single != null ? single.copy().send() : null,
true
);
return new LLRange(min, max, single);
}
}

6
src/main/java/it/cavallium/dbengine/database/LLSearchResult.java
View File

@ -1,13 +1,13 @@
package it.cavallium.dbengine.database;
import java.util.function.BiFunction;
import java.util.stream.Stream;
import org.jetbrains.annotations.NotNull;
import reactor.core.publisher.Flux;
public record LLSearchResult(Flux<LLSearchResultShard> results) {
public record LLSearchResult(Stream<LLSearchResultShard> results) {
@NotNull
public static BiFunction<LLSearchResult, LLSearchResult, LLSearchResult> accumulator() {
return (a, b) -> new LLSearchResult(Flux.merge(a.results, b.results));
return (a, b) -> new LLSearchResult(Stream.concat(a.results, b.results));
}
}

19
src/main/java/it/cavallium/dbengine/database/LLSearchResultShard.java
View File

@ -1,32 +1,27 @@
package it.cavallium.dbengine.database;
import io.netty5.buffer.Drop;
import io.netty5.buffer.Owned;
import io.netty5.buffer.internal.ResourceSupport;
import it.cavallium.dbengine.client.LuceneIndexImpl;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.lucene.LuceneCloseable;
import it.cavallium.dbengine.utils.SimpleResource;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import reactor.core.publisher.Flux;
public class LLSearchResultShard extends SimpleResource implements DiscardingCloseable {
private static final Logger LOG = LogManager.getLogger(LLSearchResultShard.class);
private final Flux<LLKeyScore> results;
private final Stream<LLKeyScore> results;
private final TotalHitsCount totalHitsCount;
public LLSearchResultShard(Flux<LLKeyScore> results, TotalHitsCount totalHitsCount) {
public LLSearchResultShard(Stream<LLKeyScore> results, TotalHitsCount totalHitsCount) {
this.results = results;
this.totalHitsCount = totalHitsCount;
}
public static LLSearchResultShard withResource(Flux<LLKeyScore> results,
public static LLSearchResultShard withResource(Stream<LLKeyScore> results,
TotalHitsCount totalHitsCount,
SafeCloseable closeableResource) {
if (closeableResource instanceof LuceneCloseable luceneCloseable) {
@ -36,7 +31,7 @@ public class LLSearchResultShard extends SimpleResource implements DiscardingClo
}
}
public Flux<LLKeyScore> results() {
public Stream<LLKeyScore> results() {
ensureOpen();
return results;
}
@ -74,7 +69,7 @@ public class LLSearchResultShard extends SimpleResource implements DiscardingClo
private final List<SafeCloseable> resources;
public ResourcesLLSearchResultShard(Flux<LLKeyScore> resultsFlux,
public ResourcesLLSearchResultShard(Stream<LLKeyScore> resultsFlux,
TotalHitsCount count,
List<SafeCloseable> resources) {
super(resultsFlux, count);
@ -102,7 +97,7 @@ public class LLSearchResultShard extends SimpleResource implements DiscardingClo
private final List<LuceneCloseable> resources;
public LuceneLLSearchResultShard(Flux<LLKeyScore> resultsFlux,
public LuceneLLSearchResultShard(Stream<LLKeyScore> resultsFlux,
TotalHitsCount count,
List<LuceneCloseable> resources) {
super(resultsFlux, count);

26
src/main/java/it/cavallium/dbengine/database/LLSingleton.java
View File

@ -1,32 +1,22 @@
package it.cavallium.dbengine.database;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.BufferAllocator;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.disk.BinarySerializationFunction;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.unimi.dsi.fastutil.bytes.ByteList;
import java.util.function.Function;
import java.io.IOException;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
public interface LLSingleton extends LLKeyValueDatabaseStructure {
Buf get(@Nullable LLSnapshot snapshot);
BufferAllocator getAllocator();
void set(Buf value);
Mono<Buffer> get(@Nullable LLSnapshot snapshot);
Mono<Void> set(Mono<Buffer> value);
default Mono<Buffer> update(BinarySerializationFunction updater,
UpdateReturnMode updateReturnMode) {
return this
.updateAndGetDelta(updater)
.transform(prev -> LLUtils.resolveLLDelta(prev, updateReturnMode));
default Buf update(BinarySerializationFunction updater, UpdateReturnMode updateReturnMode) {
var prev = this.updateAndGetDelta(updater);
return LLUtils.resolveLLDelta(prev, updateReturnMode);
}
Mono<LLDelta> updateAndGetDelta(BinarySerializationFunction updater);
LLDelta updateAndGetDelta(BinarySerializationFunction updater);
String getColumnName();

6
src/main/java/it/cavallium/dbengine/database/LLSnapshottable.java
View File

@ -1,10 +1,10 @@
package it.cavallium.dbengine.database;
import reactor.core.publisher.Mono;
import java.io.IOException;
public interface LLSnapshottable {
Mono<LLSnapshot> takeSnapshot();
LLSnapshot takeSnapshot();
Mono<Void> releaseSnapshot(LLSnapshot snapshot);
void releaseSnapshot(LLSnapshot snapshot);
}

1
src/main/java/it/cavallium/dbengine/database/LLTerm.java
View File

@ -1,7 +1,6 @@
package it.cavallium.dbengine.database;
import java.util.Objects;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.BytesRef;
public class LLTerm {

726
src/main/java/it/cavallium/dbengine/database/LLUtils.java
View File

@ -1,22 +1,11 @@
package it.cavallium.dbengine.database;
import static io.netty5.buffer.StandardAllocationTypes.OFF_HEAP;
import static io.netty5.buffer.internal.InternalBufferUtils.NO_OP_DROP;
import static it.cavallium.dbengine.lucene.LuceneUtils.luceneScheduler;
import static org.apache.commons.lang3.ArrayUtils.EMPTY_BYTE_ARRAY;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import io.netty5.buffer.AllocatorControl;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.BufferAllocator;
import io.netty5.buffer.BufferComponent;
import io.netty5.buffer.CompositeBuffer;
import io.netty5.buffer.Drop;
import io.netty5.util.Resource;
import io.netty5.util.Send;
import io.netty5.util.IllegalReferenceCountException;
import it.cavallium.dbengine.database.serialization.SerializationException;
import io.netty.util.IllegalReferenceCountException;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.cavallium.dbengine.lucene.LuceneCloseable;
import it.cavallium.dbengine.lucene.LuceneUtils;
@ -26,8 +15,6 @@ import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodHandles.Lookup;
import java.lang.invoke.MethodType;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@ -36,11 +23,8 @@ import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
import java.util.function.ToIntFunction;
import java.util.function.Consumer;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.Marker;
@ -69,16 +53,7 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.rocksdb.AbstractImmutableNativeReference;
import org.rocksdb.AbstractNativeReference;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.ReadOptions;
import org.rocksdb.RocksDB;
import reactor.core.Disposable;
import reactor.core.Fuseable.QueueSubscription;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Hooks;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
@SuppressWarnings("unused")
public class LLUtils {
@ -89,13 +64,11 @@ public class LLUtils {
public static final int INITIAL_DIRECT_READ_BYTE_BUF_SIZE_BYTES = 4096;
public static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.allocateDirect(0).asReadOnlyBuffer();
private static final AllocatorControl NO_OP_ALLOCATION_CONTROL = (AllocatorControl) BufferAllocator.offHeapUnpooled();
private static final byte[] RESPONSE_TRUE = new byte[]{1};
private static final byte[] RESPONSE_FALSE = new byte[]{0};
private static final byte[] RESPONSE_TRUE_BUF = new byte[]{1};
private static final byte[] RESPONSE_FALSE_BUF = new byte[]{0};
public static final byte[][] LEXICONOGRAPHIC_ITERATION_SEEKS = new byte[256][1];
public static final AtomicBoolean hookRegistered = new AtomicBoolean();
public static final boolean MANUAL_READAHEAD = false;
public static final boolean ALLOW_STATIC_OPTIONS = false;
@ -111,29 +84,38 @@ public class LLUtils {
private static final MethodHandle IS_ACCESSIBLE_METHOD_HANDLE;
private static final MethodHandle IS_IN_NON_BLOCKING_THREAD_MH;
private static final Consumer<Object> NULL_CONSUMER = ignored -> {};
static {
for (int i1 = 0; i1 < 256; i1++) {
var b = LEXICONOGRAPHIC_ITERATION_SEEKS[i1];
b[0] = (byte) i1;
}
var methodType = MethodType.methodType(boolean.class);
MethodHandle isAccessibleMethodHandle = null;
try {
isAccessibleMethodHandle = PUBLIC_LOOKUP.findVirtual(AbstractNativeReference.class, "isAccessible", methodType);
} catch (NoSuchMethodException e) {
logger.debug("Failed to find isAccessible(): no such method");
} catch (IllegalAccessException e) {
logger.debug("Failed to find isAccessible()", e);
{
var methodType = MethodType.methodType(boolean.class);
MethodHandle isAccessibleMethodHandle = null;
try {
isAccessibleMethodHandle = PUBLIC_LOOKUP.findVirtual(AbstractNativeReference.class, "isAccessible", methodType);
} catch (NoSuchMethodException e) {
logger.debug("Failed to find isAccessible(): no such method");
} catch (IllegalAccessException e) {
logger.debug("Failed to find isAccessible()", e);
}
IS_ACCESSIBLE_METHOD_HANDLE = isAccessibleMethodHandle;
}
IS_ACCESSIBLE_METHOD_HANDLE = isAccessibleMethodHandle;
initHooks();
}
{
MethodHandle isInNonBlockingThreadMethodHandle = null;
try {
var clz = Objects.requireNonNull(PUBLIC_LOOKUP.findClass("reactor.core.scheduler.Schedulers"),
"reactor.core.scheduler.Schedulers not found");
public static void initHooks() {
if (hookRegistered.compareAndSet(false, true)) {
Hooks.onNextDropped(LLUtils::onNextDropped);
//todo: add Hooks.onDiscard when it will be implemented
// Hooks.onDiscard(LLUtils::onDiscard);
var methodType = MethodType.methodType(boolean.class);
isInNonBlockingThreadMethodHandle = PUBLIC_LOOKUP.findStatic(clz, "isInNonBlockingThread", methodType);
} catch (NoSuchMethodException | ClassNotFoundException | IllegalAccessException | NullPointerException e) {
logger.debug("Failed to obtain access to reactor core schedulers");
}
IS_IN_NON_BLOCKING_THREAD_MH = isInNonBlockingThreadMethodHandle;
}
}
@ -141,26 +123,17 @@ public class LLUtils {
return response[0] == 1;
}
public static boolean responseToBoolean(Send<Buffer> responseToReceive) {
try (var response = responseToReceive.receive()) {
assert response.readableBytes() == 1;
return response.getByte(response.readerOffset()) == 1;
}
}
public static boolean responseToBoolean(Buffer response) {
try (response) {
assert response.readableBytes() == 1;
return response.getByte(response.readerOffset()) == 1;
}
public static boolean responseToBoolean(Buf response) {
assert response.size() == 1;
return response.getBoolean(0);
}
public static byte[] booleanToResponse(boolean bool) {
return bool ? RESPONSE_TRUE : RESPONSE_FALSE;
}
public static Buffer booleanToResponseByteBuffer(BufferAllocator alloc, boolean bool) {
return alloc.allocate(1).writeByte(bool ? (byte) 1 : 0);
public static Buf booleanToResponseByteBuffer(boolean bool) {
return Buf.wrap(new byte[] {bool ? (byte) 1 : 0});
}
@Nullable
@ -307,9 +280,9 @@ public class LLUtils {
return new it.cavallium.dbengine.database.LLKeyScore(hit.docId(), hit.shardId(), hit.score(), hit.key());
}
public static String toStringSafe(@Nullable Buffer key) {
public static String toStringSafe(byte @Nullable[] key) {
try {
if (key == null || key.isAccessible()) {
if (key == null) {
return toString(key);
} else {
return "(released)";
@ -319,7 +292,7 @@ public class LLUtils {
}
}
public static String toStringSafe(byte @Nullable[] key) {
public static String toStringSafe(@Nullable Buf key) {
try {
if (key == null) {
return toString(key);
@ -333,7 +306,7 @@ public class LLUtils {
public static String toStringSafe(@Nullable LLRange range) {
try {
if (range == null || !range.isClosed()) {
if (range == null) {
return toString(range);
} else {
return "(released)";
@ -349,60 +322,21 @@ public class LLUtils {
} else if (range.isAll()) {
return "ξ";
} else if (range.hasMin() && range.hasMax()) {
return "[" + toStringSafe(range.getMinUnsafe()) + "," + toStringSafe(range.getMaxUnsafe()) + ")";
return "[" + toStringSafe(range.getMin()) + "," + toStringSafe(range.getMax()) + ")";
} else if (range.hasMin()) {
return "[" + toStringSafe(range.getMinUnsafe()) + ",*)";
return "[" + toStringSafe(range.getMin()) + ",*)";
} else if (range.hasMax()) {
return "[*," + toStringSafe(range.getMaxUnsafe()) + ")";
return "[*," + toStringSafe(range.getMax()) + ")";
} else {
return "∅";
}
}
public static String toString(@Nullable Buffer key) {
public static String toString(@Nullable Buf key) {
if (key == null) {
return "null";
} else {
int startIndex = key.readerOffset();
int iMax = key.readableBytes() - 1;
int iLimit = 128;
if (iMax <= -1) {
return "[]";
} else {
StringBuilder arraySB = new StringBuilder();
StringBuilder asciiSB = new StringBuilder();
boolean isAscii = true;
arraySB.append('[');
int i = 0;
while (true) {
var byteVal = key.getUnsignedByte(startIndex + i);
arraySB.append(byteVal);
if (isAscii) {
if (byteVal >= 32 && byteVal < 127) {
asciiSB.append((char) byteVal);
} else if (byteVal == 0) {
asciiSB.append('␀');
} else {
isAscii = false;
asciiSB = null;
}
}
if (i == iLimit) {
arraySB.append("…");
}
if (i == iMax || i == iLimit) {
if (isAscii) {
return asciiSB.insert(0, "\"").append("\"").toString();
} else {
return arraySB.append(']').toString();
}
}
arraySB.append(", ");
++i;
}
}
return toString(key.asArray());
}
}
@ -453,21 +387,11 @@ public class LLUtils {
}
}
public static boolean equals(Buffer a, Buffer b) {
public static boolean equals(Buf a, Buf b) {
if (a == null && b == null) {
return true;
} else if (a != null && b != null) {
var aCur = a.openCursor();
var bCur = b.openCursor();
if (aCur.bytesLeft() != bCur.bytesLeft()) {
return false;
}
while (aCur.readByte() && bCur.readByte()) {
if (aCur.getByte() != bCur.getByte()) {
return false;
}
}
return true;
return a.equals(b);
} else {
return false;
}
@ -481,123 +405,27 @@ public class LLUtils {
* <p>
* {@code a[aStartIndex : aStartIndex + length] == b[bStartIndex : bStartIndex + length]}
*/
public static boolean equals(Buffer a, int aStartIndex, Buffer b, int bStartIndex, int length) {
var aCur = a.openCursor(aStartIndex, length);
var bCur = b.openCursor(bStartIndex, length);
if (aCur.bytesLeft() != bCur.bytesLeft()) {
return false;
}
while (aCur.readByte() && bCur.readByte()) {
if (aCur.getByte() != bCur.getByte()) {
return false;
}
}
return true;
public static boolean equals(Buf a, int aStartIndex, Buf b, int bStartIndex, int length) {
return a.equals(aStartIndex, b, bStartIndex, length);
}
public static byte[] toArray(@Nullable Buffer key) {
/**
*
* @return the inner array, DO NOT MODIFY IT
*/
public static byte[] asArray(@Nullable Buf key) {
if (key == null) {
return EMPTY_BYTE_ARRAY;
}
byte[] array = new byte[key.readableBytes()];
key.copyInto(key.readerOffset(), array, 0, key.readableBytes());
return array;
return key.asArray();
}
public static List<byte[]> toArray(List<Buffer> input) {
List<byte[]> result = new ArrayList<>(input.size());
for (Buffer byteBuf : input) {
result.add(toArray(byteBuf));
}
return result;
}
public static int hashCode(Buffer buf) {
public static int hashCode(Buf buf) {
if (buf == null) {
return 0;
}
int result = 1;
var cur = buf.openCursor();
while (cur.readByte()) {
var element = cur.getByte();
result = 31 * result + element;
}
return result;
}
/**
* @return null if size is equal to RocksDB.NOT_FOUND
*/
@Nullable
public static Buffer readNullableDirectNioBuffer(BufferAllocator alloc, ToIntFunction<ByteBuffer> reader) {
if (alloc.getAllocationType() != OFF_HEAP) {
throw new UnsupportedOperationException("Allocator type is not direct: " + alloc);
}
var directBuffer = alloc.allocate(INITIAL_DIRECT_READ_BYTE_BUF_SIZE_BYTES);
try {
assert directBuffer.readerOffset() == 0;
assert directBuffer.writerOffset() == 0;
var directBufferWriter = ((BufferComponent) directBuffer).writableBuffer();
assert directBufferWriter.position() == 0;
assert directBufferWriter.capacity() >= directBuffer.capacity();
assert directBufferWriter.isDirect();
int trueSize = reader.applyAsInt(directBufferWriter);
if (trueSize == RocksDB.NOT_FOUND) {
directBuffer.close();
return null;
}
int readSize = directBufferWriter.limit();
if (trueSize < readSize) {
throw new IllegalStateException();
} else if (trueSize == readSize) {
return directBuffer.writerOffset(directBufferWriter.limit());
} else {
assert directBuffer.readerOffset() == 0;
directBuffer.ensureWritable(trueSize);
assert directBuffer.writerOffset() == 0;
directBufferWriter = ((BufferComponent) directBuffer).writableBuffer();
assert directBufferWriter.position() == 0;
assert directBufferWriter.isDirect();
reader.applyAsInt(directBufferWriter.position(0));
return directBuffer.writerOffset(trueSize);
}
} catch (Throwable t) {
directBuffer.close();
throw t;
}
}
public static void ensureBlocking() {
if (Schedulers.isInNonBlockingThread()) {
throw new UnsupportedOperationException("Called collect in a nonblocking thread");
}
}
// todo: remove this ugly method
/**
* cleanup resource
* @param cleanupOnSuccess if true the resource will be cleaned up if the function is successful
*/
public static <U, T extends Resource<T>> Mono<U> usingSendResource(Mono<Send<T>> resourceSupplier,
Function<T, Mono<U>> resourceClosure,
boolean cleanupOnSuccess) {
return Mono.usingWhen(resourceSupplier.map(Send::receive), resourceClosure, r -> {
if (cleanupOnSuccess) {
return Mono.fromRunnable(() -> r.close());
} else {
return Mono.empty();
}
}, (r, ex) -> Mono.fromRunnable(() -> {
if (r.isAccessible()) {
r.close();
}
}), r -> Mono.fromRunnable(() -> {
if (r.isAccessible()) {
r.close();
}
}));
return buf.hashCode();
}
public static boolean isSet(ScoreDoc[] scoreDocs) {
@ -609,26 +437,6 @@ public class LLUtils {
return true;
}
public static Send<Buffer> empty(BufferAllocator allocator) {
try {
return allocator.allocate(0).send();
} catch (Exception ex) {
try (var empty = CompositeBuffer.compose(allocator)) {
assert empty.readableBytes() == 0;
assert empty.capacity() == 0;
return empty.send();
}
}
}
public static Send<Buffer> copy(BufferAllocator allocator, Buffer buf) {
if (CompositeBuffer.isComposite(buf) && buf.capacity() == 0) {
return empty(allocator);
} else {
return buf.copy().send();
}
}
public static boolean isBoundedRange(LLRange rangeShared) {
return rangeShared.hasMin() && rangeShared.hasMax();
}
@ -649,124 +457,26 @@ public class LLUtils {
//noinspection resource
readOptions = new ReadOptions();
}
if (boundedRange || smallRange) {
readOptions.setFillCache(canFillCache);
} else {
var hugeRange = !boundedRange && !smallRange;
if (hugeRange) {
if (readOptions.readaheadSize() <= 0) {
readOptions.setReadaheadSize(4 * 1024 * 1024); // 4MiB
}
readOptions.setFillCache(false);
readOptions.setVerifyChecksums(false);
}
if (FORCE_DISABLE_CHECKSUM_VERIFICATION) {
readOptions.setVerifyChecksums(false);
}
readOptions.setFillCache(canFillCache && !hugeRange);
readOptions.setVerifyChecksums(!FORCE_DISABLE_CHECKSUM_VERIFICATION && !hugeRange);
return readOptions;
}
public static Mono<Void> finalizeResource(Resource<?> resource) {
Mono<Void> runnable = Mono.fromRunnable(() -> LLUtils.finalizeResourceNow(resource));
if (resource instanceof LuceneCloseable) {
return runnable.transform(LuceneUtils::scheduleLucene);
} else {
return runnable;
}
}
public static Mono<Void> finalizeResource(SafeCloseable resource) {
Mono<Void> runnable = Mono.fromRunnable(resource::close);
if (resource instanceof LuceneCloseable) {
return runnable.transform(LuceneUtils::scheduleLucene);
} else {
return runnable;
}
}
public static void finalizeResourceNow(Resource<?> resource) {
if (resource.isAccessible()) {
resource.close();
}
public static void finalizeResource(SafeCloseable resource) {
resource.close();
}
public static void finalizeResourceNow(SafeCloseable resource) {
resource.close();
}
public static <V> Flux<V> handleDiscard(Flux<V> flux) {
return flux.doOnDiscard(Object.class, LLUtils::onDiscard);
}
public static <V> Mono<V> handleDiscard(Mono<V> flux) {
return flux.doOnDiscard(Object.class, LLUtils::onDiscard);
}
/**
* Obtain the resource, then run the closure.
* If the closure publisher returns a single element, then the resource is kept open,
* otherwise it is closed.
*/
public static <T extends AutoCloseable, U> Mono<U> singleOrClose(Mono<T> resourceMono,
Function<T, Mono<U>> closure) {
return Mono.usingWhen(resourceMono, resource -> {
if (resource instanceof LuceneCloseable) {
return closure.apply(resource).publishOn(luceneScheduler()).doOnSuccess(s -> {
if (s == null) {
try {
resource.close();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}).publishOn(Schedulers.parallel());
} else {
return closure.apply(resource).doOnSuccess(s -> {
if (s == null) {
try {
resource.close();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
});
}
}, resource -> Mono.empty(), (resource, ex) -> Mono.fromCallable(() -> {
resource.close();
return null;
}), r -> (r instanceof SafeCloseable s) ? LLUtils.finalizeResource(s) : Mono.fromCallable(() -> {
r.close();
return null;
}));
}
public static Disposable scheduleRepeated(Scheduler scheduler, Runnable action, Duration delay) {
var currentDisposable = new AtomicReference<Disposable>();
var disposed = new AtomicBoolean(false);
scheduleRepeatedInternal(scheduler, action, delay, currentDisposable, disposed);
return () -> {
disposed.set(true);
currentDisposable.get().dispose();
};
}
private static void scheduleRepeatedInternal(Scheduler scheduler,
Runnable action,
Duration delay,
AtomicReference<Disposable> currentDisposable,
AtomicBoolean disposed) {
if (disposed.get()) return;
currentDisposable.set(scheduler.schedule(() -> {
if (disposed.get()) return;
try {
action.run();
} catch (Throwable ex) {
logger.error(ex);
}
scheduleRepeatedInternal(scheduler, action, delay, currentDisposable, disposed);
}, delay.toMillis(), TimeUnit.MILLISECONDS));
}
public static boolean isAccessible(AbstractNativeReference abstractNativeReference) {
if (IS_ACCESSIBLE_METHOD_HANDLE != null) {
try {
@ -778,220 +488,116 @@ public class LLUtils {
return true;
}
@Deprecated
public record DirectBuffer(@NotNull Buffer buffer, @NotNull ByteBuffer byteBuffer) {}
@NotNull
public static ByteBuffer newDirect(int size) {
return ByteBuffer.allocateDirect(size);
public static Buf unmodifiableBytes(Buf previous) {
previous.freeze();
return previous;
}
private static Drop<Buffer> drop() {
// We cannot reliably drop unsafe memory. We have to rely on the cleaner to do that.
return NO_OP_DROP;
}
public static boolean isReadOnlyDirect(Buffer inputBuffer) {
return inputBuffer instanceof BufferComponent component && component.readableNativeAddress() != 0;
}
public static ByteBuffer getReadOnlyDirect(Buffer inputBuffer) {
assert isReadOnlyDirect(inputBuffer);
return ((BufferComponent) inputBuffer).readableBuffer();
}
public static Buffer fromByteArray(BufferAllocator alloc, byte[] array) {
Buffer result = alloc.allocate(array.length);
result.writeBytes(array);
return result;
}
@NotNull
public static Buffer readDirectNioBuffer(BufferAllocator alloc, ToIntFunction<ByteBuffer> reader) {
var nullable = readNullableDirectNioBuffer(alloc, reader);
if (nullable == null) {
throw new IllegalStateException("A non-nullable buffer read operation tried to return a \"not found\" element");
}
return nullable;
}
public static Buffer compositeBuffer(BufferAllocator alloc, Send<Buffer> buffer) {
return buffer.receive();
}
@NotNull
public static Buffer compositeBuffer(BufferAllocator alloc,
@NotNull Send<Buffer> buffer1,
@NotNull Send<Buffer> buffer2) {
var b1 = buffer1.receive();
try (var b2 = buffer2.receive()) {
if (b1.writerOffset() < b1.capacity() || b2.writerOffset() < b2.capacity()) {
b1.ensureWritable(b2.readableBytes(), b2.readableBytes(), true);
b2.copyInto(b2.readerOffset(), b1, b1.writerOffset(), b2.readableBytes());
b1.writerOffset(b1.writerOffset() + b2.readableBytes());
return b1;
} else {
return alloc.compose(List.of(b1.send(), b2.send()));
}
}
}
@NotNull
public static Buffer compositeBuffer(BufferAllocator alloc,
@NotNull Send<Buffer> buffer1,
@NotNull Send<Buffer> buffer2,
@NotNull Send<Buffer> buffer3) {
var b1 = buffer1.receive();
try (var b2 = buffer2.receive()) {
try (var b3 = buffer3.receive()) {
if (b1.writerOffset() < b1.capacity()
|| b2.writerOffset() < b2.capacity()
|| b3.writerOffset() < b3.capacity()) {
b1.ensureWritable(b2.readableBytes(), b2.readableBytes(), true);
b2.copyInto(b2.readerOffset(), b1, b1.writerOffset(), b2.readableBytes());
b1.writerOffset(b1.writerOffset() + b2.readableBytes());
b1.ensureWritable(b3.readableBytes(), b3.readableBytes(), true);
b3.copyInto(b3.readerOffset(), b1, b1.writerOffset(), b3.readableBytes());
b1.writerOffset(b1.writerOffset() + b3.readableBytes());
return b1;
} else {
return alloc.compose(List.of(b1.send(), b2.send(), b3.send()));
}
}
}
}
public static <T> Mono<T> resolveDelta(Mono<Delta<T>> prev, UpdateReturnMode updateReturnMode) {
return prev.handle((delta, sink) -> {
switch (updateReturnMode) {
case GET_NEW_VALUE -> {
var current = delta.current();
if (current != null) {
sink.next(current);
} else {
sink.complete();
}
}
case GET_OLD_VALUE -> {
var previous = delta.previous();
if (previous != null) {
sink.next(previous);
} else {
sink.complete();
}
}
case NOTHING -> sink.complete();
default -> sink.error(new IllegalStateException());
}
});
}
public static Mono<Buffer> resolveLLDelta(Mono<LLDelta> prev, UpdateReturnMode updateReturnMode) {
return prev.mapNotNull(delta -> {
final Buffer previous = delta.previousUnsafe();
final Buffer current = delta.currentUnsafe();
return switch (updateReturnMode) {
case GET_NEW_VALUE -> {
if (previous != null && previous.isAccessible()) {
previous.close();
}
yield current;
}
case GET_OLD_VALUE -> {
if (current != null && current.isAccessible()) {
current.close();
}
yield previous;
}
case NOTHING -> {
if (previous != null && previous.isAccessible()) {
previous.close();
}
if (current != null && current.isAccessible()) {
current.close();
}
yield null;
}
};
});
}
public static <T, U> Mono<Delta<U>> mapDelta(Mono<Delta<T>> mono,
SerializationFunction<@NotNull T, @Nullable U> mapper) {
return mono.handle((delta, sink) -> {
public static boolean isInNonBlockingThread() {
if (IS_IN_NON_BLOCKING_THREAD_MH != null) {
try {
T prev = delta.previous();
T curr = delta.current();
U newPrev;
U newCurr;
if (prev != null) {
newPrev = mapper.apply(prev);
} else {
newPrev = null;
}
if (curr != null) {
newCurr = mapper.apply(curr);
} else {
newCurr = null;
}
sink.next(new Delta<>(newPrev, newCurr));
} catch (SerializationException ex) {
sink.error(ex);
return (boolean) IS_IN_NON_BLOCKING_THREAD_MH.invokeExact();
} catch (Throwable e) {
throw new RuntimeException(e);
}
});
}
return false;
}
public static <U> Mono<Delta<U>> mapLLDelta(Mono<LLDelta> mono,
SerializationFunction<@NotNull Buffer, @Nullable U> mapper) {
return Mono.usingWhen(mono, delta -> Mono.fromCallable(() -> {
Buffer prev = delta.previousUnsafe();
Buffer curr = delta.currentUnsafe();
U newPrev;
U newCurr;
if (prev != null) {
newPrev = mapper.apply(prev);
} else {
newPrev = null;
}
if (curr != null) {
newCurr = mapper.apply(curr);
} else {
newCurr = null;
}
return new Delta<>(newPrev, newCurr);
}), LLUtils::finalizeResource);
public static Buf copy(Buf buf) {
return buf.copy();
}
public static Buf asByteList(byte[] array) {
return Buf.wrap(array);
}
public static Buf toByteList(byte[] array) {
return Buf.copyOf(array);
}
public static Buf compositeBuffer(Buf buffer) {
return buffer;
}
@NotNull
public static Buf compositeBuffer(Buf buffer1, Buf buffer2) {
// todo: create a composite buffer without allocating a new array
var out = Buf.create(buffer1.size() + buffer2.size());
out.addAll(buffer1);
out.addAll(buffer2);
return out;
}
@NotNull
public static Buf compositeBuffer(Buf buffer1, Buf buffer2, Buf buffer3) {
// todo: create a composite buffer without allocating a new array
var out = Buf.create(buffer1.size() + buffer2.size());
out.addAll(buffer1);
out.addAll(buffer2);
out.addAll(buffer3);
return out;
}
public static <T> T resolveDelta(Delta<T> delta, UpdateReturnMode updateReturnMode) {
return switch (updateReturnMode) {
case GET_NEW_VALUE -> delta.current();
case GET_OLD_VALUE -> delta.previous();
case NOTHING -> null;
};
}
public static Buf resolveLLDelta(LLDelta delta, UpdateReturnMode updateReturnMode) {
final Buf previous = delta.previous();
final Buf current = delta.current();
return switch (updateReturnMode) {
case GET_NEW_VALUE -> current;
case GET_OLD_VALUE -> previous;
case NOTHING -> null;
};
}
public static <T, U> Delta<U> mapDelta(Delta<T> delta, SerializationFunction<@NotNull T, @Nullable U> mapper) {
T prev = delta.previous();
T curr = delta.current();
U newPrev;
U newCurr;
if (prev != null) {
newPrev = mapper.apply(prev);
} else {
newPrev = null;
}
if (curr != null) {
newCurr = mapper.apply(curr);
} else {
newCurr = null;
}
return new Delta<>(newPrev, newCurr);
}
public static <U> Delta<U> mapLLDelta(LLDelta delta, SerializationFunction<@NotNull Buf, @Nullable U> mapper) {
var prev = delta.previous();
var curr = delta.current();
U newPrev;
U newCurr;
if (prev != null) {
newPrev = mapper.apply(prev);
} else {
newPrev = null;
}
if (curr != null) {
newCurr = mapper.apply(curr);
} else {
newCurr = null;
}
return new Delta<>(newPrev, newCurr);
}
public static <R, V> boolean isDeltaChanged(Delta<V> delta) {
return !Objects.equals(delta.previous(), delta.current());
}
public static boolean isDirect(Buffer key) {
var readableComponents = key.countReadableComponents();
if (readableComponents == 0) {
return true;
} else if (readableComponents == 1) {
return key.isDirect();
} else {
return false;
}
}
public static String deserializeString(Send<Buffer> bufferSend, int readerOffset, int length, Charset charset) {
try (var buffer = bufferSend.receive()) {
byte[] bytes = new byte[Math.min(length, buffer.readableBytes())];
buffer.copyInto(readerOffset, bytes, 0, length);
return new String(bytes, charset);
}
}
public static String deserializeString(@NotNull Buffer buffer, int readerOffset, int length, Charset charset) {
byte[] bytes = new byte[Math.min(length, buffer.readableBytes())];
buffer.copyInto(readerOffset, bytes, 0, length);
return new String(bytes, charset);
}
public static int utf8MaxBytes(String deserialized) {
return deserialized.length() * 3;
}
@ -1015,18 +621,14 @@ public class LLUtils {
}
private static void closeResource(Object next, boolean manual) {
if (next instanceof Send<?> send) {
send.close();
} if (next instanceof SafeCloseable closeable) {
if (next instanceof SafeCloseable closeable) {
if (manual || closeable instanceof DiscardingCloseable) {
if (!manual && !LuceneUtils.isLuceneThread() && closeable instanceof LuceneCloseable luceneCloseable) {
luceneScheduler().schedule(() -> luceneCloseable.close());
luceneCloseable.close();
} else {
closeable.close();
}
}
} else if (next instanceof Resource<?> resource && resource.isAccessible()) {
resource.close();
} else if (next instanceof List<?> iterable) {
iterable.forEach(obj -> closeResource(obj, manual));
} else if (next instanceof Set<?> iterable) {
@ -1080,4 +682,10 @@ public class LLUtils {
return term.getValueBytesRef();
}
}
public static <X> void consume(Stream<X> stream) {
try (stream) {
stream.forEach(NULL_CONSUMER);
}
}
}

27
src/main/java/it/cavallium/dbengine/database/OptionalBuf.java
View File

@ -1,6 +1,6 @@
package it.cavallium.dbengine.database;
import io.netty5.buffer.Buffer;
import it.cavallium.dbengine.buffers.Buf;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Optional;
@ -9,20 +9,20 @@ import java.util.function.Function;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public final class OptionalBuf implements DiscardingCloseable {
public final class OptionalBuf {
private static final OptionalBuf EMPTY = new OptionalBuf(null);
private final Buffer buffer;
private final Buf buffer;
private OptionalBuf(@Nullable Buffer buffer) {
private OptionalBuf(@Nullable Buf buffer) {
this.buffer = buffer;
}
public static OptionalBuf ofNullable(@Nullable Buffer buffer) {
public static OptionalBuf ofNullable(@Nullable Buf buffer) {
return new OptionalBuf(buffer);
}
public static OptionalBuf of(@NotNull Buffer buffer) {
public static OptionalBuf of(@NotNull Buf buffer) {
Objects.requireNonNull(buffer);
return new OptionalBuf(buffer);
}
@ -31,13 +31,6 @@ public final class OptionalBuf implements DiscardingCloseable {
return EMPTY;
}
@Override
public void close() {
if (buffer != null && buffer.isAccessible()) {
buffer.close();
}
}
@Override
public String toString() {
if (buffer != null) {
@ -66,21 +59,21 @@ public final class OptionalBuf implements DiscardingCloseable {
return buffer != null ? buffer.hashCode() : 0;
}
public Buffer get() {
public Buf get() {
if (buffer == null) {
throw new NoSuchElementException();
}
return buffer;
}
public Buffer orElse(Buffer alternative) {
public Buf orElse(Buf alternative) {
if (buffer == null) {
return alternative;
}
return buffer;
}
public void ifPresent(Consumer<Buffer> consumer) {
public void ifPresent(Consumer<Buf> consumer) {
if (buffer != null) {
consumer.accept(buffer);
}
@ -94,7 +87,7 @@ public final class OptionalBuf implements DiscardingCloseable {
return buffer == null;
}
public <U> Optional<U> map(Function<Buffer, U> mapper) {
public <U> Optional<U> map(Function<Buf, U> mapper) {
if (buffer != null) {
return Optional.of(mapper.apply(buffer));
} else {

57
src/main/java/it/cavallium/dbengine/database/RangeSupplier.java
View File

@ -1,57 +0,0 @@
package it.cavallium.dbengine.database;
import io.netty5.util.Send;
import java.util.function.Supplier;
public abstract class RangeSupplier implements DiscardingCloseable, Supplier<LLRange> {
public static RangeSupplier of(Supplier<LLRange> supplier) {
return new SimpleSupplier(supplier);
}
public static RangeSupplier ofOwned(LLRange supplier) {
return new CopySupplier(supplier);
}
public static RangeSupplier ofShared(LLRange supplier) {
return new SimpleSupplier(supplier::copy);
}
private static final class SimpleSupplier extends RangeSupplier {
private final Supplier<LLRange> supplier;
public SimpleSupplier(Supplier<LLRange> supplier) {
this.supplier = supplier;
}
@Override
public LLRange get() {
return supplier.get();
}
@Override
public void close() {
}
}
private static final class CopySupplier extends RangeSupplier {
private final LLRange supplier;
public CopySupplier(LLRange supplier) {
this.supplier = supplier;
}
@Override
public LLRange get() {
return supplier.copy();
}
@Override
public void close() {
supplier.close();
}
}
}

3
src/main/java/it/cavallium/dbengine/database/SafeCloseable.java
View File

@ -1,7 +1,6 @@
package it.cavallium.dbengine.database;
public interface SafeCloseable extends io.netty5.util.SafeCloseable {
public interface SafeCloseable extends AutoCloseable {
@Override
void close();
}

9
src/main/java/it/cavallium/dbengine/database/SubStageEntry.java
View File

@ -4,7 +4,7 @@ import it.cavallium.dbengine.database.collections.DatabaseStage;
import java.util.Map.Entry;
import java.util.Objects;
public final class SubStageEntry<T, U extends DatabaseStage<?>> implements DiscardingCloseable, Entry<T, U> {
public final class SubStageEntry<T, U extends DatabaseStage<?>> implements Entry<T, U> {
private final T key;
private final U value;
@ -14,13 +14,6 @@ public final class SubStageEntry<T, U extends DatabaseStage<?>> implements Disca
this.value = value;
}
@Override
public void close() {
if (value != null) {
value.close();
}
}
@Override
public T getKey() {
return key;

18
src/main/java/it/cavallium/dbengine/database/collections/DatabaseEmpty.java
View File

@ -1,11 +1,11 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.BufferAllocator;
import it.cavallium.dbengine.database.BufSupplier;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.buffers.BufDataInput;
import it.cavallium.dbengine.buffers.BufDataOutput;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.Serializer;
import java.util.function.Supplier;
import org.jetbrains.annotations.NotNull;
public class DatabaseEmpty {
@ -13,16 +13,16 @@ public class DatabaseEmpty {
@SuppressWarnings({"unused", "InstantiationOfUtilityClass"})
public static final Nothing NOTHING = new Nothing();
public static Serializer<Nothing> nothingSerializer(BufferAllocator bufferAllocator) {
public static Serializer<Nothing> nothingSerializer() {
return new Serializer<>() {
@Override
public @NotNull Nothing deserialize(@NotNull Buffer serialized) {
public @NotNull Nothing deserialize(@NotNull BufDataInput in) throws SerializationException {
return NOTHING;
}
@Override
public void serialize(@NotNull Nothing deserialized, Buffer output) {
public void serialize(@NotNull Nothing deserialized, BufDataOutput out) throws SerializationException {
}
@ -36,8 +36,8 @@ public class DatabaseEmpty {
private DatabaseEmpty() {
}
public static DatabaseStageEntry<Nothing> create(LLDictionary dictionary, BufSupplier key) {
return new DatabaseMapSingle<>(dictionary, key, nothingSerializer(dictionary.getAllocator()));
public static DatabaseStageEntry<Nothing> create(LLDictionary dictionary, Buf key) {
return new DatabaseMapSingle<>(dictionary, key, nothingSerializer());
}
public static final class Nothing {

32
src/main/java/it/cavallium/dbengine/database/collections/DatabaseInt.java
View File

@ -1,14 +1,12 @@
package it.cavallium.dbengine.database.collections;
import com.google.common.primitives.Ints;
import it.cavallium.dbengine.buffers.BufDataInput;
import it.cavallium.dbengine.buffers.BufDataOutput;
import it.cavallium.dbengine.database.LLKeyValueDatabaseStructure;
import it.cavallium.dbengine.database.LLSingleton;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
public class DatabaseInt implements LLKeyValueDatabaseStructure {
@ -17,28 +15,18 @@ public class DatabaseInt implements LLKeyValueDatabaseStructure {
public DatabaseInt(LLSingleton singleton) {
this.singleton = singleton;
this.serializer = SerializerFixedBinaryLength.intSerializer(singleton.getAllocator());
this.serializer = SerializerFixedBinaryLength.intSerializer();
}
public Mono<Integer> get(@Nullable LLSnapshot snapshot) {
var resultMono = singleton.get(snapshot);
return Mono.usingWhen(resultMono,
result -> Mono.fromSupplier(() -> serializer.deserialize(result)),
LLUtils::finalizeResource
);
public Integer get(@Nullable LLSnapshot snapshot) {
var result = singleton.get(snapshot);
return serializer.deserialize(BufDataInput.create(result));
}
public Mono<Void> set(int value) {
return singleton.set(Mono.fromCallable(() -> {
var buf = singleton.getAllocator().allocate(Integer.BYTES);
try {
serializer.serialize(value, buf);
return buf;
} catch (Throwable ex) {
buf.close();
throw ex;
}
}));
public void set(int value) {
var buf = BufDataOutput.createLimited(Integer.BYTES);
serializer.serialize(value, buf);
singleton.set(buf.asList());
}
@Override

89
src/main/java/it/cavallium/dbengine/database/collections/DatabaseLong.java
View File

@ -1,16 +1,14 @@
package it.cavallium.dbengine.database.collections;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.buffers.BufDataInput;
import it.cavallium.dbengine.buffers.BufDataOutput;
import it.cavallium.dbengine.database.LLKeyValueDatabaseStructure;
import it.cavallium.dbengine.database.LLSingleton;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
public class DatabaseLong implements LLKeyValueDatabaseStructure {
@ -20,82 +18,63 @@ public class DatabaseLong implements LLKeyValueDatabaseStructure {
public DatabaseLong(LLSingleton singleton) {
this.singleton = singleton;
this.serializer = SerializerFixedBinaryLength.longSerializer(singleton.getAllocator());
this.bugSerializer = SerializerFixedBinaryLength.intSerializer(singleton.getAllocator());
this.serializer = SerializerFixedBinaryLength.longSerializer();
this.bugSerializer = SerializerFixedBinaryLength.intSerializer();
}
public Mono<Long> get(@Nullable LLSnapshot snapshot) {
var resultMono = singleton.get(snapshot);
return Mono.usingWhen(resultMono,
result -> Mono.fromSupplier(() -> {
if (result.readableBytes() == 4) {
return (long) (int) bugSerializer.deserialize(result);
} else {
return serializer.deserialize(result);
}
}),
LLUtils::finalizeResource
);
public Long get(@Nullable LLSnapshot snapshot) {
var result = BufDataInput.create(singleton.get(snapshot));
if (result.available() == 4) {
return (long) (int) bugSerializer.deserialize(result);
} else {
return serializer.deserialize(result);
}
}
public Mono<Long> incrementAndGet() {
public Long incrementAndGet() {
return addAnd(1, UpdateReturnMode.GET_NEW_VALUE);
}
public Mono<Long> getAndIncrement() {
public Long getAndIncrement() {
return addAnd(1, UpdateReturnMode.GET_OLD_VALUE);
}
public Mono<Long> decrementAndGet() {
public Long decrementAndGet() {
return addAnd(-1, UpdateReturnMode.GET_NEW_VALUE);
}
public Mono<Long> getAndDecrement() {
public Long getAndDecrement() {
return addAnd(-1, UpdateReturnMode.GET_OLD_VALUE);
}
public Mono<Long> addAndGet(long count) {
public Long addAndGet(long count) {
return addAnd(count, UpdateReturnMode.GET_NEW_VALUE);
}
public Mono<Long> getAndAdd(long count) {
public Long getAndAdd(long count) {
return addAnd(count, UpdateReturnMode.GET_OLD_VALUE);
}
private Mono<Long> addAnd(long count, UpdateReturnMode updateReturnMode) {
var resultMono = singleton.update(prev -> {
try (prev) {
if (prev != null) {
var prevLong = prev.readLong();
var alloc = singleton.getAllocator();
var buf = alloc.allocate(Long.BYTES);
buf.writeLong(prevLong + count);
return buf;
} else {
var alloc = singleton.getAllocator();
var buf = alloc.allocate(Long.BYTES);
buf.writeLong(count);
return buf;
}
private Long addAnd(long count, UpdateReturnMode updateReturnMode) {
var result = singleton.update(prev -> {
if (prev != null) {
var prevLong = prev.getLong(0);
var buf = Buf.createZeroes(Long.BYTES);
buf.setLong(0, prevLong + count);
return buf;
} else {
var buf = Buf.createZeroes(Long.BYTES);
buf.setLong(0, count);
return buf;
}
}, updateReturnMode);
return Mono.usingWhen(resultMono,
result -> Mono.fromSupplier(result::readLong),
LLUtils::finalizeResource
).single();
return result.getLong(0);
}
public Mono<Void> set(long value) {
return singleton.set(Mono.fromCallable(() -> {
var buf = singleton.getAllocator().allocate(Long.BYTES);
try {
serializer.serialize(value, buf);
} catch (Throwable ex) {
buf.close();
throw ex;
}
return buf;
}));
public void set(long value) {
var buf = BufDataOutput.createLimited(Long.BYTES);
serializer.serialize(value, buf);
singleton.set(buf.asList());
}
@Override

562
src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionary.java
View File

@ -1,13 +1,11 @@
package it.cavallium.dbengine.database.collections;
import static java.util.Objects.requireNonNullElseGet;
import io.netty5.buffer.Buffer;
import io.netty5.util.Resource;
import io.netty5.buffer.internal.ResourceSupport;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.buffers.BufDataInput;
import it.cavallium.dbengine.buffers.BufDataOutput;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.BufSupplier;
import it.cavallium.dbengine.database.Delta;
import it.cavallium.dbengine.database.LLDelta;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLDictionaryResultType;
import it.cavallium.dbengine.database.LLEntry;
@ -22,22 +20,21 @@ import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import it.cavallium.dbengine.utils.InternalMonoUtils;
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMaps;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
/**
* Optimized implementation of "DatabaseMapDictionary with SubStageGetterSingle"
@ -50,11 +47,11 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
private final Serializer<U> valueSerializer;
protected DatabaseMapDictionary(LLDictionary dictionary,
@Nullable BufSupplier prefixKeySupplier,
@Nullable Buf prefixKey,
SerializerFixedBinaryLength<T> keySuffixSerializer,
Serializer<U> valueSerializer) {
// Do not retain or release or use the prefixKey here
super(dictionary, prefixKeySupplier, keySuffixSerializer, new SubStageGetterSingle<>(valueSerializer), 0);
super(dictionary, prefixKey, keySuffixSerializer, new SubStageGetterSingle<>(valueSerializer), 0);
this.valueSerializer = valueSerializer;
}
@ -65,65 +62,48 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
}
public static <T, U> DatabaseMapDictionary<T, U> tail(LLDictionary dictionary,
@Nullable BufSupplier prefixKeySupplier,
@Nullable Buf prefixKey,
SerializerFixedBinaryLength<T> keySuffixSerializer,
Serializer<U> valueSerializer) {
return new DatabaseMapDictionary<>(dictionary, prefixKeySupplier, keySuffixSerializer, valueSerializer);
return new DatabaseMapDictionary<>(dictionary, prefixKey, keySuffixSerializer, valueSerializer);
}
public static <K, V> Flux<Entry<K, V>> getLeavesFrom(DatabaseMapDictionary<K, V> databaseMapDictionary,
public static <K, V> Stream<Entry<K, V>> getLeavesFrom(DatabaseMapDictionary<K, V> databaseMapDictionary,
CompositeSnapshot snapshot,
Mono<K> keyMin,
Mono<K> keyMax,
boolean reverse, boolean smallRange) {
Mono<Optional<K>> keyMinOptMono = keyMin.map(Optional::of).defaultIfEmpty(Optional.empty());
Mono<Optional<K>> keyMaxOptMono = keyMax.map(Optional::of).defaultIfEmpty(Optional.empty());
@Nullable K keyMin,
@Nullable K keyMax,
boolean reverse,
boolean smallRange) {
return Mono.zip(keyMinOptMono, keyMaxOptMono).flatMapMany(entry -> {
var keyMinOpt = entry.getT1();
var keyMaxOpt = entry.getT2();
if (keyMinOpt.isPresent() || keyMaxOpt.isPresent()) {
return databaseMapDictionary.getAllValues(snapshot,
keyMinOpt.orElse(null),
keyMaxOpt.orElse(null),
reverse,
smallRange
);
} else {
return databaseMapDictionary.getAllValues(snapshot, smallRange);
}
});
if (keyMin != null || keyMax != null) {
return databaseMapDictionary.getAllValues(snapshot,
keyMin,
keyMax,
reverse,
smallRange
);
} else {
return databaseMapDictionary.getAllValues(snapshot, smallRange);
}
}
public static <K> Flux<K> getKeyLeavesFrom(DatabaseMapDictionary<K, ?> databaseMapDictionary,
public static <K> Stream<K> getKeyLeavesFrom(DatabaseMapDictionary<K, ?> databaseMapDictionary,
CompositeSnapshot snapshot,
Mono<K> keyMin,
Mono<K> keyMax,
boolean reverse, boolean smallRange) {
Mono<Optional<K>> keyMinOptMono = keyMin.map(Optional::of).defaultIfEmpty(Optional.empty());
Mono<Optional<K>> keyMaxOptMono = keyMax.map(Optional::of).defaultIfEmpty(Optional.empty());
@Nullable K keyMin,
@Nullable K keyMax,
boolean reverse,
boolean smallRange) {
return Mono.zip(keyMinOptMono, keyMaxOptMono).flatMapMany(keys -> {
var keyMinOpt = keys.getT1();
var keyMaxOpt = keys.getT2();
Flux<? extends Entry<K, ? extends DatabaseStageEntry<?>>> stagesFlux;
if (keyMinOpt.isPresent() || keyMaxOpt.isPresent()) {
stagesFlux = databaseMapDictionary
.getAllStages(snapshot, keyMinOpt.orElse(null), keyMaxOpt.orElse(null), reverse, smallRange);
} else {
stagesFlux = databaseMapDictionary.getAllStages(snapshot, smallRange);
}
return stagesFlux.doOnNext(e -> e.getValue().close())
.doOnDiscard(Entry.class, e -> {
if (e.getValue() instanceof DatabaseStageEntry<?> resource) {
LLUtils.onDiscard(resource);
}
})
.map(Entry::getKey);
});
Stream<? extends Entry<K, ? extends DatabaseStageEntry<?>>> stagesFlux;
if (keyMin != null || keyMax != null) {
stagesFlux = databaseMapDictionary.getAllStages(snapshot, keyMin, keyMax, reverse, smallRange);
} else {
stagesFlux = databaseMapDictionary.getAllStages(snapshot, smallRange);
}
return stagesFlux.map(Entry::getKey);
}
private @Nullable U deserializeValue(T keySuffix, Buffer value) {
private @Nullable U deserializeValue(T keySuffix, BufDataInput value) {
try {
return valueSerializer.deserialize(value);
} catch (IndexOutOfBoundsException ex) {
@ -131,19 +111,16 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
if (exMessage != null && exMessage.contains("read 0 to 0, write 0 to ")) {
var totalZeroBytesErrors = this.totalZeroBytesErrors.incrementAndGet();
if (totalZeroBytesErrors < 512 || totalZeroBytesErrors % 10000 == 0) {
try (var keyPrefix = keyPrefixSupplier.get()) {
try (var keySuffixBytes = serializeKeySuffixToKey(keySuffix)) {
LOG.error(
"Unexpected zero-bytes value at "
+ dictionary.getDatabaseName() + ":" + dictionary.getColumnName()
+ ":" + LLUtils.toStringSafe(keyPrefix) + ":" + keySuffix
+ "(" + LLUtils.toStringSafe(keySuffixBytes) + ") total=" + totalZeroBytesErrors);
} catch (SerializationException e) {
LOG.error(
"Unexpected zero-bytes value at " + dictionary.getDatabaseName() + ":" + dictionary.getColumnName()
+ ":" + LLUtils.toStringSafe(keyPrefix) + ":" + keySuffix + "(?) total="
+ totalZeroBytesErrors);
}
var keySuffixBytes = serializeKeySuffixToKey(keySuffix);
try {
LOG.error(
"Unexpected zero-bytes value at " + dictionary.getDatabaseName() + ":" + dictionary.getColumnName()
+ ":" + LLUtils.toStringSafe(keyPrefix) + ":" + keySuffix + "(" + LLUtils.toStringSafe(
keySuffixBytes) + ") total=" + totalZeroBytesErrors);
} catch (SerializationException e) {
LOG.error(
"Unexpected zero-bytes value at " + dictionary.getDatabaseName() + ":" + dictionary.getColumnName()
+ ":" + LLUtils.toStringSafe(keyPrefix) + ":" + keySuffix + "(?) total=" + totalZeroBytesErrors);
}
}
return null;
@ -153,139 +130,110 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
}
}
private Buffer serializeValue(U value) throws SerializationException {
private Buf serializeValue(U value) throws SerializationException {
var valSizeHint = valueSerializer.getSerializedSizeHint();
if (valSizeHint == -1) valSizeHint = 128;
var valBuf = dictionary.getAllocator().allocate(valSizeHint);
try {
valueSerializer.serialize(value, valBuf);
return valBuf;
} catch (Throwable t) {
valBuf.close();
throw t;
}
var valBuf = BufDataOutput.create(valSizeHint);
valueSerializer.serialize(value, valBuf);
return valBuf.asList();
}
private Buffer serializeKeySuffixToKey(T keySuffix) throws SerializationException {
Buffer keyBuf;
if (keyPrefixSupplier != null) {
keyBuf = keyPrefixSupplier.get();
} else {
keyBuf = this.dictionary.getAllocator().allocate(keyPrefixLength + keySuffixLength + keyExtLength);
}
try {
assert keyBuf.readableBytes() == keyPrefixLength;
keyBuf.ensureWritable(keySuffixLength + keyExtLength);
serializeSuffix(keySuffix, keyBuf);
assert keyBuf.readableBytes() == keyPrefixLength + keySuffixLength + keyExtLength;
return keyBuf;
} catch (Throwable t) {
keyBuf.close();
throw t;
private Buf serializeKeySuffixToKey(T keySuffix) throws SerializationException {
BufDataOutput keyBuf = BufDataOutput.createLimited(keyPrefixLength + keySuffixLength + keyExtLength);
if (keyPrefix != null) {
keyBuf.writeBytes(keyPrefix);
}
assert keyBuf.size() == keyPrefixLength;
serializeSuffixTo(keySuffix, keyBuf);
assert keyBuf.size() == keyPrefixLength + keySuffixLength + keyExtLength;
return keyBuf.asList();
}
private Buffer toKey(Buffer suffixKey) {
assert suffixKeyLengthConsistency(suffixKey.readableBytes());
if (keyPrefixSupplier != null) {
var result = LLUtils.compositeBuffer(dictionary.getAllocator(), keyPrefixSupplier.get().send(), suffixKey.send());
try {
assert result.readableBytes() == keyPrefixLength + keySuffixLength + keyExtLength;
return result;
} catch (Throwable t) {
result.close();
throw t;
}
private Buf toKey(Buf suffixKey) {
assert suffixKeyLengthConsistency(suffixKey.size());
if (keyPrefix != null) {
var result = keyPrefix.copy();
result.addAll(suffixKey);
assert result.size() == keyPrefixLength + keySuffixLength + keyExtLength;
return result;
} else {
assert suffixKey.readableBytes() == keyPrefixLength + keySuffixLength + keyExtLength;
assert suffixKey.size() == keyPrefixLength + keySuffixLength + keyExtLength;
return suffixKey;
}
}
@Override
public Mono<Object2ObjectSortedMap<T, U>> get(@Nullable CompositeSnapshot snapshot) {
return dictionary
.getRange(resolveSnapshot(snapshot), rangeMono, false, true)
public Object2ObjectSortedMap<T, U> get(@Nullable CompositeSnapshot snapshot) {
var map = dictionary
.getRange(resolveSnapshot(snapshot), range, false, true)
.map(entry -> {
Entry<T, U> deserializedEntry;
try (entry) {
T key;
var serializedKey = entry.getKeyUnsafe();
var serializedValue = entry.getValueUnsafe();
splitPrefix(serializedKey).close();
suffixKeyLengthConsistency(serializedKey.readableBytes());
key = deserializeSuffix(serializedKey);
U value = valueSerializer.deserialize(serializedValue);
deserializedEntry = Map.entry(key, value);
}
T key;
// serializedKey
var buf1 = BufDataInput.create(entry.getKey());
var serializedValue = BufDataInput.create(entry.getValue());
// after this, it becomes serializedSuffixAndExt
buf1.skipNBytes(keyPrefixLength);
suffixAndExtKeyConsistency(buf1.available());
key = deserializeSuffix(serializedValue);
U value = valueSerializer.deserialize(serializedValue);
deserializedEntry = Map.entry(key, value);
return deserializedEntry;
})
.collectMap(Entry::getKey, Entry::getValue, Object2ObjectLinkedOpenHashMap::new)
.map(map -> (Object2ObjectSortedMap<T, U>) map)
.filter(map -> !map.isEmpty());
.collect(Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) -> a, Object2ObjectLinkedOpenHashMap::new));
return map.isEmpty() ? null : map;
}
@Override
public Mono<Object2ObjectSortedMap<T, U>> setAndGetPrevious(Object2ObjectSortedMap<T, U> value) {
return this
.get(null)
.concatWith(dictionary
.setRange(rangeMono,
Flux.fromIterable(Collections.unmodifiableMap(value).entrySet()).map(entry -> serializeEntry(entry)),
true
)
.as(InternalMonoUtils::toAny))
.singleOrEmpty();
public Object2ObjectSortedMap<T, U> setAndGetPrevious(Object2ObjectSortedMap<T, U> value) {
Object2ObjectSortedMap<T, U> prev = this.get(null);
if (value == null || value.isEmpty()) {
dictionary.clear();
} else {
dictionary.setRange(range, value.entrySet().stream().map(this::serializeEntry), true);
}
return prev;
}
@Override
public Mono<Object2ObjectSortedMap<T, U>> clearAndGetPrevious() {
return this
.setAndGetPrevious(Object2ObjectSortedMaps.emptyMap());
public Object2ObjectSortedMap<T, U> clearAndGetPrevious() {
return this.setAndGetPrevious(Object2ObjectSortedMaps.emptyMap());
}
@Override
public Mono<Long> leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return dictionary.sizeRange(resolveSnapshot(snapshot), rangeMono, fast);
public long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return dictionary.sizeRange(resolveSnapshot(snapshot), range, fast);
}
@Override
public Mono<Boolean> isEmpty(@Nullable CompositeSnapshot snapshot) {
return dictionary.isRangeEmpty(resolveSnapshot(snapshot), rangeMono, false);
public boolean isEmpty(@Nullable CompositeSnapshot snapshot) {
return dictionary.isRangeEmpty(resolveSnapshot(snapshot), range, false);
}
@Override
public Mono<DatabaseStageEntry<U>> at(@Nullable CompositeSnapshot snapshot, T keySuffix) {
return Mono.fromCallable(() ->
new DatabaseMapSingle<>(dictionary, BufSupplier.ofOwned(serializeKeySuffixToKey(keySuffix)), valueSerializer));
public @NotNull DatabaseStageEntry<U> at(@Nullable CompositeSnapshot snapshot, T keySuffix) {
return new DatabaseMapSingle<>(dictionary, serializeKeySuffixToKey(keySuffix), valueSerializer);
}
@Override
public Mono<Boolean> containsKey(@Nullable CompositeSnapshot snapshot, T keySuffix) {
return dictionary
.isRangeEmpty(resolveSnapshot(snapshot),
Mono.fromCallable(() -> LLRange.singleUnsafe(serializeKeySuffixToKey(keySuffix))),
true
)
.map(empty -> !empty);
public boolean containsKey(@Nullable CompositeSnapshot snapshot, T keySuffix) {
return !dictionary.isRangeEmpty(resolveSnapshot(snapshot),
LLRange.single(serializeKeySuffixToKey(keySuffix)), true);
}
@Override
public Mono<U> getValue(@Nullable CompositeSnapshot snapshot, T keySuffix) {
return Mono.usingWhen(dictionary
.get(resolveSnapshot(snapshot), Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix))),
value -> Mono.fromCallable(() -> deserializeValue(keySuffix, value)),
LLUtils::finalizeResource);
public U getValue(@Nullable CompositeSnapshot snapshot, T keySuffix) {
var keySuffixBuf = serializeKeySuffixToKey(keySuffix);
Buf value = dictionary.get(resolveSnapshot(snapshot), keySuffixBuf);
return deserializeValue(keySuffix, BufDataInput.create(value));
}
@Override
public Mono<Void> putValue(T keySuffix, U value) {
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix)).single();
var valueMono = Mono.fromCallable(() -> serializeValue(value)).single();
return Mono.usingWhen(dictionary.put(keyMono, valueMono, LLDictionaryResultType.VOID),
v -> Mono.empty(),
LLUtils::finalizeResource
);
public void putValue(T keySuffix, U value) {
var keyMono = serializeKeySuffixToKey(keySuffix);
var valueMono = serializeValue(value);
dictionary.put(keyMono, valueMono, LLDictionaryResultType.VOID);
}
@Override
@ -294,21 +242,19 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
}
@Override
public Mono<U> updateValue(T keySuffix, UpdateReturnMode updateReturnMode,
public U updateValue(T keySuffix,
UpdateReturnMode updateReturnMode,
SerializationFunction<@Nullable U, @Nullable U> updater) {
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix));
return Mono.usingWhen(dictionary.update(keyMono, getSerializedUpdater(updater), updateReturnMode),
result -> Mono.fromCallable(() -> deserializeValue(keySuffix, result)),
LLUtils::finalizeResource
);
var keyMono = serializeKeySuffixToKey(keySuffix);
var result = dictionary.update(keyMono, getSerializedUpdater(updater), updateReturnMode);
return deserializeValue(keySuffix, BufDataInput.create(result));
}
@Override
public Mono<Delta<U>> updateValueAndGetDelta(T keySuffix, SerializationFunction<@Nullable U, @Nullable U> updater) {
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix));
return dictionary
.updateAndGetDelta(keyMono, getSerializedUpdater(updater))
.transform(mono -> LLUtils.mapLLDelta(mono, serialized -> valueSerializer.deserialize(serialized)));
public Delta<U> updateValueAndGetDelta(T keySuffix, SerializationFunction<@Nullable U, @Nullable U> updater) {
var keyMono = serializeKeySuffixToKey(keySuffix);
LLDelta delta = dictionary.updateAndGetDelta(keyMono, getSerializedUpdater(updater));
return LLUtils.mapLLDelta(delta, in -> valueSerializer.deserialize(BufDataInput.create(in)));
}
public BinarySerializationFunction getSerializedUpdater(SerializationFunction<@Nullable U, @Nullable U> updater) {
@ -317,9 +263,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
if (oldSerialized == null) {
result = updater.apply(null);
} else {
try (oldSerialized) {
result = updater.apply(valueSerializer.deserialize(oldSerialized));
}
result = updater.apply(valueSerializer.deserialize(BufDataInput.create(oldSerialized)));
}
if (result == null) {
return null;
@ -329,101 +273,81 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
};
}
public KVSerializationFunction<@NotNull T, @Nullable Buffer, @Nullable Buffer> getSerializedUpdater(
public KVSerializationFunction<@NotNull T, @Nullable Buf, @Nullable Buf> getSerializedUpdater(
KVSerializationFunction<@NotNull T, @Nullable U, @Nullable U> updater) {
return (key, oldSerialized) -> {
try (oldSerialized) {
U result;
if (oldSerialized == null) {
result = updater.apply(key, null);
} else {
try (oldSerialized) {
result = updater.apply(key, valueSerializer.deserialize(oldSerialized));
}
}
if (result == null) {
return null;
} else {
return serializeValue(result);
}
U result;
if (oldSerialized == null) {
result = updater.apply(key, null);
} else {
result = updater.apply(key, valueSerializer.deserialize(BufDataInput.create(oldSerialized)));
}
if (result == null) {
return null;
} else {
return serializeValue(result);
}
};
}
@Override
public Mono<U> putValueAndGetPrevious(T keySuffix, U value) {
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix));
var valueMono = Mono.fromCallable(() -> serializeValue(value));
return Mono.usingWhen(dictionary.put(keyMono, valueMono, LLDictionaryResultType.PREVIOUS_VALUE),
valueBuf -> Mono.fromCallable(() -> deserializeValue(keySuffix, valueBuf)),
LLUtils::finalizeResource
);
public U putValueAndGetPrevious(T keySuffix, U value) {
var keyMono = serializeKeySuffixToKey(keySuffix);
var valueMono = serializeValue(value);
var valueBuf = dictionary.put(keyMono, valueMono, LLDictionaryResultType.PREVIOUS_VALUE);
return deserializeValue(keySuffix, BufDataInput.create(valueBuf));
}
@Override
public Mono<Boolean> putValueAndGetChanged(T keySuffix, U value) {
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix));
var valueMono = Mono.fromCallable(() -> serializeValue(value));
return Mono
.usingWhen(dictionary.put(keyMono, valueMono, LLDictionaryResultType.PREVIOUS_VALUE),
valueBuf -> Mono.fromCallable(() -> deserializeValue(keySuffix, valueBuf)),
LLUtils::finalizeResource
)
.map(oldValue -> !Objects.equals(oldValue, value))
.defaultIfEmpty(value != null);
public boolean putValueAndGetChanged(T keySuffix, U value) {
var keyMono = serializeKeySuffixToKey(keySuffix);
var valueMono = serializeValue(value);
var oldValueBuf = dictionary.put(keyMono, valueMono, LLDictionaryResultType.PREVIOUS_VALUE);
var oldValue = deserializeValue(keySuffix, BufDataInput.create(oldValueBuf));
if (oldValue == null) {
return value != null;
} else {
return !Objects.equals(oldValue, value);
}
}
@Override
public Mono<Void> remove(T keySuffix) {
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix));
return dictionary
.remove(keyMono, LLDictionaryResultType.VOID)
.doOnNext(LLUtils::finalizeResourceNow)
.then();
public void remove(T keySuffix) {
var keyMono = serializeKeySuffixToKey(keySuffix);
dictionary.remove(keyMono, LLDictionaryResultType.VOID);
}
@Override
public Mono<U> removeAndGetPrevious(T keySuffix) {
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix));
return Mono.usingWhen(dictionary.remove(keyMono, LLDictionaryResultType.PREVIOUS_VALUE),
valueBuf -> Mono.fromCallable(() -> deserializeValue(keySuffix, valueBuf)),
LLUtils::finalizeResource
);
public U removeAndGetPrevious(T keySuffix) {
var keyMono = serializeKeySuffixToKey(keySuffix);
var valueBuf = dictionary.remove(keyMono, LLDictionaryResultType.PREVIOUS_VALUE);
return deserializeValue(keySuffix, BufDataInput.create(valueBuf));
}
@Override
public Mono<Boolean> removeAndGetStatus(T keySuffix) {
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix));
return dictionary
.remove(keyMono, LLDictionaryResultType.PREVIOUS_VALUE_EXISTENCE)
.map(response -> LLUtils.responseToBoolean(response));
public boolean removeAndGetStatus(T keySuffix) {
var keyMono = serializeKeySuffixToKey(keySuffix);
return LLUtils.responseToBoolean(dictionary.remove(keyMono, LLDictionaryResultType.PREVIOUS_VALUE_EXISTENCE));
}
@Override
public Flux<Optional<U>> getMulti(@Nullable CompositeSnapshot snapshot, Flux<T> keys) {
public Stream<Optional<U>> getMulti(@Nullable CompositeSnapshot snapshot, Stream<T> keys) {
var mappedKeys = keys.map(keySuffix -> serializeKeySuffixToKey(keySuffix));
return dictionary
.getMulti(resolveSnapshot(snapshot), mappedKeys)
.map(valueBufOpt -> {
try (valueBufOpt) {
if (valueBufOpt.isPresent()) {
return Optional.of(valueSerializer.deserialize(valueBufOpt.get()));
} else {
return Optional.empty();
}
if (valueBufOpt.isPresent()) {
return Optional.of(valueSerializer.deserialize(BufDataInput.create(valueBufOpt.get())));
} else {
return Optional.empty();
}
});
}
private LLEntry serializeEntry(T keySuffix, U value) throws SerializationException {
var key = serializeKeySuffixToKey(keySuffix);
try {
var serializedValue = serializeValue(value);
return LLEntry.of(key, serializedValue);
} catch (Throwable t) {
key.close();
throw t;
}
var serializedValue = serializeValue(value);
return LLEntry.of(key, serializedValue);
}
private LLEntry serializeEntry(Entry<T, U> entry) throws SerializationException {
@ -431,59 +355,56 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
}
@Override
public Mono<Void> putMulti(Flux<Entry<T, U>> entries) {
var serializedEntries = entries.map(entry -> serializeEntry(entry));
return dictionary.putMulti(serializedEntries);
public void putMulti(Stream<Entry<T, U>> entries) {
try (var serializedEntries = entries.map(entry -> serializeEntry(entry))) {
dictionary.putMulti(serializedEntries);
}
}
@Override
public Flux<Boolean> updateMulti(Flux<T> keys,
public Stream<Boolean> updateMulti(Stream<T> keys,
KVSerializationFunction<T, @Nullable U, @Nullable U> updater) {
var sharedKeys = keys.publish().refCount(2);
var serializedKeys = sharedKeys.map(keySuffix -> serializeKeySuffixToKey(keySuffix));
List<T> sharedKeys = keys.toList();
var serializedKeys = sharedKeys.stream().map(keySuffix -> serializeKeySuffixToKey(keySuffix));
var serializedUpdater = getSerializedUpdater(updater);
return dictionary.updateMulti(sharedKeys, serializedKeys, serializedUpdater);
return dictionary.updateMulti(sharedKeys.stream(), serializedKeys, serializedUpdater);
}
@Override
public Flux<SubStageEntry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
return getAllStages(snapshot, rangeMono, false, smallRange);
public Stream<SubStageEntry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
return getAllStages(snapshot, range, false, smallRange);
}
private LLRange getPatchedRange(@NotNull LLRange range, @Nullable T keyMin, @Nullable T keyMax)
throws SerializationException {
Buffer keyMinBuf = serializeSuffixForRange(keyMin);
Buf keyMinBuf = serializeSuffixForRange(keyMin);
if (keyMinBuf == null) {
keyMinBuf = range.getMinCopy();
keyMinBuf = range.getMin();
}
Buffer keyMaxBuf = serializeSuffixForRange(keyMax);
Buf keyMaxBuf = serializeSuffixForRange(keyMax);
if (keyMaxBuf == null) {
keyMaxBuf = range.getMaxCopy();
keyMaxBuf = range.getMax();
}
return LLRange.ofUnsafe(keyMinBuf, keyMaxBuf);
return LLRange.of(keyMinBuf, keyMaxBuf);
}
private Buffer serializeSuffixForRange(@Nullable T key) throws SerializationException {
private Buf serializeSuffixForRange(@Nullable T key) throws SerializationException {
if (key == null) {
return null;
}
var keyWithoutExtBuf =
keyPrefixSupplier == null ? alloc.allocate(keySuffixLength + keyExtLength) : keyPrefixSupplier.get();
try {
keyWithoutExtBuf.ensureWritable(keySuffixLength + keyExtLength);
serializeSuffix(key, keyWithoutExtBuf);
return keyWithoutExtBuf;
} catch (Throwable ex) {
keyWithoutExtBuf.close();
throw ex;
var keyWithoutExtBuf = BufDataOutput.createLimited(keyPrefixLength + keySuffixLength);
if (keyPrefix != null) {
keyWithoutExtBuf.writeBytes(keyPrefix);
}
serializeSuffixTo(key, keyWithoutExtBuf);
return keyWithoutExtBuf.asList();
}
/**
* Get all stages
* @param reverse if true, the results will go backwards from the specified key (inclusive)
*/
public Flux<SubStageEntry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot,
public Stream<SubStageEntry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot,
@Nullable T keyMin,
@Nullable T keyMax,
boolean reverse,
@ -491,48 +412,38 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
if (keyMin == null && keyMax == null) {
return getAllStages(snapshot, smallRange);
} else {
Mono<LLRange> boundedRangeMono = rangeMono.map(range -> {
try (range) {
return getPatchedRange(range, keyMin, keyMax);
}
});
return getAllStages(snapshot, boundedRangeMono, reverse, smallRange);
LLRange boundedRange = getPatchedRange(range, keyMin, keyMax);
return getAllStages(snapshot, boundedRange, reverse, smallRange);
}
}
private Flux<SubStageEntry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot,
Mono<LLRange> sliceRangeMono, boolean reverse, boolean smallRange) {
private Stream<SubStageEntry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot,
LLRange sliceRange, boolean reverse, boolean smallRange) {
return dictionary
.getRangeKeys(resolveSnapshot(snapshot), sliceRangeMono, reverse, smallRange)
.getRangeKeys(resolveSnapshot(snapshot), sliceRange, reverse, smallRange)
.map(keyBuf -> {
try (keyBuf) {
assert keyBuf.readableBytes() == keyPrefixLength + keySuffixLength + keyExtLength;
// Remove prefix. Keep only the suffix and the ext
splitPrefix(keyBuf).close();
suffixKeyLengthConsistency(keyBuf.readableBytes());
var bufSupplier = BufSupplier.ofOwned(toKey(keyBuf.copy()));
try {
T keySuffix = deserializeSuffix(keyBuf);
var subStage = new DatabaseMapSingle<>(dictionary, bufSupplier, valueSerializer);
return new SubStageEntry<>(keySuffix, subStage);
} catch (Throwable ex) {
bufSupplier.close();
throw ex;
}
}
assert keyBuf.size() == keyPrefixLength + keySuffixLength + keyExtLength;
// Remove prefix. Keep only the suffix and the ext
var suffixAndExtIn = BufDataInput.create(keyBuf);
suffixAndExtIn.skipBytes(keyPrefixLength);
suffixKeyLengthConsistency(suffixAndExtIn.available());
T keySuffix = deserializeSuffix(suffixAndExtIn);
var subStage = new DatabaseMapSingle<>(dictionary, keyBuf, valueSerializer);
return new SubStageEntry<>(keySuffix, subStage);
});
}
@Override
public Flux<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
return getAllValues(snapshot, rangeMono, false, smallRange);
public Stream<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
return getAllValues(snapshot, range, false, smallRange);
}
/**
* Get all values
* @param reverse if true, the results will go backwards from the specified key (inclusive)
*/
public Flux<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot,
public Stream<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot,
@Nullable T keyMin,
@Nullable T keyMax,
boolean reverse,
@ -540,59 +451,52 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
if (keyMin == null && keyMax == null) {
return getAllValues(snapshot, smallRange);
} else {
Mono<LLRange> boundedRangeMono = Mono.usingWhen(rangeMono,
range -> Mono.fromCallable(() -> getPatchedRange(range, keyMin, keyMax)),
LLUtils::finalizeResource);
return getAllValues(snapshot, boundedRangeMono, reverse, smallRange);
LLRange boundedRange = getPatchedRange(range, keyMin, keyMax);
return getAllValues(snapshot, boundedRange, reverse, smallRange);
}
}
private Flux<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot,
Mono<LLRange> sliceRangeMono,
private Stream<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot,
LLRange sliceRangeMono,
boolean reverse, boolean smallRange) {
return dictionary
.getRange(resolveSnapshot(snapshot), sliceRangeMono, reverse, smallRange)
.map((serializedEntry) -> {
Entry<T, U> entry;
try (serializedEntry) {
var keyBuf = serializedEntry.getKeyUnsafe();
assert keyBuf != null;
assert keyBuf.readableBytes() == keyPrefixLength + keySuffixLength + keyExtLength;
// Remove prefix. Keep only the suffix and the ext
splitPrefix(keyBuf).close();
assert suffixKeyLengthConsistency(keyBuf.readableBytes());
T keySuffix = deserializeSuffix(keyBuf);
var keyBuf = serializedEntry.getKey();
assert keyBuf != null;
assert keyBuf.size() == keyPrefixLength + keySuffixLength + keyExtLength;
assert serializedEntry.getValueUnsafe() != null;
U value = valueSerializer.deserialize(serializedEntry.getValueUnsafe());
entry = Map.entry(keySuffix, value);
}
// Remove prefix. Keep only the suffix and the ext
var suffixAndExtIn = BufDataInput.create(keyBuf);
suffixAndExtIn.skipBytes(keyPrefixLength);
assert suffixKeyLengthConsistency(suffixAndExtIn.available());
T keySuffix = deserializeSuffix(suffixAndExtIn);
assert serializedEntry.getValue() != null;
U value = valueSerializer.deserialize(BufDataInput.create(serializedEntry.getValue()));
entry = Map.entry(keySuffix, value);
return entry;
});
}
@Override
public Flux<Entry<T, U>> setAllValuesAndGetPrevious(Flux<Entry<T, U>> entries) {
return Flux.usingWhen(Mono.just(true),
b -> this.getAllValues(null, false),
b -> dictionary.setRange(rangeMono, entries.map(entry -> serializeEntry(entry)), false)
);
public Stream<Entry<T, U>> setAllValuesAndGetPrevious(Stream<Entry<T, U>> entries) {
var previous = this.getAllValues(null, false);
dictionary.setRange(range, entries.map(entry -> serializeEntry(entry)), false);
return previous;
}
@Override
public Mono<Void> clear() {
return Mono.using(() -> rangeSupplier.get(), range -> {
if (range.isAll()) {
return dictionary.clear();
} else if (range.isSingle()) {
return dictionary
.remove(Mono.fromCallable(() -> range.getSingleUnsafe()), LLDictionaryResultType.VOID)
.doOnNext(LLUtils::finalizeResourceNow)
.then();
} else {
return dictionary.setRange(rangeMono, Flux.empty(), false);
}
}, LLUtils::finalizeResourceNow);
public void clear() {
if (range.isAll()) {
dictionary.clear();
} else if (range.isSingle()) {
dictionary.remove(range.getSingleUnsafe(), LLDictionaryResultType.VOID);
} else {
dictionary.setRange(range, Stream.empty(), false);
}
}
}

452
src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionaryDeep.java
View File

@ -1,137 +1,106 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.BufferAllocator;
import io.netty5.buffer.DefaultBufferAllocators;
import io.netty5.buffer.Drop;
import io.netty5.buffer.Owned;
import io.netty5.util.Resource;
import io.netty5.buffer.internal.ResourceSupport;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.buffers.BufDataInput;
import it.cavallium.dbengine.buffers.BufDataOutput;
import it.cavallium.dbengine.client.BadBlock;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.BufSupplier;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLDictionaryResultType;
import it.cavallium.dbengine.database.LLRange;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.RangeSupplier;
import it.cavallium.dbengine.database.SubStageEntry;
import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.collections.DatabaseEmpty.Nothing;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import it.cavallium.dbengine.utils.InternalMonoUtils;
import it.cavallium.dbengine.utils.SimpleResource;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletionException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Stream;
import org.apache.commons.lang3.function.TriFunction;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
// todo: implement optimized methods (which?)
public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extends SimpleResource implements
DatabaseStageMap<T, U, US> {
public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implements DatabaseStageMap<T, U, US> {
private static final Logger LOG = LogManager.getLogger(DatabaseMapDictionaryDeep.class);
protected final LLDictionary dictionary;
protected final BufferAllocator alloc;
private final AtomicLong totalZeroBytesErrors = new AtomicLong();
protected final SubStageGetter<U, US> subStageGetter;
protected final SerializerFixedBinaryLength<T> keySuffixSerializer;
protected final int keyPrefixLength;
protected final int keySuffixLength;
protected final int keyExtLength;
protected final Mono<LLRange> rangeMono;
protected final LLRange range;
protected RangeSupplier rangeSupplier;
protected BufSupplier keyPrefixSupplier;
protected Buf keyPrefix;
private static void incrementPrefix(Buffer prefix, int prefixLength) {
assert prefix.readableBytes() >= prefixLength;
assert prefix.readerOffset() == 0;
final var originalKeyLength = prefix.readableBytes();
private static void incrementPrefix(Buf modifiablePrefix, int prefixLength) {
assert modifiablePrefix.size() >= prefixLength;
final var originalKeyLength = modifiablePrefix.size();
boolean overflowed = true;
final int ff = 0xFF;
int writtenBytes = 0;
for (int i = prefixLength - 1; i >= 0; i--) {
int iByte = prefix.getUnsignedByte(i);
int iByte = Byte.toUnsignedInt(modifiablePrefix.getByte(i));
if (iByte != ff) {
prefix.setUnsignedByte(i, iByte + 1);
modifiablePrefix.set(i, (byte) (iByte + 1));
writtenBytes++;
overflowed = false;
break;
} else {
prefix.setUnsignedByte(i, 0x00);
modifiablePrefix.set(i, (byte) 0x00);
writtenBytes++;
}
}
assert prefixLength - writtenBytes >= 0;
if (overflowed) {
assert prefix.writerOffset() == originalKeyLength;
prefix.ensureWritable(1, 1, true);
prefix.writerOffset(originalKeyLength + 1);
modifiablePrefix.add((byte) 0);
for (int i = 0; i < originalKeyLength; i++) {
prefix.setUnsignedByte(i, 0xFF);
modifiablePrefix.set(i, (byte) 0xFF);
}
prefix.setUnsignedByte(originalKeyLength, (byte) 0x00);
modifiablePrefix.set(originalKeyLength, (byte) 0x00);
}
}
static void firstRangeKey(Buffer prefixKey, int prefixLength, Buffer suffixAndExtZeroes) {
zeroFillKeySuffixAndExt(prefixKey, prefixLength, suffixAndExtZeroes);
static Buf firstRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
var modifiablePrefixKey = Buf.create(prefixLength + suffixAndExtZeroes.size());
modifiablePrefixKey.addAll(prefixKey);
zeroFillKeySuffixAndExt(modifiablePrefixKey, prefixLength, suffixAndExtZeroes);
return modifiablePrefixKey;
}
static void nextRangeKey(Buffer prefixKey, int prefixLength, Buffer suffixAndExtZeroes) {
zeroFillKeySuffixAndExt(prefixKey, prefixLength, suffixAndExtZeroes);
incrementPrefix(prefixKey, prefixLength);
static Buf nextRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
var modifiablePrefixKey = Buf.create(prefixLength + suffixAndExtZeroes.size());
modifiablePrefixKey.addAll(prefixKey);
zeroFillKeySuffixAndExt(modifiablePrefixKey, prefixLength, suffixAndExtZeroes);
incrementPrefix(modifiablePrefixKey, prefixLength);
return modifiablePrefixKey;
}
@Deprecated
static void firstRangeKey(Buffer prefixKey, int prefixLength, int suffixLength, int extLength) {
try (var zeroBuf = DefaultBufferAllocators.offHeapAllocator().allocate(suffixLength + extLength)) {
zeroBuf.fill((byte) 0);
zeroBuf.writerOffset(suffixLength + extLength);
zeroFillKeySuffixAndExt(prefixKey, prefixLength, zeroBuf);
}
}
@Deprecated
static void nextRangeKey(Buffer prefixKey, int prefixLength, int suffixLength, int extLength) {
try (var zeroBuf = DefaultBufferAllocators.offHeapAllocator().allocate(suffixLength + extLength)) {
zeroBuf.fill((byte) 0);
zeroBuf.writerOffset(suffixLength + extLength);
zeroFillKeySuffixAndExt(prefixKey, prefixLength, zeroBuf);
incrementPrefix(prefixKey, prefixLength);
}
}
protected static void zeroFillKeySuffixAndExt(@NotNull Buffer prefixKey,
int prefixLength, Buffer suffixAndExtZeroes) {
/**
* @param modifiablePrefixKey This field content will be modified
*/
protected static void zeroFillKeySuffixAndExt(@NotNull Buf modifiablePrefixKey, int prefixLength, Buf suffixAndExtZeroes) {
//noinspection UnnecessaryLocalVariable
var result = prefixKey;
var suffixLengthAndExtLength = suffixAndExtZeroes.readableBytes();
assert result.readableBytes() == prefixLength;
var result = modifiablePrefixKey;
var suffixLengthAndExtLength = suffixAndExtZeroes.size();
assert result.size() == prefixLength;
assert suffixLengthAndExtLength > 0 : "Suffix length + ext length is < 0: " + suffixLengthAndExtLength;
prefixKey.ensureWritable(suffixLengthAndExtLength);
suffixAndExtZeroes.copyInto(suffixAndExtZeroes.readerOffset(),
prefixKey,
prefixKey.writerOffset(),
suffixLengthAndExtLength
);
prefixKey.skipWritableBytes(suffixLengthAndExtLength);
result.size(prefixLength + suffixLengthAndExtLength);
modifiablePrefixKey.addAll(suffixAndExtZeroes);
assert modifiablePrefixKey.size() == prefixLength + suffixAndExtZeroes.size() : "Result buffer size is wrong";
}
/**
@ -150,94 +119,55 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
}
public static <T, U, US extends DatabaseStage<U>> DatabaseMapDictionaryDeep<T, U, US> deepIntermediate(
LLDictionary dictionary, BufSupplier prefixKey, SerializerFixedBinaryLength<T> keySuffixSerializer,
LLDictionary dictionary, Buf prefixKey, SerializerFixedBinaryLength<T> keySuffixSerializer,
SubStageGetter<U, US> subStageGetter, int keyExtLength) {
return new DatabaseMapDictionaryDeep<>(dictionary, prefixKey, keySuffixSerializer, subStageGetter, keyExtLength);
}
@SuppressWarnings({"unchecked", "rawtypes"})
protected DatabaseMapDictionaryDeep(LLDictionary dictionary, @Nullable BufSupplier prefixKeySupplier,
protected DatabaseMapDictionaryDeep(LLDictionary dictionary, @Nullable Buf prefixKey,
SerializerFixedBinaryLength<T> keySuffixSerializer, SubStageGetter<U, US> subStageGetter, int keyExtLength) {
try (var prefixKey = prefixKeySupplier != null ? prefixKeySupplier.get() : null) {
this.dictionary = dictionary;
this.alloc = dictionary.getAllocator();
this.subStageGetter = subStageGetter;
this.keySuffixSerializer = keySuffixSerializer;
this.keyPrefixLength = prefixKey != null ? prefixKey.readableBytes() : 0;
this.keySuffixLength = keySuffixSerializer.getSerializedBinaryLength();
this.keyExtLength = keyExtLength;
try (var keySuffixAndExtZeroBuffer = alloc
.allocate(keySuffixLength + keyExtLength)
.fill((byte) 0)
.writerOffset(keySuffixLength + keyExtLength)
.makeReadOnly()) {
assert keySuffixAndExtZeroBuffer.readableBytes() == keySuffixLength + keyExtLength :
"Key suffix and ext zero buffer readable length is not equal"
+ " to the key suffix length + key ext length. keySuffixAndExtZeroBuffer="
+ keySuffixAndExtZeroBuffer.readableBytes() + " keySuffixLength=" + keySuffixLength + " keyExtLength="
+ keyExtLength;
assert keySuffixAndExtZeroBuffer.readableBytes() > 0;
var firstKey = prefixKey != null ? prefixKeySupplier.get()
: alloc.allocate(keyPrefixLength + keySuffixLength + keyExtLength);
try {
firstRangeKey(firstKey, keyPrefixLength, keySuffixAndExtZeroBuffer);
var nextRangeKey = prefixKey != null ? prefixKeySupplier.get()
: alloc.allocate(keyPrefixLength + keySuffixLength + keyExtLength);
try {
nextRangeKey(nextRangeKey, keyPrefixLength, keySuffixAndExtZeroBuffer);
assert prefixKey == null || prefixKey.isAccessible();
assert keyPrefixLength == 0 || !LLUtils.equals(firstKey, nextRangeKey);
if (keyPrefixLength == 0) {
this.rangeSupplier = RangeSupplier.ofOwned(LLRange.all());
firstKey.close();
nextRangeKey.close();
} else {
this.rangeSupplier = RangeSupplier.ofOwned(LLRange.ofUnsafe(firstKey, nextRangeKey));
}
this.rangeMono = Mono.fromSupplier(rangeSupplier);
assert subStageKeysConsistency(keyPrefixLength + keySuffixLength + keyExtLength);
} catch (Throwable t) {
nextRangeKey.close();
throw t;
}
} catch (Throwable t) {
firstKey.close();
throw t;
}
this.keyPrefixSupplier = prefixKeySupplier;
}
} catch (Throwable t) {
if (prefixKeySupplier != null) {
prefixKeySupplier.close();
}
throw t;
this.dictionary = dictionary;
this.subStageGetter = subStageGetter;
this.keySuffixSerializer = keySuffixSerializer;
this.keyPrefixLength = prefixKey != null ? prefixKey.size() : 0;
this.keySuffixLength = keySuffixSerializer.getSerializedBinaryLength();
this.keyExtLength = keyExtLength;
var keySuffixAndExtZeroBuffer = Buf.createZeroes(keySuffixLength + keyExtLength);
assert keySuffixAndExtZeroBuffer.size() == keySuffixLength + keyExtLength :
"Key suffix and ext zero buffer readable length is not equal"
+ " to the key suffix length + key ext length. keySuffixAndExtZeroBuffer="
+ keySuffixAndExtZeroBuffer.size() + " keySuffixLength=" + keySuffixLength + " keyExtLength="
+ keyExtLength;
assert keySuffixAndExtZeroBuffer.size() > 0;
var firstKey = firstRangeKey(prefixKey, keyPrefixLength, keySuffixAndExtZeroBuffer);
var nextRangeKey = nextRangeKey(prefixKey, keyPrefixLength, keySuffixAndExtZeroBuffer);
assert keyPrefixLength == 0 || !LLUtils.equals(firstKey, nextRangeKey);
if (keyPrefixLength == 0) {
this.range = LLRange.all();
} else {
this.range = LLRange.of(firstKey, nextRangeKey);
}
}
assert subStageKeysConsistency(keyPrefixLength + keySuffixLength + keyExtLength);
@SuppressWarnings({"unchecked", "rawtypes"})
this.keyPrefix = prefixKey;
}
private DatabaseMapDictionaryDeep(LLDictionary dictionary,
BufferAllocator alloc,
SubStageGetter<U, US> subStageGetter,
SerializerFixedBinaryLength<T> keySuffixSerializer,
int keyPrefixLength,
int keySuffixLength,
int keyExtLength,
Mono<LLRange> rangeMono,
RangeSupplier rangeSupplier,
BufSupplier keyPrefixSupplier,
Runnable onClose) {
LLRange range,
Buf keyPrefix) {
this.dictionary = dictionary;
this.alloc = alloc;
this.subStageGetter = subStageGetter;
this.keySuffixSerializer = keySuffixSerializer;
this.keyPrefixLength = keyPrefixLength;
this.keySuffixLength = keySuffixLength;
this.keyExtLength = keyExtLength;
this.rangeMono = rangeMono;
this.range = range;
this.rangeSupplier = rangeSupplier;
this.keyPrefixSupplier = keyPrefixSupplier;
this.keyPrefix = keyPrefix;
}
@SuppressWarnings("unused")
@ -256,16 +186,39 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
}
/**
* Removes the prefix from the key
* @return the prefix
*/
protected Buffer splitPrefix(Buffer key) {
assert key.readableBytes() == keyPrefixLength + keySuffixLength + keyExtLength
|| key.readableBytes() == keyPrefixLength + keySuffixLength;
var prefix = key.readSplit(this.keyPrefixLength);
assert key.readableBytes() == keySuffixLength + keyExtLength
|| key.readableBytes() == keySuffixLength;
return prefix;
protected Buf prefixSubList(Buf key) {
assert key.size() == keyPrefixLength + keySuffixLength + keyExtLength
|| key.size() == keyPrefixLength + keySuffixLength;
return key.subList(0, this.keyPrefixLength);
}
/**
* @return the suffix
*/
protected Buf suffixSubList(Buf key) {
assert key.size() == keyPrefixLength + keySuffixLength + keyExtLength
|| key.size() == keyPrefixLength + keySuffixLength;
return key.subList(this.keyPrefixLength, keyPrefixLength + keySuffixLength);
}
/**
* @return the suffix
*/
protected Buf suffixAndExtSubList(Buf key) {
assert key.size() == keyPrefixLength + keySuffixLength + keyExtLength
|| key.size() == keyPrefixLength + keySuffixLength;
return key.subList(this.keyPrefixLength, key.size());
}
/**
* @return the ext
*/
protected Buf extSubList(Buf key) {
assert key.size() == keyPrefixLength + keySuffixLength + keyExtLength
|| key.size() == keyPrefixLength + keySuffixLength;
return key.subList(this.keyPrefixLength + this.keySuffixLength, key.size());
}
protected LLSnapshot resolveSnapshot(@Nullable CompositeSnapshot snapshot) {
@ -277,30 +230,23 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
}
@Override
public Mono<Long> leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return dictionary.sizeRange(resolveSnapshot(snapshot), rangeMono, fast);
public long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return dictionary.sizeRange(resolveSnapshot(snapshot), range, fast);
}
@Override
public Mono<Boolean> isEmpty(@Nullable CompositeSnapshot snapshot) {
return dictionary.isRangeEmpty(resolveSnapshot(snapshot), rangeMono, false);
public boolean isEmpty(@Nullable CompositeSnapshot snapshot) {
return dictionary.isRangeEmpty(resolveSnapshot(snapshot), range, false);
}
@Override
public Mono<US> at(@Nullable CompositeSnapshot snapshot, T keySuffix) {
var suffixKeyWithoutExt = Mono.fromCallable(() -> {
var keyWithoutExtBuf = keyPrefixSupplier == null
? alloc.allocate(keySuffixLength + keyExtLength) : keyPrefixSupplier.get();
try {
keyWithoutExtBuf.ensureWritable(keySuffixLength + keyExtLength);
serializeSuffix(keySuffix, keyWithoutExtBuf);
} catch (Throwable ex) {
keyWithoutExtBuf.close();
throw ex;
}
return keyWithoutExtBuf;
});
return this.subStageGetter.subStage(dictionary, snapshot, suffixKeyWithoutExt);
public @NotNull US at(@Nullable CompositeSnapshot snapshot, T keySuffix) {
BufDataOutput bufOutput = BufDataOutput.createLimited(keyPrefixLength + keySuffixLength + keyExtLength);
if (keyPrefix != null) {
bufOutput.writeBytes(keyPrefix);
}
serializeSuffixTo(keySuffix, bufOutput);
return this.subStageGetter.subStage(dictionary, snapshot, bufOutput.asList());
}
@Override
@ -309,39 +255,22 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
}
@Override
public Flux<BadBlock> badBlocks() {
return dictionary.badBlocks(rangeMono);
public Stream<BadBlock> badBlocks() {
return dictionary.badBlocks(range);
}
@Override
public Flux<SubStageEntry<T, US>> getAllStages(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
public Stream<SubStageEntry<T, US>> getAllStages(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
return dictionary
.getRangeKeyPrefixes(resolveSnapshot(snapshot), rangeMono, keyPrefixLength + keySuffixLength, smallRange)
.flatMapSequential(groupKeyWithoutExt -> this.subStageGetter
.subStage(dictionary, snapshot, Mono.fromCallable(() -> groupKeyWithoutExt.copy()))
.map(us -> {
T deserializedSuffix;
try (var splittedGroupSuffix = splitGroupSuffix(groupKeyWithoutExt)) {
deserializedSuffix = this.deserializeSuffix(splittedGroupSuffix);
return new SubStageEntry<>(deserializedSuffix, us);
}
})
.doFinally(s -> groupKeyWithoutExt.close())
);
}
/**
* Split the input. The input will become the ext, the returned data will be the group suffix
* @param groupKey group key, will become ext
* @return group suffix
*/
private Buffer splitGroupSuffix(@NotNull Buffer groupKey) {
assert subStageKeysConsistency(groupKey.readableBytes())
|| subStageKeysConsistency(groupKey.readableBytes() + keyExtLength);
this.splitPrefix(groupKey).close();
assert subStageKeysConsistency(keyPrefixLength + groupKey.readableBytes())
|| subStageKeysConsistency(keyPrefixLength + groupKey.readableBytes() + keyExtLength);
return groupKey.readSplit(keySuffixLength);
.getRangeKeyPrefixes(resolveSnapshot(snapshot), range, keyPrefixLength + keySuffixLength, smallRange)
.parallel()
.map(groupKeyWithoutExt -> {
T deserializedSuffix;
var splittedGroupSuffix = suffixSubList(groupKeyWithoutExt);
deserializedSuffix = this.deserializeSuffix(BufDataInput.create(splittedGroupSuffix));
return new SubStageEntry<>(deserializedSuffix,
this.subStageGetter.subStage(dictionary, snapshot, groupKeyWithoutExt));
});
}
private boolean subStageKeysConsistency(int totalKeyLength) {
@ -357,51 +286,45 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
}
@Override
public Flux<Entry<T, U>> setAllValuesAndGetPrevious(Flux<Entry<T, U>> entries) {
return this
.getAllValues(null, false)
.concatWith(this
.clear()
.then(this.putMulti(entries))
.as(InternalMonoUtils::toAny)
);
public void setAllValues(Stream<Entry<T, U>> entries) {
this.clear();
this.putMulti(entries);
}
@Override
public Mono<Void> clear() {
return Mono.using(() -> rangeSupplier.get(), range -> {
if (range.isAll()) {
return dictionary.clear();
} else if (range.isSingle()) {
return dictionary
.remove(Mono.fromCallable(() -> range.getSingleUnsafe()), LLDictionaryResultType.VOID)
.doOnNext(resource -> LLUtils.finalizeResourceNow(resource))
.then();
} else {
return dictionary.setRange(rangeMono, Flux.empty(), false);
}
}, resource -> LLUtils.finalizeResourceNow(resource));
public Stream<Entry<T, U>> setAllValuesAndGetPrevious(Stream<Entry<T, U>> entries) {
return this.getAllValues(null, false).onClose(() -> setAllValues(entries));
}
protected T deserializeSuffix(@NotNull Buffer keySuffix) throws SerializationException {
assert suffixKeyLengthConsistency(keySuffix.readableBytes());
var result = keySuffixSerializer.deserialize(keySuffix);
return result;
@Override
public void clear() {
if (range.isAll()) {
dictionary.clear();
} else if (range.isSingle()) {
dictionary.remove(range.getSingleUnsafe(), LLDictionaryResultType.VOID);
} else {
dictionary.setRange(range, Stream.empty(), false);
}
}
protected void serializeSuffix(T keySuffix, Buffer output) throws SerializationException {
output.ensureWritable(keySuffixLength);
var beforeWriterOffset = output.writerOffset();
protected T deserializeSuffix(@NotNull BufDataInput keySuffix) throws SerializationException {
assert suffixKeyLengthConsistency(keySuffix.available());
return keySuffixSerializer.deserialize(keySuffix);
}
protected void serializeSuffixTo(T keySuffix, BufDataOutput output) throws SerializationException {
assert suffixKeyLengthConsistency(output.size());
var beforeWriterOffset = output.size();
keySuffixSerializer.serialize(keySuffix, output);
var afterWriterOffset = output.writerOffset();
var afterWriterOffset = output.size();
assert suffixKeyLengthConsistency(afterWriterOffset - beforeWriterOffset)
: "Invalid key suffix length: " + (afterWriterOffset - beforeWriterOffset) + ". Expected: " + keySuffixLength;
}
public static <K1, K2, V, R> Flux<R> getAllLeaves2(DatabaseMapDictionaryDeep<K1, Object2ObjectSortedMap<K2, V>, ? extends DatabaseStageMap<K2, V, DatabaseStageEntry<V>>> deepMap,
public static <K1, K2, V, R> Stream<R> getAllLeaves2(DatabaseMapDictionaryDeep<K1, Object2ObjectSortedMap<K2, V>, ? extends DatabaseStageMap<K2, V, DatabaseStageEntry<V>>> deepMap,
CompositeSnapshot snapshot,
TriFunction<K1, K2, V, R> merger,
@NotNull Mono<K1> savedProgressKey1) {
@Nullable K1 savedProgressKey1) {
var keySuffix1Serializer = deepMap.keySuffixSerializer;
SerializerFixedBinaryLength<?> keySuffix2Serializer;
Serializer<?> valueSerializer;
@ -434,64 +357,47 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
throw new IllegalArgumentException();
}
var savedProgressKey1Opt = savedProgressKey1.map(value1 -> Optional.of(value1)).defaultIfEmpty(Optional.empty());
var firstKey = Optional.ofNullable(savedProgressKey1);
var fullRange = deepMap.range;
return deepMap
.dictionary
.getRange(deepMap.resolveSnapshot(snapshot), Mono.zip(savedProgressKey1Opt, deepMap.rangeMono).handle((tuple, sink) -> {
var firstKey = tuple.getT1();
var fullRange = tuple.getT2();
try {
if (firstKey.isPresent()) {
try (fullRange) {
try (var key1Buf = deepMap.alloc.allocate(keySuffix1Serializer.getSerializedBinaryLength())) {
keySuffix1Serializer.serialize(firstKey.get(), key1Buf);
sink.next(LLRange.of(key1Buf.send(), fullRange.getMax()));
} catch (SerializationException e) {
sink.error(e);
}
}
} else {
sink.next(fullRange);
}
} catch (Throwable ex) {
try {
fullRange.close();
} catch (Throwable ex2) {
LOG.error(ex2);
}
sink.error(ex);
}
}), false, false)
.concatMapIterable(entry -> {
LLRange range;
if (firstKey.isPresent()) {
var key1Buf = BufDataOutput.create(keySuffix1Serializer.getSerializedBinaryLength());
keySuffix1Serializer.serialize(firstKey.get(), key1Buf);
range = LLRange.of(key1Buf.asList(), fullRange.getMax());
} else {
range = fullRange;
}
return deepMap.dictionary.getRange(deepMap.resolveSnapshot(snapshot), range, false, false)
.flatMap(entry -> {
K1 key1 = null;
Object key2 = null;
try (entry) {
var keyBuf = entry.getKeyUnsafe();
var valueBuf = entry.getValueUnsafe();
try {
var keyBuf = entry.getKey();
var valueBuf = entry.getValue();
try {
assert keyBuf != null;
keyBuf.skipReadableBytes(deepMap.keyPrefixLength);
try (var key1Buf = keyBuf.split(deepMap.keySuffixLength)) {
key1 = keySuffix1Serializer.deserialize(key1Buf);
}
key2 = keySuffix2Serializer.deserialize(keyBuf);
var suffix1And2 = BufDataInput.create(keyBuf.subList(deepMap.keyPrefixLength, deepMap.keyPrefixLength + deepMap.keySuffixLength + deepMap.keyExtLength));
key1 = keySuffix1Serializer.deserialize(suffix1And2);
key2 = keySuffix2Serializer.deserialize(suffix1And2);
assert valueBuf != null;
Object value = valueSerializer.deserialize(valueBuf);
Object value = valueSerializer.deserialize(BufDataInput.create(valueBuf));
if (isHashedSet) {
//noinspection unchecked
Set<K2> set = (Set<K2>) value;
K1 finalKey1 = key1;
//noinspection unchecked
return set.stream().map(e -> merger.apply(finalKey1, e, (V) Nothing.INSTANCE)).toList();
return set.stream().map(e -> merger.apply(finalKey1, e, (V) Nothing.INSTANCE));
} else if (isHashed) {
//noinspection unchecked
Set<Entry<K2, V>> set = (Set<Entry<K2, V>>) value;
K1 finalKey1 = key1;
return set.stream().map(e -> merger.apply(finalKey1, e.getKey(), e.getValue())).toList();
return set.stream().map(e -> merger.apply(finalKey1, e.getKey(), e.getValue()));
} else {
//noinspection unchecked
return List.of(merger.apply(key1, (K2) key2, (V) value));
return Stream.of(merger.apply(key1, (K2) key2, (V) value));
}
} catch (IndexOutOfBoundsException ex) {
var exMessage = ex.getMessage();
@ -504,7 +410,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
+ ":" + key2
+ "](" + LLUtils.toStringSafe(keyBuf) + ") total=" + totalZeroBytesErrors);
}
return List.of();
return Stream.empty();
} else {
throw ex;
}
@ -514,22 +420,4 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
}
});
}
@Override
protected void onClose() {
try {
if (rangeSupplier != null) {
rangeSupplier.close();
}
} catch (Throwable ex) {
LOG.error("Failed to close range", ex);
}
try {
if (keyPrefixSupplier != null) {
keyPrefixSupplier.close();
}
} catch (Throwable ex) {
LOG.error("Failed to close keyPrefix", ex);
}
}
}

139
src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionaryHashed.java
View File

@ -1,49 +1,40 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.BufferAllocator;
import io.netty5.buffer.Drop;
import io.netty5.buffer.Owned;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.client.BadBlock;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.BufSupplier;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLUtils;
import io.netty5.buffer.internal.ResourceSupport;
import it.cavallium.dbengine.database.SubStageEntry;
import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import it.cavallium.dbengine.utils.SimpleResource;
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import it.unimi.dsi.fastutil.objects.ObjectArraySet;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@SuppressWarnings("unused")
public class DatabaseMapDictionaryHashed<T, U, TH> extends SimpleResource implements
DatabaseStageMap<T, U, DatabaseStageEntry<U>> {
public class DatabaseMapDictionaryHashed<T, U, TH> implements DatabaseStageMap<T, U, DatabaseStageEntry<U>> {
private static final Logger logger = LogManager.getLogger(DatabaseMapDictionaryHashed.class);
private final BufferAllocator alloc;
private final Function<T, TH> keySuffixHashFunction;
private final DatabaseMapDictionary<TH, ObjectArraySet<Entry<T, U>>> subDictionary;
protected DatabaseMapDictionaryHashed(LLDictionary dictionary,
@Nullable BufSupplier prefixKeySupplier,
@Nullable Buf prefixKeySupplier,
Serializer<T> keySuffixSerializer,
Serializer<U> valueSerializer,
Function<T, TH> keySuffixHashFunction,
@ -52,7 +43,6 @@ public class DatabaseMapDictionaryHashed<T, U, TH> extends SimpleResource implem
if (updateMode != UpdateMode.ALLOW) {
throw new IllegalArgumentException("Hashed maps only works when UpdateMode is ALLOW");
}
this.alloc = dictionary.getAllocator();
ValueWithHashSerializer<T, U> valueWithHashSerializer
= new ValueWithHashSerializer<>(keySuffixSerializer, valueSerializer);
ValuesSetSerializer<Entry<T, U>> valuesSetSerializer
@ -62,11 +52,8 @@ public class DatabaseMapDictionaryHashed<T, U, TH> extends SimpleResource implem
this.keySuffixHashFunction = keySuffixHashFunction;
}
private DatabaseMapDictionaryHashed(BufferAllocator alloc,
Function<T, TH> keySuffixHashFunction,
DatabaseStage<Object2ObjectSortedMap<TH, ObjectArraySet<Entry<T, U>>>> subDictionary,
Drop<DatabaseMapDictionaryHashed<T, U, TH>> drop) {
this.alloc = alloc;
private DatabaseMapDictionaryHashed(Function<T, TH> keySuffixHashFunction,
DatabaseStage<Object2ObjectSortedMap<TH, ObjectArraySet<Entry<T, U>>>> subDictionary) {
this.keySuffixHashFunction = keySuffixHashFunction;
this.subDictionary = (DatabaseMapDictionary<TH, ObjectArraySet<Entry<T, U>>>) subDictionary;
@ -88,7 +75,7 @@ public class DatabaseMapDictionaryHashed<T, U, TH> extends SimpleResource implem
}
public static <T, U, UH> DatabaseMapDictionaryHashed<T, U, UH> tail(LLDictionary dictionary,
@Nullable BufSupplier prefixKeySupplier,
@Nullable Buf prefixKeySupplier,
Serializer<T> keySuffixSerializer,
Serializer<U> valueSerializer,
Function<T, UH> keySuffixHashFunction,
@ -121,36 +108,35 @@ public class DatabaseMapDictionaryHashed<T, U, TH> extends SimpleResource implem
}
@Override
public Mono<Object2ObjectSortedMap<T, U>> get(@Nullable CompositeSnapshot snapshot) {
return subDictionary.get(snapshot).map(map -> deserializeMap(map));
public Object2ObjectSortedMap<T, U> get(@Nullable CompositeSnapshot snapshot) {
var v = subDictionary.get(snapshot);
return v != null ? deserializeMap(v) : null;
}
@Override
public Mono<Object2ObjectSortedMap<T, U>> getOrDefault(@Nullable CompositeSnapshot snapshot,
Mono<Object2ObjectSortedMap<T, U>> defaultValue) {
return this.get(snapshot).switchIfEmpty(defaultValue);
public Object2ObjectSortedMap<T, U> getOrDefault(@Nullable CompositeSnapshot snapshot,
Object2ObjectSortedMap<T, U> defaultValue) {
return Objects.requireNonNullElse(this.get(snapshot), defaultValue);
}
@Override
public Mono<Void> set(Object2ObjectSortedMap<T, U> map) {
return Mono.fromSupplier(() -> this.serializeMap(map)).flatMap(value -> subDictionary.set(value));
public void set(Object2ObjectSortedMap<T, U> map) {
var value = this.serializeMap(map);
subDictionary.set(value);
}
@Override
public Mono<Boolean> setAndGetChanged(Object2ObjectSortedMap<T, U> map) {
return Mono
.fromSupplier(() -> this.serializeMap(map))
.flatMap(value -> subDictionary.setAndGetChanged(value))
.single();
public boolean setAndGetChanged(Object2ObjectSortedMap<T, U> map) {
return subDictionary.setAndGetChanged(this.serializeMap(map));
}
@Override
public Mono<Boolean> clearAndGetStatus() {
public boolean clearAndGetStatus() {
return subDictionary.clearAndGetStatus();
}
@Override
public Mono<Boolean> isEmpty(@Nullable CompositeSnapshot snapshot) {
public boolean isEmpty(@Nullable CompositeSnapshot snapshot) {
return subDictionary.isEmpty(snapshot);
}
@ -160,20 +146,17 @@ public class DatabaseMapDictionaryHashed<T, U, TH> extends SimpleResource implem
}
@Override
public Flux<BadBlock> badBlocks() {
public Stream<BadBlock> badBlocks() {
return this.subDictionary.badBlocks();
}
@Override
public Mono<DatabaseStageEntry<U>> at(@Nullable CompositeSnapshot snapshot, T key) {
return this
.atPrivate(snapshot, key, keySuffixHashFunction.apply(key))
.map(cast -> cast);
public @NotNull DatabaseStageEntry<U> at(@Nullable CompositeSnapshot snapshot, T key) {
return this.atPrivate(snapshot, key, keySuffixHashFunction.apply(key));
}
private Mono<DatabaseSingleBucket<T, U, TH>> atPrivate(@Nullable CompositeSnapshot snapshot, T key, TH hash) {
return subDictionary.at(snapshot, hash)
.map(entry -> new DatabaseSingleBucket<T, U, TH>(entry, key));
private DatabaseSingleBucket<T, U, TH> atPrivate(@Nullable CompositeSnapshot snapshot, T key, TH hash) {
return new DatabaseSingleBucket<T, U, TH>(subDictionary.at(snapshot, hash), key);
}
@Override
@ -182,57 +165,55 @@ public class DatabaseMapDictionaryHashed<T, U, TH> extends SimpleResource implem
}
@Override
public Flux<SubStageEntry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot,
public Stream<SubStageEntry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot,
boolean smallRange) {
return subDictionary
.getAllValues(snapshot, smallRange)
.map(Entry::getValue)
.map(Collections::unmodifiableSet)
.flatMap(bucket -> Flux
.fromIterable(bucket)
.flatMap(bucket -> bucket.stream()
.map(Entry::getKey)
.flatMap(key -> this.at(snapshot, key).map(stage -> new SubStageEntry<>(key, stage))));
.map(key -> new SubStageEntry<>(key, this.at(snapshot, key))));
}
@Override
public Flux<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
public Stream<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
return subDictionary
.getAllValues(snapshot, smallRange)
.map(Entry::getValue)
.map(Collections::unmodifiableSet)
.concatMapIterable(list -> list);
.flatMap(Collection::stream);
}
@Override
public Flux<Entry<T, U>> setAllValuesAndGetPrevious(Flux<Entry<T, U>> entries) {
return entries.flatMap(entry -> Mono.usingWhen(this.at(null, entry.getKey()),
stage -> stage.setAndGetPrevious(entry.getValue()).map(prev -> Map.entry(entry.getKey(), prev)),
LLUtils::finalizeResource
));
public Stream<Entry<T, U>> setAllValuesAndGetPrevious(Stream<Entry<T, U>> entries) {
return entries.mapMulti((entry, sink) -> {
var prev = this.at(null, entry.getKey()).setAndGetPrevious(entry.getValue());
if (prev != null) {
sink.accept(Map.entry(entry.getKey(), prev));
}
});
}
@Override
public Mono<Void> clear() {
return subDictionary.clear();
public void clear() {
subDictionary.clear();
}
@Override
public Mono<Object2ObjectSortedMap<T, U>> setAndGetPrevious(Object2ObjectSortedMap<T, U> value) {
return Mono
.fromSupplier(() -> this.serializeMap(value))
.flatMap(value1 -> subDictionary.setAndGetPrevious(value1))
.map(map -> deserializeMap(map));
public Object2ObjectSortedMap<T, U> setAndGetPrevious(Object2ObjectSortedMap<T, U> value) {
var v = subDictionary.setAndGetPrevious(this.serializeMap(value));
return v != null ? deserializeMap(v) : null;
}
@Override
public Mono<Object2ObjectSortedMap<T, U>> clearAndGetPrevious() {
return subDictionary
.clearAndGetPrevious()
.map(map -> deserializeMap(map));
public Object2ObjectSortedMap<T, U> clearAndGetPrevious() {
var v = subDictionary.clearAndGetPrevious();
return v != null ? deserializeMap(v) : null;
}
@Override
public Mono<Long> leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
public long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return subDictionary.leavesCount(snapshot, fast);
}
@ -245,13 +226,14 @@ public class DatabaseMapDictionaryHashed<T, U, TH> extends SimpleResource implem
@Override
public ValueGetter<T, U> getAsyncDbValueGetter(@Nullable CompositeSnapshot snapshot) {
ValueGetter<TH, ObjectArraySet<Entry<T, U>>> getter = subDictionary.getAsyncDbValueGetter(snapshot);
return key -> getter
.get(keySuffixHashFunction.apply(key))
.flatMap(set -> this.extractValueTransformation(set, key));
}
private Mono<U> extractValueTransformation(ObjectArraySet<Entry<T, U>> entries, T key) {
return Mono.fromCallable(() -> extractValue(entries, key));
return key -> {
ObjectArraySet<Entry<T, U>> set = getter.get(keySuffixHashFunction.apply(key));
if (set != null) {
return this.extractValue(set, key);
} else {
return null;
}
};
}
@Nullable
@ -299,15 +281,4 @@ public class DatabaseMapDictionaryHashed<T, U, TH> extends SimpleResource implem
return null;
}
}
@Override
protected void onClose() {
try {
if (subDictionary != null) {
subDictionary.close();
}
} catch (Throwable ex) {
logger.error("Failed to close subDictionary", ex);
}
}
}

171
src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapSingle.java
View File

@ -1,12 +1,10 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.Drop;
import io.netty5.buffer.Owned;
import io.netty5.buffer.internal.ResourceSupport;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.buffers.BufDataInput;
import it.cavallium.dbengine.buffers.BufDataOutput;
import it.cavallium.dbengine.client.BadBlock;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.BufSupplier;
import it.cavallium.dbengine.database.Delta;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLDictionaryResultType;
@ -14,32 +12,26 @@ import it.cavallium.dbengine.database.LLRange;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.disk.BinarySerializationFunction;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.utils.SimpleResource;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public class DatabaseMapSingle<U> extends SimpleResource implements DatabaseStageEntry<U> {
public final class DatabaseMapSingle<U> implements DatabaseStageEntry<U> {
private static final Logger LOG = LogManager.getLogger(DatabaseMapSingle.class);
private final AtomicLong totalZeroBytesErrors = new AtomicLong();
private final LLDictionary dictionary;
private final Mono<Buffer> keyMono;
private final Buf key;
private final Serializer<U> serializer;
private final BufSupplier keySupplier;
public DatabaseMapSingle(LLDictionary dictionary, BufSupplier keySupplier, Serializer<U> serializer) {
public DatabaseMapSingle(LLDictionary dictionary, Buf key, Serializer<U> serializer) {
this.dictionary = dictionary;
this.keySupplier = keySupplier;
this.keyMono = Mono.fromSupplier(() -> keySupplier.get());
this.key = key;
this.serializer = serializer;
}
@ -51,127 +43,98 @@ public class DatabaseMapSingle<U> extends SimpleResource implements DatabaseStag
}
}
private U deserializeValue(Buffer value) {
private U deserializeValue(Buf value) {
try {
return serializer.deserialize(value);
return serializer.deserialize(BufDataInput.create(value));
} catch (IndexOutOfBoundsException ex) {
var exMessage = ex.getMessage();
if (exMessage != null && exMessage.contains("read 0 to 0, write 0 to ")) {
try (var key = keySupplier.get()) {
LOG.error("Unexpected zero-bytes value at "
+ dictionary.getDatabaseName() + ":" + dictionary.getColumnName() + ":" + LLUtils.toStringSafe(key));
}
LOG.error("Unexpected zero-bytes value at %s:%s:%s".formatted(dictionary.getDatabaseName(),
dictionary.getColumnName(),
LLUtils.toStringSafe(key)
));
return null;
} else {
throw ex;
}
} catch (SerializationException ex) {
throw ex;
}
}
private Buffer serializeValue(U value) throws SerializationException {
var valSizeHint = serializer.getSerializedSizeHint();
if (valSizeHint == -1) valSizeHint = 128;
var valBuf = dictionary.getAllocator().allocate(valSizeHint);
try {
serializer.serialize(value, valBuf);
return valBuf;
} catch (Throwable ex) {
valBuf.close();
throw ex;
private Buf serializeValue(U value) throws SerializationException {
BufDataOutput valBuf = BufDataOutput.create(serializer.getSerializedSizeHint());
serializer.serialize(value, valBuf);
return valBuf.asList();
}
@Override
public U get(@Nullable CompositeSnapshot snapshot) {
var result = dictionary.get(resolveSnapshot(snapshot), key);
if (result != null) {
return deserializeValue(result);
} else {
return null;
}
}
@Override
public Mono<U> get(@Nullable CompositeSnapshot snapshot) {
return Mono.usingWhen(dictionary.get(resolveSnapshot(snapshot), keyMono),
buf -> Mono.fromSupplier(() -> deserializeValue(buf)),
LLUtils::finalizeResource
);
public U setAndGetPrevious(U value) {
var serializedKey = value != null ? serializeValue(value) : null;
var result = dictionary.put(key, serializedKey, LLDictionaryResultType.PREVIOUS_VALUE);
if (result != null) {
return deserializeValue(result);
} else {
return null;
}
}
@Override
public Mono<U> setAndGetPrevious(U value) {
return Mono.usingWhen(dictionary
.put(keyMono, Mono.fromCallable(() -> serializeValue(value)), LLDictionaryResultType.PREVIOUS_VALUE),
buf -> Mono.fromSupplier(() -> deserializeValue(buf)),
LLUtils::finalizeResource);
}
@Override
public Mono<U> update(SerializationFunction<@Nullable U, @Nullable U> updater,
public U update(SerializationFunction<@Nullable U, @Nullable U> updater,
UpdateReturnMode updateReturnMode) {
var resultMono = dictionary
.update(keyMono, (oldValueSer) -> {
try (oldValueSer) {
U result;
if (oldValueSer == null) {
result = updater.apply(null);
} else {
U deserializedValue = serializer.deserialize(oldValueSer);
result = updater.apply(deserializedValue);
}
if (result == null) {
return null;
} else {
return serializeValue(result);
}
}
}, updateReturnMode);
return Mono.usingWhen(resultMono,
result -> Mono.fromSupplier(() -> deserializeValue(result)),
LLUtils::finalizeResource
);
Buf resultBytes = dictionary.update(key, this.createUpdater(updater), updateReturnMode);
return deserializeValue(resultBytes);
}
@Override
public Mono<Delta<U>> updateAndGetDelta(SerializationFunction<@Nullable U, @Nullable U> updater) {
return dictionary
.updateAndGetDelta(keyMono, (oldValueSer) -> {
U result;
if (oldValueSer == null) {
result = updater.apply(null);
} else {
U deserializedValue = serializer.deserialize(oldValueSer);
result = updater.apply(deserializedValue);
}
if (result == null) {
return null;
} else {
return serializeValue(result);
}
}).transform(mono -> LLUtils.mapLLDelta(mono, serialized -> serializer.deserialize(serialized)));
public Delta<U> updateAndGetDelta(SerializationFunction<@Nullable U, @Nullable U> updater) {
var delta = dictionary.updateAndGetDelta(key, this.createUpdater(updater));
return LLUtils.mapLLDelta(delta, bytes -> serializer.deserialize(BufDataInput.create(bytes)));
}
private BinarySerializationFunction createUpdater(SerializationFunction<U, U> updater) {
return oldBytes -> {
U result;
if (oldBytes == null) {
result = updater.apply(null);
} else {
U deserializedValue = serializer.deserialize(BufDataInput.create(oldBytes));
result = updater.apply(deserializedValue);
}
if (result == null) {
return null;
} else {
return serializeValue(result);
}
};
}
@Override
public Mono<U> clearAndGetPrevious() {
return Mono.usingWhen(dictionary.remove(keyMono, LLDictionaryResultType.PREVIOUS_VALUE),
result -> Mono.fromSupplier(() -> deserializeValue(result)),
LLUtils::finalizeResource
);
public U clearAndGetPrevious() {
return deserializeValue(dictionary.remove(key, LLDictionaryResultType.PREVIOUS_VALUE));
}
@Override
public Mono<Long> leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return dictionary
.isRangeEmpty(resolveSnapshot(snapshot), keyMono.map(single -> LLRange.singleUnsafe(single)), false)
.map(empty -> empty ? 0L : 1L);
public long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return dictionary.isRangeEmpty(resolveSnapshot(snapshot), LLRange.single(key), false) ? 0L : 1L;
}
@Override
public Mono<Boolean> isEmpty(@Nullable CompositeSnapshot snapshot) {
return dictionary
.isRangeEmpty(resolveSnapshot(snapshot), keyMono.map(single -> LLRange.singleUnsafe(single)), true);
public boolean isEmpty(@Nullable CompositeSnapshot snapshot) {
return dictionary.isRangeEmpty(resolveSnapshot(snapshot), LLRange.single(key), true);
}
@Override
public Flux<BadBlock> badBlocks() {
return dictionary.badBlocks(keyMono.map(single -> LLRange.singleUnsafe(single)));
public Stream<BadBlock> badBlocks() {
return dictionary.badBlocks(LLRange.single(key));
}
@Override
protected void onClose() {
keySupplier.close();
}
}

34
src/main/java/it/cavallium/dbengine/database/collections/DatabaseSetDictionary.java
View File

@ -1,32 +1,21 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.Drop;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.BufSupplier;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.collections.DatabaseEmpty.Nothing;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
@SuppressWarnings("unused")
public class DatabaseSetDictionary<T> extends DatabaseMapDictionary<T, Nothing> {
protected DatabaseSetDictionary(LLDictionary dictionary,
BufSupplier prefixKeySupplier,
Buf prefixKeySupplier,
SerializerFixedBinaryLength<T> keySuffixSerializer) {
super(dictionary,
prefixKeySupplier,
keySuffixSerializer,
DatabaseEmpty.nothingSerializer(dictionary.getAllocator())
);
super(dictionary, prefixKeySupplier, keySuffixSerializer, DatabaseEmpty.nothingSerializer());
}
public static <T> DatabaseSetDictionary<T> simple(LLDictionary dictionary,
@ -35,24 +24,27 @@ public class DatabaseSetDictionary<T> extends DatabaseMapDictionary<T, Nothing>
}
public static <T> DatabaseSetDictionary<T> tail(LLDictionary dictionary,
BufSupplier prefixKeySupplier,
Buf prefixKeySupplier,
SerializerFixedBinaryLength<T> keySuffixSerializer) {
return new DatabaseSetDictionary<>(dictionary, prefixKeySupplier, keySuffixSerializer);
}
public Mono<Set<T>> getKeySet(@Nullable CompositeSnapshot snapshot) {
return get(snapshot).map(Map::keySet);
public Set<T> getKeySet(@Nullable CompositeSnapshot snapshot) {
var v = get(snapshot);
return v != null ? v.keySet() : null;
}
public Mono<Set<T>> setAndGetPreviousKeySet(Set<T> value) {
public Set<T> setAndGetPreviousKeySet(Set<T> value) {
var hm = new Object2ObjectLinkedOpenHashMap<T, Nothing>();
for (T t : value) {
hm.put(t, DatabaseEmpty.NOTHING);
}
return setAndGetPrevious(hm).map(Map::keySet);
var v = setAndGetPrevious(hm);
return v != null ? v.keySet() : null;
}
public Mono<Set<T>> clearAndGetPreviousKeySet() {
return clearAndGetPrevious().map(Map::keySet);
public Set<T> clearAndGetPreviousKeySet() {
var v = clearAndGetPrevious();
return v != null ? v.keySet() : null;
}
}

31
src/main/java/it/cavallium/dbengine/database/collections/DatabaseSetDictionaryHashed.java
View File

@ -1,36 +1,28 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.Drop;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.BufSupplier;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.collections.DatabaseEmpty.Nothing;
import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
@SuppressWarnings("unused")
public class DatabaseSetDictionaryHashed<T, TH> extends DatabaseMapDictionaryHashed<T, Nothing, TH> {
protected DatabaseSetDictionaryHashed(LLDictionary dictionary,
@Nullable BufSupplier prefixKeySupplier,
@Nullable Buf prefixKeySupplier,
Serializer<T> keySuffixSerializer,
Function<T, TH> keySuffixHashFunction,
SerializerFixedBinaryLength<TH> keySuffixHashSerializer) {
super(dictionary,
prefixKeySupplier,
keySuffixSerializer,
DatabaseEmpty.nothingSerializer(dictionary.getAllocator()),
DatabaseEmpty.nothingSerializer(),
keySuffixHashFunction,
keySuffixHashSerializer
);
@ -49,7 +41,7 @@ public class DatabaseSetDictionaryHashed<T, TH> extends DatabaseMapDictionaryHas
}
public static <T, TH> DatabaseSetDictionaryHashed<T, TH> tail(LLDictionary dictionary,
@Nullable BufSupplier prefixKeySupplier,
@Nullable Buf prefixKeySupplier,
Serializer<T> keySuffixSerializer,
Function<T, TH> keyHashFunction,
SerializerFixedBinaryLength<TH> keyHashSerializer) {
@ -61,19 +53,22 @@ public class DatabaseSetDictionaryHashed<T, TH> extends DatabaseMapDictionaryHas
);
}
public Mono<Set<T>> getKeySet(@Nullable CompositeSnapshot snapshot) {
return get(snapshot).map(Map::keySet);
public Set<T> getKeySet(@Nullable CompositeSnapshot snapshot) {
var v = get(snapshot);
return v != null ? v.keySet() : null;
}
public Mono<Set<T>> setAndGetPreviousKeySet(Set<T> value) {
public Set<T> setAndGetPreviousKeySet(Set<T> value) {
var hm = new Object2ObjectLinkedOpenHashMap<T, Nothing>();
for (T t : value) {
hm.put(t, DatabaseEmpty.NOTHING);
}
return setAndGetPrevious(hm).map(Map::keySet);
var v = setAndGetPrevious(hm);
return v != null ? v.keySet() : null;
}
public Mono<Set<T>> clearAndGetPreviousKeySet() {
return clearAndGetPrevious().map(Map::keySet);
public Set<T> clearAndGetPreviousKeySet() {
var v = clearAndGetPrevious();
return v != null ? v.keySet() : null;
}
}

80
src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleBucket.java
View File

@ -1,30 +1,24 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Drop;
import io.netty5.buffer.Owned;
import io.netty5.util.Send;
import it.cavallium.dbengine.client.BadBlock;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.Delta;
import it.cavallium.dbengine.database.LLUtils;
import io.netty5.buffer.internal.ResourceSupport;
import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.cavallium.dbengine.utils.SimpleResource;
import it.unimi.dsi.fastutil.objects.ObjectArraySet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@SuppressWarnings("unused")
public class DatabaseSingleBucket<K, V, TH> extends SimpleResource implements DatabaseStageEntry<V> {
public class DatabaseSingleBucket<K, V, TH> implements DatabaseStageEntry<V> {
private static final Logger logger = LogManager.getLogger(DatabaseSingleBucket.class);
@ -43,33 +37,35 @@ public class DatabaseSingleBucket<K, V, TH> extends SimpleResource implements Da
}
@Override
public Mono<V> get(@Nullable CompositeSnapshot snapshot) {
return bucketStage.get(snapshot).flatMap(entries -> extractValueTransformation(entries));
public V get(@Nullable CompositeSnapshot snapshot) {
var entries = bucketStage.get(snapshot);
return entries != null ? extractValue(entries) : null;
}
@Override
public Mono<V> getOrDefault(@Nullable CompositeSnapshot snapshot, Mono<V> defaultValue) {
return bucketStage.get(snapshot).flatMap(entries -> extractValueTransformation(entries)).switchIfEmpty(defaultValue);
public V getOrDefault(@Nullable CompositeSnapshot snapshot, V defaultValue) {
var entries = bucketStage.get(snapshot);
return entries != null ? extractValue(entries) : defaultValue;
}
@Override
public Mono<Void> set(V value) {
return this.update(prev -> value, UpdateReturnMode.NOTHING).then();
public void set(V value) {
this.update(prev -> value, UpdateReturnMode.NOTHING);
}
@Override
public Mono<V> setAndGetPrevious(V value) {
public V setAndGetPrevious(V value) {
return this.update(prev -> value, UpdateReturnMode.GET_OLD_VALUE);
}
@Override
public Mono<Boolean> setAndGetChanged(V value) {
return this.updateAndGetDelta(prev -> value).map(delta -> LLUtils.isDeltaChanged(delta));
public boolean setAndGetChanged(V value) {
return LLUtils.isDeltaChanged(this.updateAndGetDelta(prev -> value));
}
@Override
public Mono<V> update(SerializationFunction<@Nullable V, @Nullable V> updater, UpdateReturnMode updateReturnMode) {
return bucketStage
public V update(SerializationFunction<@Nullable V, @Nullable V> updater, UpdateReturnMode updateReturnMode) {
var result = bucketStage
.update(oldBucket -> {
V oldValue = extractValue(oldBucket);
V newValue = updater.apply(oldValue);
@ -79,13 +75,13 @@ public class DatabaseSingleBucket<K, V, TH> extends SimpleResource implements Da
} else {
return this.insertValueOrCreate(oldBucket, newValue);
}
}, updateReturnMode)
.flatMap(entries -> extractValueTransformation(entries));
}, updateReturnMode);
return result != null ? extractValue(result) : null;
}
@Override
public Mono<Delta<V>> updateAndGetDelta(SerializationFunction<@Nullable V, @Nullable V> updater) {
return bucketStage.updateAndGetDelta(oldBucket -> {
public Delta<V> updateAndGetDelta(SerializationFunction<@Nullable V, @Nullable V> updater) {
var delta = bucketStage.updateAndGetDelta(oldBucket -> {
V oldValue = extractValue(oldBucket);
var result = updater.apply(oldValue);
if (result == null) {
@ -93,32 +89,33 @@ public class DatabaseSingleBucket<K, V, TH> extends SimpleResource implements Da
} else {
return this.insertValueOrCreate(oldBucket, result);
}
}).transform(mono -> LLUtils.mapDelta(mono, entries -> extractValue(entries)));
});
return LLUtils.mapDelta(delta, this::extractValue);
}
@Override
public Mono<Void> clear() {
return this.update(prev -> null, UpdateReturnMode.NOTHING).then();
public void clear() {
this.update(prev -> null, UpdateReturnMode.NOTHING);
}
@Override
public Mono<V> clearAndGetPrevious() {
public V clearAndGetPrevious() {
return this.update(prev -> null, UpdateReturnMode.GET_OLD_VALUE);
}
@Override
public Mono<Boolean> clearAndGetStatus() {
return this.updateAndGetDelta(prev -> null).map(delta -> LLUtils.isDeltaChanged(delta));
public boolean clearAndGetStatus() {
return LLUtils.isDeltaChanged(this.updateAndGetDelta(prev -> null));
}
@Override
public Mono<Long> leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return this.get(snapshot).map(prev -> 1L).defaultIfEmpty(0L);
public long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return this.get(snapshot) != null ? 1L : 0L;
}
@Override
public Mono<Boolean> isEmpty(@Nullable CompositeSnapshot snapshot) {
return this.get(snapshot).map(prev -> true).defaultIfEmpty(true);
public boolean isEmpty(@Nullable CompositeSnapshot snapshot) {
return this.get(snapshot) == null;
}
@Override
@ -127,14 +124,10 @@ public class DatabaseSingleBucket<K, V, TH> extends SimpleResource implements Da
}
@Override
public Flux<BadBlock> badBlocks() {
public Stream<BadBlock> badBlocks() {
return bucketStage.badBlocks();
}
private Mono<V> extractValueTransformation(Set<Entry<K, V>> entries) {
return Mono.fromCallable(() -> extractValue(entries));
}
@Nullable
private V extractValue(Set<Entry<K, V>> entries) {
if (entries == null) return null;
@ -188,15 +181,4 @@ public class DatabaseSingleBucket<K, V, TH> extends SimpleResource implements Da
return null;
}
}
@Override
protected void onClose() {
try {
if (bucketStage != null) {
bucketStage.close();
}
} catch (Throwable ex) {
logger.error("Failed to close bucketStage", ex);
}
}
}

107
src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleMapped.java
View File

@ -1,9 +1,5 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Drop;
import io.netty5.buffer.Owned;
import io.netty5.util.Send;
import io.netty5.buffer.internal.ResourceSupport;
import it.cavallium.dbengine.client.BadBlock;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.client.Mapper;
@ -12,126 +8,108 @@ import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.cavallium.dbengine.utils.SimpleResource;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.stream.Stream;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.publisher.SynchronousSink;
@SuppressWarnings("unused")
public class DatabaseSingleMapped<A, B> extends SimpleResource implements DatabaseStageEntry<A> {
private static final Logger logger = LogManager.getLogger(DatabaseSingleMapped.class);
public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> {
private final Mapper<A, B> mapper;
private final DatabaseStageEntry<B> serializedSingle;
@SuppressWarnings({"unchecked", "rawtypes"})
public DatabaseSingleMapped(DatabaseStageEntry<B> serializedSingle, Mapper<A, B> mapper,
Drop<DatabaseSingleMapped<A, B>> drop) {
public DatabaseSingleMapped(DatabaseStageEntry<B> serializedSingle, Mapper<A, B> mapper) {
this.serializedSingle = serializedSingle;
this.mapper = mapper;
}
@SuppressWarnings({"unchecked", "rawtypes"})
private DatabaseSingleMapped(DatabaseStage<B> serializedSingle, Mapper<A, B> mapper,
Drop<DatabaseSingleMapped<A, B>> drop) {
private DatabaseSingleMapped(DatabaseStage<B> serializedSingle, Mapper<A, B> mapper) {
this.mapper = mapper;
this.serializedSingle = (DatabaseStageEntry<B>) serializedSingle;
}
private void deserializeSink(B value, SynchronousSink<A> sink) {
try {
sink.next(this.unMap(value));
} catch (SerializationException ex) {
sink.error(ex);
}
@Override
public A get(@Nullable CompositeSnapshot snapshot) {
var data = serializedSingle.get(snapshot);
if (data == null) return null;
return this.unMap(data);
}
@Override
public Mono<A> get(@Nullable CompositeSnapshot snapshot) {
return serializedSingle.get(snapshot).handle((value, sink) -> deserializeSink(value, sink));
public A getOrDefault(@Nullable CompositeSnapshot snapshot, A defaultValue) {
var value = serializedSingle.get(snapshot);
if (value == null) return defaultValue;
return this.unMap(value);
}
@Override
public Mono<A> getOrDefault(@Nullable CompositeSnapshot snapshot, Mono<A> defaultValue) {
return serializedSingle.get(snapshot).handle((B value, SynchronousSink<A> sink) -> deserializeSink(value, sink)).switchIfEmpty(defaultValue);
public void set(A value) {
B mappedValue = value != null ? map(value) : null;
serializedSingle.set(mappedValue);
}
@Override
public Mono<Void> set(A value) {
return Mono
.fromCallable(() -> map(value))
.flatMap(value1 -> serializedSingle.set(value1));
public A setAndGetPrevious(A value) {
var mappedValue = value != null ? map(value) : null;
var prev = serializedSingle.setAndGetPrevious(mappedValue);
return prev != null ? unMap(prev) : null;
}
@Override
public Mono<A> setAndGetPrevious(A value) {
return Mono
.fromCallable(() -> map(value))
.flatMap(value2 -> serializedSingle.setAndGetPrevious(value2))
.handle((value1, sink) -> deserializeSink(value1, sink));
public boolean setAndGetChanged(A value) {
var mappedValue = value != null ? map(value) : null;
return serializedSingle.setAndGetChanged(mappedValue);
}
@Override
public Mono<Boolean> setAndGetChanged(A value) {
return Mono
.fromCallable(() -> map(value))
.flatMap(value1 -> serializedSingle.setAndGetChanged(value1))
.single();
}
@Override
public Mono<A> update(SerializationFunction<@Nullable A, @Nullable A> updater,
UpdateReturnMode updateReturnMode) {
return serializedSingle.update(oldValue -> {
public A update(SerializationFunction<@Nullable A, @Nullable A> updater, UpdateReturnMode updateReturnMode) {
B prev = serializedSingle.update(oldValue -> {
var result = updater.apply(oldValue == null ? null : this.unMap(oldValue));
if (result == null) {
return null;
} else {
return this.map(result);
}
}, updateReturnMode).handle((value, sink) -> deserializeSink(value, sink));
}, updateReturnMode);
return prev != null ? unMap(prev) : null;
}
@Override
public Mono<Delta<A>> updateAndGetDelta(SerializationFunction<@Nullable A, @Nullable A> updater) {
return serializedSingle.updateAndGetDelta(oldValue -> {
public Delta<A> updateAndGetDelta(SerializationFunction<@Nullable A, @Nullable A> updater) {
var delta = serializedSingle.updateAndGetDelta(oldValue -> {
var result = updater.apply(oldValue == null ? null : this.unMap(oldValue));
if (result == null) {
return null;
} else {
return this.map(result);
}
}).transform(mono -> LLUtils.mapDelta(mono, bytes -> unMap(bytes)));
});
return LLUtils.mapDelta(delta, this::unMap);
}
@Override
public Mono<Void> clear() {
return serializedSingle.clear();
public void clear() {
serializedSingle.clear();
}
@Override
public Mono<A> clearAndGetPrevious() {
return serializedSingle.clearAndGetPrevious().handle((value, sink) -> deserializeSink(value, sink));
public A clearAndGetPrevious() {
var prev = serializedSingle.clearAndGetPrevious();
return prev != null ? unMap(prev) : null;
}
@Override
public Mono<Boolean> clearAndGetStatus() {
public boolean clearAndGetStatus() {
return serializedSingle.clearAndGetStatus();
}
@Override
public Mono<Long> leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
public long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return serializedSingle.leavesCount(snapshot, fast);
}
@Override
public Mono<Boolean> isEmpty(@Nullable CompositeSnapshot snapshot) {
public boolean isEmpty(@Nullable CompositeSnapshot snapshot) {
return serializedSingle.isEmpty(snapshot);
}
@ -141,7 +119,7 @@ public class DatabaseSingleMapped<A, B> extends SimpleResource implements Databa
}
@Override
public Flux<BadBlock> badBlocks() {
public Stream<BadBlock> badBlocks() {
return this.serializedSingle.badBlocks();
}
@ -152,9 +130,4 @@ public class DatabaseSingleMapped<A, B> extends SimpleResource implements Databa
private B map(A bytes) throws SerializationException {
return mapper.map(bytes);
}
@Override
protected void onClose() {
serializedSingle.close();
}
}

162
src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleton.java
View File

@ -1,9 +1,8 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.Drop;
import io.netty5.buffer.Owned;
import io.netty5.buffer.internal.ResourceSupport;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.buffers.BufDataInput;
import it.cavallium.dbengine.buffers.BufDataOutput;
import it.cavallium.dbengine.client.BadBlock;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.Delta;
@ -14,23 +13,18 @@ import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.utils.InternalMonoUtils;
import it.cavallium.dbengine.utils.SimpleResource;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.publisher.SynchronousSink;
public class DatabaseSingleton<U> extends SimpleResource implements DatabaseStageEntry<U> {
public class DatabaseSingleton<U> implements DatabaseStageEntry<U> {
private static final Logger LOG = LogManager.getLogger(DatabaseSingleton.class);
private final LLSingleton singleton;
private final Serializer<U> serializer;
@SuppressWarnings({"unchecked", "rawtypes"})
public DatabaseSingleton(LLSingleton singleton, Serializer<U> serializer) {
this.singleton = singleton;
this.serializer = serializer;
@ -44,13 +38,9 @@ public class DatabaseSingleton<U> extends SimpleResource implements DatabaseStag
}
}
private U deserializeValue(Buffer value) {
private U deserializeValue(Buf value) {
try {
U deserializedValue;
try (value) {
deserializedValue = serializer.deserialize(value);
}
return deserializedValue;
return serializer.deserialize(BufDataInput.create(value));
} catch (IndexOutOfBoundsException ex) {
var exMessage = ex.getMessage();
if (exMessage != null && exMessage.contains("read 0 to 0, write 0 to ")) {
@ -63,124 +53,96 @@ public class DatabaseSingleton<U> extends SimpleResource implements DatabaseStag
}
}
private Buffer serializeValue(U value) throws SerializationException {
private Buf serializeValue(U value) throws SerializationException {
var valSizeHint = serializer.getSerializedSizeHint();
if (valSizeHint == -1) valSizeHint = 128;
var valBuf = singleton.getAllocator().allocate(valSizeHint);
try {
serializer.serialize(value, valBuf);
return valBuf;
} catch (Throwable ex) {
valBuf.close();
throw ex;
}
var valBuf = BufDataOutput.create(valSizeHint);
serializer.serialize(value, valBuf);
return valBuf.asList();
}
@Override
public Mono<U> get(@Nullable CompositeSnapshot snapshot) {
var resultMono = singleton.get(resolveSnapshot(snapshot));
return Mono.usingWhen(resultMono,
result -> Mono.fromSupplier(() -> this.deserializeValue(result)),
LLUtils::finalizeResource
);
public U get(@Nullable CompositeSnapshot snapshot) {
Buf result = singleton.get(resolveSnapshot(snapshot));
return this.deserializeValue(result);
}
@Override
public Mono<Void> set(U value) {
return singleton.set(Mono.fromCallable(() -> serializeValue(value)));
public void set(U value) {
singleton.set(serializeValue(value));
}
@Override
public Mono<U> setAndGetPrevious(U value) {
var resultMono = Flux
.concat(singleton.get(null),
singleton.set(Mono.fromCallable(() -> serializeValue(value))).as(InternalMonoUtils::toAny)
)
.last();
return Mono.usingWhen(resultMono,
result -> Mono.fromSupplier(() -> this.deserializeValue(result)),
LLUtils::finalizeResource
);
public U setAndGetPrevious(U value) {
var prev = singleton.get(null);
singleton.set(serializeValue(value));
return this.deserializeValue(prev);
}
@Override
public Mono<U> update(SerializationFunction<@Nullable U, @Nullable U> updater,
public U update(SerializationFunction<@Nullable U, @Nullable U> updater,
UpdateReturnMode updateReturnMode) {
var resultMono = singleton
Buf resultBuf = singleton
.update((oldValueSer) -> {
try (oldValueSer) {
U result;
if (oldValueSer == null) {
result = updater.apply(null);
} else {
U deserializedValue = serializer.deserialize(oldValueSer);
result = updater.apply(deserializedValue);
}
if (result == null) {
return null;
} else {
return serializeValue(result);
}
U result;
if (oldValueSer == null) {
result = updater.apply(null);
} else {
U deserializedValue = serializer.deserialize(BufDataInput.create(oldValueSer));
result = updater.apply(deserializedValue);
}
if (result == null) {
return null;
} else {
return serializeValue(result);
}
}, updateReturnMode);
return Mono.usingWhen(resultMono,
result -> Mono.fromSupplier(() -> this.deserializeValue(result)),
LLUtils::finalizeResource
);
return this.deserializeValue(resultBuf);
}
@Override
public Mono<Delta<U>> updateAndGetDelta(SerializationFunction<@Nullable U, @Nullable U> updater) {
return singleton
.updateAndGetDelta((oldValueSer) -> {
try (oldValueSer) {
U result;
if (oldValueSer == null) {
result = updater.apply(null);
} else {
U deserializedValue = serializer.deserialize(oldValueSer);
result = updater.apply(deserializedValue);
}
if (result == null) {
return null;
} else {
return serializeValue(result);
}
}
}).transform(mono -> LLUtils.mapLLDelta(mono, serialized -> serializer.deserialize(serialized)));
public Delta<U> updateAndGetDelta(SerializationFunction<@Nullable U, @Nullable U> updater) {
var mono = singleton.updateAndGetDelta((oldValueSer) -> {
U result;
if (oldValueSer == null) {
result = updater.apply(null);
} else {
U deserializedValue = serializer.deserialize(BufDataInput.create(oldValueSer));
result = updater.apply(deserializedValue);
}
if (result == null) {
return null;
} else {
return serializeValue(result);
}
});
return LLUtils.mapLLDelta(mono, serialized -> serializer.deserialize(BufDataInput.create(serialized)));
}
@Override
public Mono<Void> clear() {
return singleton.set(Mono.empty());
public void clear() {
singleton.set(null);
}
@Override
public Mono<U> clearAndGetPrevious() {
var resultMono = Flux.concat(singleton.get(null), singleton.set(Mono.empty()).as(InternalMonoUtils::toAny)).last();
return Mono.usingWhen(resultMono,
result -> Mono.fromSupplier(() -> this.deserializeValue(result)),
LLUtils::finalizeResource
);
public U clearAndGetPrevious() {
var result = singleton.get(null);
singleton.set(null);
return this.deserializeValue(result);
}
@Override
public Mono<Long> leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return singleton.get(null).map(unused -> 1L).defaultIfEmpty(0L);
public long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return singleton.get(null) != null ? 1L : 0L;
}
@Override
public Mono<Boolean> isEmpty(@Nullable CompositeSnapshot snapshot) {
return singleton.get(null).map(t -> false).defaultIfEmpty(true);
public boolean isEmpty(@Nullable CompositeSnapshot snapshot) {
return singleton.get(null) == null;
}
@Override
public Flux<BadBlock> badBlocks() {
return Flux.empty();
}
@Override
protected void onClose() {
public Stream<BadBlock> badBlocks() {
return Stream.empty();
}
}

64
src/main/java/it/cavallium/dbengine/database/collections/DatabaseStage.java
View File

@ -1,64 +1,56 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.util.Resource;
import it.cavallium.dbengine.client.BadBlock;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.Delta;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.SafeCloseable;
import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import java.util.Objects;
import java.util.stream.Stream;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public interface DatabaseStage<T> extends DatabaseStageWithEntry<T>, SafeCloseable {
public interface DatabaseStage<T> extends DatabaseStageWithEntry<T> {
Mono<T> get(@Nullable CompositeSnapshot snapshot);
@Nullable T get(@Nullable CompositeSnapshot snapshot);
default Mono<T> getOrDefault(@Nullable CompositeSnapshot snapshot,
Mono<T> defaultValue,
boolean existsAlmostCertainly) {
return get(snapshot).switchIfEmpty(defaultValue).single();
default T getOrDefault(@Nullable CompositeSnapshot snapshot, T defaultValue, boolean existsAlmostCertainly) {
return Objects.requireNonNullElse(get(snapshot), defaultValue);
}
default Mono<T> getOrDefault(@Nullable CompositeSnapshot snapshot, Mono<T> defaultValue) {
default T getOrDefault(@Nullable CompositeSnapshot snapshot, T defaultValue) {
return getOrDefault(snapshot, defaultValue, false);
}
default Mono<Void> set(T value) {
return this
.setAndGetChanged(value)
.then();
default void set(@Nullable T value) {
this.setAndGetChanged(value);
}
Mono<T> setAndGetPrevious(T value);
@Nullable T setAndGetPrevious(@Nullable T value);
default Mono<Boolean> setAndGetChanged(T value) {
return this
.setAndGetPrevious(value)
.map(oldValue -> !Objects.equals(oldValue, value))
.switchIfEmpty(Mono.fromSupplier(() -> value != null));
default boolean setAndGetChanged(@Nullable T value) {
T oldValue = this.setAndGetPrevious(value);
if (oldValue != null) {
return !Objects.equals(oldValue, value);
} else {
return value != null;
}
}
default Mono<T> update(SerializationFunction<@Nullable T, @Nullable T> updater,
UpdateReturnMode updateReturnMode) {
return this
.updateAndGetDelta(updater)
.transform(prev -> LLUtils.resolveDelta(prev, updateReturnMode));
default @Nullable T update(SerializationFunction<@Nullable T, @Nullable T> updater, UpdateReturnMode updateReturnMode) {
return LLUtils.resolveDelta(this.updateAndGetDelta(updater), updateReturnMode);
}
Mono<Delta<T>> updateAndGetDelta(SerializationFunction<@Nullable T, @Nullable T> updater);
Delta<T> updateAndGetDelta(SerializationFunction<@Nullable T, @Nullable T> updater);
default Mono<Void> clear() {
return clearAndGetStatus().then();
default void clear() {
clearAndGetStatus();
}
Mono<T> clearAndGetPrevious();
@Nullable T clearAndGetPrevious();
default Mono<Boolean> clearAndGetStatus() {
return clearAndGetPrevious().map(Objects::nonNull).defaultIfEmpty(false);
default boolean clearAndGetStatus() {
return clearAndGetPrevious() != null;
}
/**
@ -66,11 +58,11 @@ public interface DatabaseStage<T> extends DatabaseStageWithEntry<T>, SafeCloseab
* If it's a nested collection the count will include all the children recursively
* @param fast true to return an approximate value
*/
Mono<Long> leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast);
long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast);
default Mono<Boolean> isEmpty(@Nullable CompositeSnapshot snapshot) {
return leavesCount(snapshot, false).map(size -> size <= 0);
default boolean isEmpty(@Nullable CompositeSnapshot snapshot) {
return leavesCount(snapshot, false) <= 0;
}
Flux<BadBlock> badBlocks();
Stream<BadBlock> badBlocks();
}

4
src/main/java/it/cavallium/dbengine/database/collections/DatabaseStageEntry.java
View File

@ -1,9 +1,5 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.util.Resource;
import it.cavallium.dbengine.client.BadBlock;
import reactor.core.publisher.Flux;
public interface DatabaseStageEntry<U> extends DatabaseStage<U> {
@Override

271
src/main/java/it/cavallium/dbengine/database/collections/DatabaseStageMap.java
View File

@ -1,5 +1,7 @@
package it.cavallium.dbengine.database.collections;
import static it.cavallium.dbengine.database.LLUtils.consume;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.Delta;
import it.cavallium.dbengine.database.LLUtils;
@ -7,7 +9,6 @@ import it.cavallium.dbengine.database.SubStageEntry;
import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.KVSerializationFunction;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
@ -16,261 +17,211 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuples;
@SuppressWarnings("unused")
public interface DatabaseStageMap<T, U, US extends DatabaseStage<U>> extends
DatabaseStageEntry<Object2ObjectSortedMap<T, U>> {
public interface DatabaseStageMap<T, U, US extends DatabaseStage<U>> extends DatabaseStageEntry<Object2ObjectSortedMap<T, U>> {
Mono<US> at(@Nullable CompositeSnapshot snapshot, T key);
@NotNull US at(@Nullable CompositeSnapshot snapshot, T key);
default Mono<Boolean> containsKey(@Nullable CompositeSnapshot snapshot, T key) {
return Mono.usingWhen(this.at(snapshot, key),
stage -> stage.isEmpty(snapshot).map(empty -> !empty),
LLUtils::finalizeResource
);
default boolean containsKey(@Nullable CompositeSnapshot snapshot, T key) {
return !this.at(snapshot, key).isEmpty(snapshot);
}
default Mono<U> getValue(@Nullable CompositeSnapshot snapshot, T key) {
return Mono.usingWhen(this.at(snapshot, key),
stage -> stage.get(snapshot),
LLUtils::finalizeResource
);
default @Nullable U getValue(@Nullable CompositeSnapshot snapshot, T key) {
return this.at(snapshot, key).get(snapshot);
}
default Mono<U> getValueOrDefault(@Nullable CompositeSnapshot snapshot, T key, Mono<U> defaultValue) {
return getValue(snapshot, key).switchIfEmpty(defaultValue).single();
default U getValueOrDefault(@Nullable CompositeSnapshot snapshot, T key, U defaultValue) {
return Objects.requireNonNullElse(getValue(snapshot, key), defaultValue);
}
default Mono<Void> putValue(T key, U value) {
return Mono.usingWhen(at(null, key).single(), stage -> stage.set(value), LLUtils::finalizeResource);
default U getValueOrDefault(@Nullable CompositeSnapshot snapshot, T key, Supplier<U> defaultValue) {
return Objects.requireNonNullElseGet(getValue(snapshot, key), defaultValue);
}
default void putValue(T key, U value) {
at(null, key).set(value);
}
UpdateMode getUpdateMode();
default Mono<U> updateValue(T key,
default U updateValue(T key,
UpdateReturnMode updateReturnMode,
SerializationFunction<@Nullable U, @Nullable U> updater) {
return Mono.usingWhen(at(null, key).single(),
stage -> stage.update(updater, updateReturnMode),
LLUtils::finalizeResource
);
return at(null, key).update(updater, updateReturnMode);
}
default Flux<Boolean> updateMulti(Flux<T> keys, KVSerializationFunction<T, @Nullable U, @Nullable U> updater) {
return keys.flatMapSequential(key -> this.updateValue(key, prevValue -> updater.apply(key, prevValue)));
default Stream<Boolean> updateMulti(Stream<T> keys, KVSerializationFunction<T, @Nullable U, @Nullable U> updater) {
return keys.parallel().map(key -> this.updateValue(key, prevValue -> updater.apply(key, prevValue)));
}
default Mono<Boolean> updateValue(T key, SerializationFunction<@Nullable U, @Nullable U> updater) {
return updateValueAndGetDelta(key, updater).map(delta -> LLUtils.isDeltaChanged(delta)).single();
default boolean updateValue(T key, SerializationFunction<@Nullable U, @Nullable U> updater) {
return LLUtils.isDeltaChanged(updateValueAndGetDelta(key, updater));
}
default Mono<Delta<U>> updateValueAndGetDelta(T key,
SerializationFunction<@Nullable U, @Nullable U> updater) {
var stageMono = this.at(null, key).single();
return stageMono.flatMap(stage -> stage
.updateAndGetDelta(updater)
.doFinally(s -> stage.close()));
default Delta<U> updateValueAndGetDelta(T key, SerializationFunction<@Nullable U, @Nullable U> updater) {
return this.at(null, key).updateAndGetDelta(updater);
}
default Mono<U> putValueAndGetPrevious(T key, U value) {
return Mono.usingWhen(at(null, key).single(),
stage -> stage.setAndGetPrevious(value),
LLUtils::finalizeResource
);
default @Nullable U putValueAndGetPrevious(T key, @Nullable U value) {
return at(null, key).setAndGetPrevious(value);
}
/**
* @return true if the key was associated with any value, false if the key didn't exist.
*/
default Mono<Boolean> putValueAndGetChanged(T key, U value) {
return Mono
.usingWhen(at(null, key).single(), stage -> stage.setAndGetChanged(value), LLUtils::finalizeResource)
.single();
default boolean putValueAndGetChanged(T key, @Nullable U value) {
return at(null, key).setAndGetChanged(value);
}
default Mono<Void> remove(T key) {
return removeAndGetStatus(key).then();
default void remove(T key) {
removeAndGetStatus(key);
}
default Mono<U> removeAndGetPrevious(T key) {
return Mono.usingWhen(at(null, key), us -> us.clearAndGetPrevious(), LLUtils::finalizeResource);
default @Nullable U removeAndGetPrevious(T key) {
return at(null, key).clearAndGetPrevious();
}
default Mono<Boolean> removeAndGetStatus(T key) {
return removeAndGetPrevious(key).map(o -> true).defaultIfEmpty(false);
default boolean removeAndGetStatus(T key) {
return removeAndGetPrevious(key) != null;
}
/**
* GetMulti must return the elements in sequence!
*/
default Flux<Optional<U>> getMulti(@Nullable CompositeSnapshot snapshot, Flux<T> keys) {
return keys.flatMapSequential(key -> this
.getValue(snapshot, key)
.map(Optional::of)
.defaultIfEmpty(Optional.empty())
);
default Stream<Optional<U>> getMulti(@Nullable CompositeSnapshot snapshot, Stream<T> keys) {
return keys.parallel().map(key -> Optional.ofNullable(this.getValue(snapshot, key)));
}
default Mono<Void> putMulti(Flux<Entry<T, U>> entries) {
return entries.flatMap(entry -> this.putValue(entry.getKey(), entry.getValue())).then();
default void putMulti(Stream<Entry<T, U>> entries) {
try (var stream = entries.parallel()) {
stream.forEach(entry -> this.putValue(entry.getKey(), entry.getValue()));
}
}
Flux<SubStageEntry<T, US>> getAllStages(@Nullable CompositeSnapshot snapshot, boolean smallRange);
Stream<SubStageEntry<T, US>> getAllStages(@Nullable CompositeSnapshot snapshot, boolean smallRange);
default Flux<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
return this
.getAllStages(snapshot, smallRange)
.flatMapSequential(stage -> stage
.getValue()
.get(snapshot)
.map(value -> Map.entry(stage.getKey(), value))
.doFinally(s -> stage.getValue().close())
);
default Stream<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
return this.getAllStages(snapshot, smallRange).parallel().mapMulti((stage, mapper) -> {
var val = stage.getValue().get(snapshot);
if (val != null) {
mapper.accept(Map.entry(stage.getKey(), val));
}
});
}
default Mono<Void> setAllValues(Flux<Entry<T, U>> entries) {
return setAllValuesAndGetPrevious(entries).then();
default void setAllValues(Stream<Entry<T, U>> entries) {
consume(setAllValuesAndGetPrevious(entries));
}
Flux<Entry<T, U>> setAllValuesAndGetPrevious(Flux<Entry<T, U>> entries);
Stream<Entry<T, U>> setAllValuesAndGetPrevious(Stream<Entry<T, U>> entries);
default Mono<Void> clear() {
return setAllValues(Flux.empty());
default void clear() {
setAllValues(Stream.empty());
}
default Mono<Void> replaceAllValues(boolean canKeysChange,
Function<Entry<T, U>, Mono<Entry<T, U>>> entriesReplacer,
default void replaceAllValues(boolean canKeysChange,
Function<Entry<T, U>, @NotNull Entry<T, U>> entriesReplacer,
boolean smallRange) {
if (canKeysChange) {
return this.setAllValues(this.getAllValues(null, smallRange).flatMap(entriesReplacer)).then();
this.setAllValues(this.getAllValues(null, smallRange).map(entriesReplacer));
} else {
return this
.getAllValues(null, smallRange)
.flatMap(entriesReplacer)
.flatMap(replacedEntry -> this
.at(null, replacedEntry.getKey())
.flatMap(stage -> stage
.set(replacedEntry.getValue())
.doFinally(s -> stage.close())
)
)
.then();
this.getAllValues(null, smallRange).map(entriesReplacer)
.forEach(replacedEntry -> this.at(null, replacedEntry.getKey()).set(replacedEntry.getValue()));
}
}
default Mono<Void> replaceAll(Function<Entry<T, US>, Mono<Void>> entriesReplacer) {
return this
.getAllStages(null, false)
.flatMap(stage -> entriesReplacer.apply(stage)
.doFinally(s -> stage.getValue().close())
)
.then();
default void replaceAll(Consumer<Entry<T, US>> entriesReplacer) {
this.getAllStages(null, false).forEach(entriesReplacer);
}
@Override
default Mono<Object2ObjectSortedMap<T, U>> setAndGetPrevious(Object2ObjectSortedMap<T, U> value) {
return this
.setAllValuesAndGetPrevious(Flux.fromIterable(value.entrySet()))
.collectMap(Entry::getKey, Entry::getValue, Object2ObjectLinkedOpenHashMap::new)
.map(map -> (Object2ObjectSortedMap<T, U>) map)
.filter(map -> !map.isEmpty());
default Object2ObjectSortedMap<T, U> setAndGetPrevious(Object2ObjectSortedMap<T, U> value) {
Object2ObjectSortedMap<T, U> map;
if (value == null) {
map = this.clearAndGetPrevious();
} else {
map = this
.setAllValuesAndGetPrevious(value.entrySet().stream())
.collect(Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) -> a, Object2ObjectLinkedOpenHashMap::new));
}
return map;
}
@Override
default Mono<Boolean> setAndGetChanged(Object2ObjectSortedMap<T, U> value) {
return this
.setAndGetPrevious(value)
.map(oldValue -> !Objects.equals(oldValue, value.isEmpty() ? null : value))
.switchIfEmpty(Mono.fromSupplier(() -> !value.isEmpty()));
default boolean setAndGetChanged(@Nullable Object2ObjectSortedMap<T, U> value) {
if (value != null && value.isEmpty()) {
value = null;
}
var prev = this.setAndGetPrevious(value);
if (prev == null) {
return value != null;
} else {
return !Objects.equals(prev, value);
}
}
@Override
default Mono<Delta<Object2ObjectSortedMap<T, U>>> updateAndGetDelta(SerializationFunction<@Nullable Object2ObjectSortedMap<T, U>, @Nullable Object2ObjectSortedMap<T, U>> updater) {
default Delta<Object2ObjectSortedMap<T, U>> updateAndGetDelta(
SerializationFunction<@Nullable Object2ObjectSortedMap<T, U>, @Nullable Object2ObjectSortedMap<T, U>> updater) {
var updateMode = this.getUpdateMode();
if (updateMode == UpdateMode.ALLOW_UNSAFE) {
return this
Object2ObjectSortedMap<T, U> v = this
.getAllValues(null, true)
.collectMap(Entry::getKey, Entry::getValue, Object2ObjectLinkedOpenHashMap::new)
.map(map -> (Object2ObjectSortedMap<T, U>) map)
.single()
.<Tuple2<Optional<Object2ObjectSortedMap<T, U>>, Optional<Object2ObjectSortedMap<T, U>>>>handle((v, sink) -> {
if (v.isEmpty()) {
v = null;
}
try {
var result = updater.apply(v);
if (result != null && result.isEmpty()) {
result = null;
}
sink.next(Tuples.of(Optional.ofNullable(v), Optional.ofNullable(result)));
} catch (SerializationException ex) {
sink.error(ex);
}
})
.flatMap(result -> Mono
.justOrEmpty(result.getT2())
.flatMap(values -> this.setAllValues(Flux.fromIterable(values.entrySet())))
.thenReturn(new Delta<>(result.getT1().orElse(null), result.getT2().orElse(null)))
);
.collect(Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) -> a, Object2ObjectLinkedOpenHashMap::new));
if (v.isEmpty()) {
v = null;
}
var result = updater.apply(v);
if (result != null && result.isEmpty()) {
result = null;
}
this.setAllValues(result != null ? result.entrySet().stream() : null);
return new Delta<>(v, result);
} else if (updateMode == UpdateMode.ALLOW) {
return Mono.fromCallable(() -> {
throw new UnsupportedOperationException("Maps can't be updated atomically");
});
throw new UnsupportedOperationException("Maps can't be updated atomically");
} else if (updateMode == UpdateMode.DISALLOW) {
return Mono.fromCallable(() -> {
throw new UnsupportedOperationException("Map can't be updated because updates are disabled");
});
throw new UnsupportedOperationException("Map can't be updated because updates are disabled");
} else {
return Mono.fromCallable(() -> {
throw new UnsupportedOperationException("Unknown update mode: " + updateMode);
});
throw new UnsupportedOperationException("Unknown update mode: " + updateMode);
}
}
@Override
default Mono<Object2ObjectSortedMap<T, U>> clearAndGetPrevious() {
default Object2ObjectSortedMap<T, U> clearAndGetPrevious() {
return this.setAndGetPrevious(Object2ObjectSortedMaps.emptyMap());
}
@Override
default Mono<Object2ObjectSortedMap<T, U>> get(@Nullable CompositeSnapshot snapshot) {
return this
default Object2ObjectSortedMap<T, U> get(@Nullable CompositeSnapshot snapshot) {
Object2ObjectSortedMap<T, U> map = this
.getAllValues(snapshot, true)
.collectMap(Entry::getKey, Entry::getValue, Object2ObjectLinkedOpenHashMap::new)
.map(map -> (Object2ObjectSortedMap<T, U>) map)
.filter(map -> !map.isEmpty());
.collect(Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) -> a, Object2ObjectLinkedOpenHashMap::new));
return map.isEmpty() ? null : map;
}
@Override
default Mono<Long> leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return this
.getAllStages(snapshot, false)
.doOnNext(stage -> stage.getValue().close())
.count();
default long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
return this.getAllStages(snapshot, false).count();
}
/**
* Value getter doesn't lock data. Please make sure to lock before getting data.
*/
default ValueGetterBlocking<T, U> getDbValueGetter(@Nullable CompositeSnapshot snapshot) {
return k -> getValue(snapshot, k).transform(LLUtils::handleDiscard).block();
return k -> getValue(snapshot, k);
}
default ValueGetter<T, U> getAsyncDbValueGetter(@Nullable CompositeSnapshot snapshot) {
return k -> getValue(snapshot, k);
}
default ValueTransformer<T, U> getAsyncDbValueTransformer(@Nullable CompositeSnapshot snapshot) {
return keys -> {
var sharedKeys = keys.publish().refCount(2);
var values = DatabaseStageMap.this.getMulti(snapshot, sharedKeys);
return Flux.zip(sharedKeys, values, Map::entry);
};
}
}

4
src/main/java/it/cavallium/dbengine/database/collections/DatabaseStageWithEntry.java
View File

@ -1,9 +1,5 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.util.Resource;
import it.cavallium.dbengine.client.BadBlock;
import reactor.core.publisher.Mono;
public interface DatabaseStageWithEntry<T> {
DatabaseStageEntry<T> entry();

8
src/main/java/it/cavallium/dbengine/database/collections/SubStageGetter.java
View File

@ -1,16 +1,12 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.LLDictionary;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
public interface SubStageGetter<U, US extends DatabaseStage<U>> {
Mono<US> subStage(LLDictionary dictionary,
@Nullable CompositeSnapshot snapshot,
Mono<Buffer> prefixKey);
US subStage(LLDictionary dictionary, @Nullable CompositeSnapshot snapshot, Buf prefixKey);
}

18
src/main/java/it/cavallium/dbengine/database/collections/SubStageGetterHashMap.java
View File

@ -1,19 +1,13 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.util.Resource;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.BufSupplier;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import java.util.Map;
import java.util.function.Function;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
@SuppressWarnings("unused")
public class SubStageGetterHashMap<T, U, TH> implements
@ -35,16 +29,16 @@ public class SubStageGetterHashMap<T, U, TH> implements
}
@Override
public Mono<DatabaseMapDictionaryHashed<T, U, TH>> subStage(LLDictionary dictionary,
public DatabaseMapDictionaryHashed<T, U, TH> subStage(LLDictionary dictionary,
@Nullable CompositeSnapshot snapshot,
Mono<Buffer> prefixKeyMono) {
return prefixKeyMono.map(prefixKey -> DatabaseMapDictionaryHashed.tail(dictionary,
BufSupplier.ofOwned(prefixKey),
Buf prefixKey) {
return DatabaseMapDictionaryHashed.tail(dictionary,
prefixKey,
keySerializer,
valueSerializer,
keyHashFunction,
keyHashSerializer
));
);
}
public int getKeyHashBinaryLength() {

20
src/main/java/it/cavallium/dbengine/database/collections/SubStageGetterHashSet.java
View File

@ -1,22 +1,16 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.util.Resource;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.BufSupplier;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.collections.DatabaseEmpty.Nothing;
import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import java.util.Map;
import java.util.function.Function;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
@SuppressWarnings({"unused", "ClassCanBeRecord"})
@SuppressWarnings({"unused"})
public class SubStageGetterHashSet<T, TH> implements
SubStageGetter<Object2ObjectSortedMap<T, Nothing>, DatabaseSetDictionaryHashed<T, TH>> {
@ -33,15 +27,15 @@ public class SubStageGetterHashSet<T, TH> implements
}
@Override
public Mono<DatabaseSetDictionaryHashed<T, TH>> subStage(LLDictionary dictionary,
public DatabaseSetDictionaryHashed<T, TH> subStage(LLDictionary dictionary,
@Nullable CompositeSnapshot snapshot,
Mono<Buffer> prefixKeyMono) {
return prefixKeyMono.map(prefixKey -> DatabaseSetDictionaryHashed.tail(dictionary,
BufSupplier.ofOwned(prefixKey),
Buf prefixKey) {
return DatabaseSetDictionaryHashed.tail(dictionary,
prefixKey,
keySerializer,
keyHashFunction,
keyHashSerializer
));
);
}
public int getKeyHashBinaryLength() {

14
src/main/java/it/cavallium/dbengine/database/collections/SubStageGetterMap.java
View File

@ -1,14 +1,12 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.BufSupplier;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
public class SubStageGetterMap<T, U> implements
SubStageGetter<Object2ObjectSortedMap<T, U>, DatabaseMapDictionary<T, U>> {
@ -23,14 +21,10 @@ public class SubStageGetterMap<T, U> implements
}
@Override
public Mono<DatabaseMapDictionary<T, U>> subStage(LLDictionary dictionary,
public DatabaseMapDictionary<T, U> subStage(LLDictionary dictionary,
@Nullable CompositeSnapshot snapshot,
Mono<Buffer> prefixKeyMono) {
return prefixKeyMono.map(prefixKey -> DatabaseMapDictionary.tail(dictionary,
BufSupplier.ofOwned(prefixKey),
keySerializer,
valueSerializer
));
Buf prefixKey) {
return DatabaseMapDictionary.tail(dictionary, prefixKey, keySerializer, valueSerializer);
}
public int getKeyBinaryLength() {

18
src/main/java/it/cavallium/dbengine/database/collections/SubStageGetterMapDeep.java
View File

@ -1,17 +1,11 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.util.Resource;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.BufSupplier;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import java.util.Map;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
public class SubStageGetterMapDeep<T, U, US extends DatabaseStage<U>> implements
SubStageGetter<Object2ObjectSortedMap<T, U>, DatabaseMapDictionaryDeep<T, U, US>> {
@ -41,15 +35,15 @@ public class SubStageGetterMapDeep<T, U, US extends DatabaseStage<U>> implements
}
@Override
public Mono<DatabaseMapDictionaryDeep<T, U, US>> subStage(LLDictionary dictionary,
public DatabaseMapDictionaryDeep<T, U, US> subStage(LLDictionary dictionary,
@Nullable CompositeSnapshot snapshot,
Mono<Buffer> prefixKeyMono) {
return prefixKeyMono.map(prefixKey -> DatabaseMapDictionaryDeep.deepIntermediate(dictionary,
BufSupplier.ofOwned(prefixKey),
Buf prefixKey) {
return DatabaseMapDictionaryDeep.deepIntermediate(dictionary,
prefixKey,
keySerializer,
subStageGetter,
keyExtLength
));
);
}
public int getKeyBinaryLength() {

16
src/main/java/it/cavallium/dbengine/database/collections/SubStageGetterSet.java
View File

@ -1,17 +1,12 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.util.Resource;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.BufSupplier;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.collections.DatabaseEmpty.Nothing;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import java.util.Map;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
public class SubStageGetterSet<T> implements
SubStageGetter<Object2ObjectSortedMap<T, Nothing>, DatabaseSetDictionary<T>> {
@ -23,13 +18,10 @@ public class SubStageGetterSet<T> implements
}
@Override
public Mono<DatabaseSetDictionary<T>> subStage(LLDictionary dictionary,
public DatabaseSetDictionary<T> subStage(LLDictionary dictionary,
@Nullable CompositeSnapshot snapshot,
Mono<Buffer> prefixKeyMono) {
return prefixKeyMono.map(prefixKey -> DatabaseSetDictionary.tail(dictionary,
BufSupplier.ofOwned(prefixKey),
keySerializer
));
Buf prefixKey) {
return DatabaseSetDictionary.tail(dictionary, prefixKey, keySerializer);
}
public int getKeyBinaryLength() {

14
src/main/java/it/cavallium/dbengine/database/collections/SubStageGetterSingle.java
View File

@ -1,13 +1,10 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.BufSupplier;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.serialization.Serializer;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
public class SubStageGetterSingle<T> implements SubStageGetter<T, DatabaseStageEntry<T>> {
@ -18,13 +15,10 @@ public class SubStageGetterSingle<T> implements SubStageGetter<T, DatabaseStageE
}
@Override
public Mono<DatabaseStageEntry<T>> subStage(LLDictionary dictionary,
public DatabaseStageEntry<T> subStage(LLDictionary dictionary,
@Nullable CompositeSnapshot snapshot,
Mono<Buffer> keyPrefixMono) {
return keyPrefixMono.map(keyPrefix -> new DatabaseMapSingle<>(dictionary,
BufSupplier.ofOwned(keyPrefix),
serializer
));
Buf keyPrefix) {
return new DatabaseMapSingle<>(dictionary, keyPrefix, serializer);
}
}

7
src/main/java/it/cavallium/dbengine/database/collections/SubStageGetterSingleBytes.java
View File

@ -1,12 +1,11 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.serialization.Serializer;
public class SubStageGetterSingleBytes extends SubStageGetterSingle<Send<Buffer>> {
public class SubStageGetterSingleBytes extends SubStageGetterSingle<Buf> {
public SubStageGetterSingleBytes() {
super(Serializer.NOOP_SEND_SERIALIZER);
super(Serializer.NOOP_SERIALIZER);
}
}

4
src/main/java/it/cavallium/dbengine/database/collections/ValueGetter.java
View File

@ -1,11 +1,11 @@
package it.cavallium.dbengine.database.collections;
import reactor.core.publisher.Mono;
import org.jetbrains.annotations.Nullable;
public interface ValueGetter<KEY, VALUE> {
/**
* Can return Mono error IOException
*/
Mono<VALUE> get(KEY key);
@Nullable VALUE get(KEY key);
}

2
src/main/java/it/cavallium/dbengine/database/collections/ValueGetterBlocking.java
View File

@ -4,5 +4,5 @@ import java.io.IOException;
public interface ValueGetterBlocking<KEY, VALUE> {
VALUE get(KEY key) throws IOException;
VALUE get(KEY key);
}

16
src/main/java/it/cavallium/dbengine/database/collections/ValueTransformer.java
View File

@ -1,16 +0,0 @@
package it.cavallium.dbengine.database.collections;
import java.util.Map.Entry;
import java.util.Optional;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuple3;
public interface ValueTransformer<KEY, VALUE> {
/**
* Can return Flux error IOException
*/
Flux<Entry<KEY, Optional<VALUE>>> transform(Flux<KEY> keys);
}

22
src/main/java/it/cavallium/dbengine/database/collections/ValueWithHashSerializer.java
View File

@ -1,17 +1,13 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.BufferAllocator;
import io.netty5.buffer.CompositeBuffer;
import io.netty5.util.Send;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.buffers.BufDataInput;
import it.cavallium.dbengine.buffers.BufDataOutput;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.Serializer;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
class ValueWithHashSerializer<X, Y> implements Serializer<Entry<X, Y>> {
@ -26,17 +22,17 @@ class ValueWithHashSerializer<X, Y> implements Serializer<Entry<X, Y>> {
}
@Override
public @NotNull Entry<X, Y> deserialize(@NotNull Buffer serialized) throws SerializationException {
Objects.requireNonNull(serialized);
X deserializedKey = keySuffixSerializer.deserialize(serialized);
Y deserializedValue = valueSerializer.deserialize(serialized);
public @NotNull Entry<X, Y> deserialize(@NotNull BufDataInput in) throws SerializationException {
Objects.requireNonNull(in);
X deserializedKey = keySuffixSerializer.deserialize(in);
Y deserializedValue = valueSerializer.deserialize(in);
return Map.entry(deserializedKey, deserializedValue);
}
@Override
public void serialize(@NotNull Entry<X, Y> deserialized, Buffer output) throws SerializationException {
keySuffixSerializer.serialize(deserialized.getKey(), output);
valueSerializer.serialize(deserialized.getValue(), output);
public void serialize(@NotNull Entry<X, Y> deserialized, BufDataOutput out) throws SerializationException {
keySuffixSerializer.serialize(deserialized.getKey(), out);
valueSerializer.serialize(deserialized.getValue(), out);
}
@Override

22
src/main/java/it/cavallium/dbengine/database/collections/ValuesSetSerializer.java
View File

@ -1,8 +1,7 @@
package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.BufferAllocator;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.BufDataInput;
import it.cavallium.dbengine.buffers.BufDataOutput;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.Serializer;
import it.unimi.dsi.fastutil.objects.ObjectArraySet;
@ -11,7 +10,6 @@ import java.util.Objects;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
class ValuesSetSerializer<X> implements Serializer<ObjectArraySet<X>> {
@ -24,17 +22,17 @@ class ValuesSetSerializer<X> implements Serializer<ObjectArraySet<X>> {
}
@Override
public @NotNull ObjectArraySet<X> deserialize(@NotNull Buffer serialized) throws SerializationException {
public @NotNull ObjectArraySet<X> deserialize(@NotNull BufDataInput in) throws SerializationException {
try {
Objects.requireNonNull(serialized);
if (serialized.readableBytes() == 0) {
Objects.requireNonNull(in);
if (in.available() <= 0) {
logger.error("Can't deserialize, 0 bytes are readable");
return new ObjectArraySet<>();
}
int entriesLength = serialized.readInt();
int entriesLength = in.readInt();
ArrayList<X> deserializedElements = new ArrayList<>(entriesLength);
for (int i = 0; i < entriesLength; i++) {
var deserializationResult = entrySerializer.deserialize(serialized);
var deserializationResult = entrySerializer.deserialize(in);
deserializedElements.add(deserializationResult);
}
return new ObjectArraySet<>(deserializedElements);
@ -45,10 +43,10 @@ class ValuesSetSerializer<X> implements Serializer<ObjectArraySet<X>> {
}
@Override
public void serialize(@NotNull ObjectArraySet<X> deserialized, Buffer output) throws SerializationException {
output.writeInt(deserialized.size());
public void serialize(@NotNull ObjectArraySet<X> deserialized, BufDataOutput out) throws SerializationException {
out.writeInt(deserialized.size());
for (X entry : deserialized) {
entrySerializer.serialize(entry, output);
entrySerializer.serialize(entry, out);
}
}

308
src/main/java/it/cavallium/dbengine/database/disk/AbstractRocksDBColumn.java
View File

@ -1,27 +1,16 @@
package it.cavallium.dbengine.database.disk;
import static io.netty5.buffer.StandardAllocationTypes.OFF_HEAP;
import static it.cavallium.dbengine.database.LLUtils.INITIAL_DIRECT_READ_BYTE_BUF_SIZE_BYTES;
import static it.cavallium.dbengine.database.LLUtils.isReadOnlyDirect;
import static java.lang.Boolean.parseBoolean;
import static java.lang.System.getProperty;
import static java.util.Objects.requireNonNull;
import static org.rocksdb.KeyMayExist.KeyMayExistEnum.kExistsWithValue;
import static org.rocksdb.KeyMayExist.KeyMayExistEnum.kExistsWithoutValue;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.BufferAllocator;
import io.netty5.buffer.BufferComponent;
import io.netty5.buffer.DefaultBufferAllocators;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.LLRange;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.RepeatedElementList;
import it.cavallium.dbengine.database.disk.rocksdb.RocksIteratorObj;
import it.cavallium.dbengine.database.serialization.SerializationException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@ -37,12 +26,9 @@ import org.rocksdb.AbstractImmutableNativeReference;
import org.rocksdb.AbstractSlice;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.CompactRangeOptions;
import org.rocksdb.DirectSlice;
import org.rocksdb.FlushOptions;
import org.rocksdb.Holder;
import org.rocksdb.KeyMayExist;
import org.rocksdb.KeyMayExist.KeyMayExistEnum;
import org.rocksdb.Range;
import org.rocksdb.ReadOptions;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
@ -53,23 +39,15 @@ import org.rocksdb.Transaction;
import org.rocksdb.TransactionOptions;
import org.rocksdb.WriteBatch;
import org.rocksdb.WriteOptions;
import reactor.core.scheduler.Schedulers;
public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements RocksDBColumn
permits StandardRocksDBColumn, OptimisticRocksDBColumn, PessimisticRocksDBColumn {
/**
* Default: true
*/
private static final boolean USE_DIRECT_BUFFER_BOUNDS = true;
private static final byte[] NO_DATA = new byte[0];
protected static final UpdateAtomicResult RESULT_NOTHING = new UpdateAtomicResultNothing();
protected final Logger logger = LogManager.getLogger(this.getClass());
private final T db;
private final boolean nettyDirect;
private final BufferAllocator alloc;
private final ColumnFamilyHandle cfh;
protected final MeterRegistry meterRegistry;
@ -103,15 +81,11 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
private final DBColumnKeyMayExistGetter keyMayExistGetter;
public AbstractRocksDBColumn(T db,
boolean nettyDirect,
BufferAllocator alloc,
String databaseName,
ColumnFamilyHandle cfh,
MeterRegistry meterRegistry,
StampedLock closeLock) {
this.db = db;
this.nettyDirect = nettyDirect && alloc.getAllocationType() == OFF_HEAP;
this.alloc = alloc;
this.cfh = cfh;
String columnName;
try {
@ -253,18 +227,10 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
/**
* This method should not modify or move the writerIndex/readerIndex of the key
*/
static AbstractSlice<?> setIterateBound(boolean allowNettyDirect,
ReadOptions readOpts, IterateBound boundType, Buffer key) {
static AbstractSlice<?> setIterateBound(ReadOptions readOpts, IterateBound boundType, Buf key) {
requireNonNull(key);
AbstractSlice<?> slice;
if (allowNettyDirect && USE_DIRECT_BUFFER_BOUNDS && isReadOnlyDirect(key)) {
ByteBuffer keyInternalByteBuffer = ((BufferComponent) key).readableBuffer();
assert keyInternalByteBuffer.position() == 0;
slice = new DirectSlice(keyInternalByteBuffer, key.readableBytes());
assert slice.size() == key.readableBytes();
} else {
slice = new Slice(requireNonNull(LLUtils.toArray(key)));
}
slice = new Slice(requireNonNull(LLUtils.asArray(key)));
if (boundType == IterateBound.LOWER) {
readOpts.setIterateLowerBound(slice);
} else {
@ -282,23 +248,22 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
/**
* This method should not modify or move the writerIndex/readerIndex of the buffers inside the range
*/
@Override
@NotNull
public RocksIteratorObj newRocksIterator(boolean allowNettyDirect,
ReadOptions readOptions,
LLRange range,
boolean reverse) throws RocksDBException {
assert !Schedulers.isInNonBlockingThread() : "Called getRocksIterator in a nonblocking thread";
var rocksIterator = this.newIterator(readOptions, range.getMinUnsafe(), range.getMaxUnsafe());
public RocksIteratorObj newRocksIterator(ReadOptions readOptions, LLRange range, boolean reverse)
throws RocksDBException {
assert !LLUtils.isInNonBlockingThread() : "Called getRocksIterator in a nonblocking thread";
var rocksIterator = this.newIterator(readOptions, range.getMin(), range.getMax());
try {
if (reverse) {
if (!LLLocalDictionary.PREFER_AUTO_SEEK_BOUND && range.hasMax()) {
rocksIterator.seekFrom(range.getMaxUnsafe());
rocksIterator.seekFrom(range.getMax());
} else {
rocksIterator.seekToLast();
}
} else {
if (!LLLocalDictionary.PREFER_AUTO_SEEK_BOUND && range.hasMin()) {
rocksIterator.seekTo(range.getMinUnsafe());
rocksIterator.seekTo(range.getMin());
} else {
rocksIterator.seekToFirst();
}
@ -331,14 +296,8 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
RocksDBUtils.ensureOwned(rocksObject);
}
protected void ensureOwned(Buffer buffer) {
if (buffer != null && !buffer.isAccessible()) {
throw new IllegalStateException("Buffer is not accessible");
}
}
@Override
public @Nullable Buffer get(@NotNull ReadOptions readOptions, Buffer key) throws RocksDBException {
public @Nullable Buf get(@NotNull ReadOptions readOptions, Buf key) throws RocksDBException {
var closeReadLock = closeLock.readLock();
try {
ensureOpen();
@ -350,221 +309,74 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
}
@Override
public void put(@NotNull WriteOptions writeOptions, Buffer key, Buffer value) throws RocksDBException {
public void put(@NotNull WriteOptions writeOptions, Buf key, Buf value) throws RocksDBException {
var closeReadLock = closeLock.readLock();
try {
ensureOpen();
ensureOwned(writeOptions);
assert key.isAccessible();
assert value.isAccessible();
this.keyBufferSize.record(key.readableBytes());
this.writeValueBufferSize.record(value.readableBytes());
if (nettyDirect) {
// Get the key nio buffer to pass to RocksDB
ByteBuffer keyNioBuffer;
boolean mustCloseKey;
{
if (!LLUtils.isReadOnlyDirect(key)) {
// If the nio buffer is not available, copy the netty buffer into a new direct buffer
mustCloseKey = true;
var directKey = DefaultBufferAllocators.offHeapAllocator().allocate(key.readableBytes());
key.copyInto(key.readerOffset(), directKey, 0, key.readableBytes());
key = directKey;
} else {
mustCloseKey = false;
}
keyNioBuffer = ((BufferComponent) key).readableBuffer();
assert keyNioBuffer.isDirect();
assert keyNioBuffer.limit() == key.readableBytes();
}
try {
// Get the value nio buffer to pass to RocksDB
ByteBuffer valueNioBuffer;
boolean mustCloseValue;
{
if (!LLUtils.isReadOnlyDirect(value)) {
// If the nio buffer is not available, copy the netty buffer into a new direct buffer
mustCloseValue = true;
var directValue = DefaultBufferAllocators.offHeapAllocator().allocate(value.readableBytes());
value.copyInto(value.readerOffset(), directValue, 0, value.readableBytes());
value = directValue;
} else {
mustCloseValue = false;
}
valueNioBuffer = ((BufferComponent) value).readableBuffer();
assert valueNioBuffer.isDirect();
assert valueNioBuffer.limit() == value.readableBytes();
}
try {
db.put(cfh, writeOptions, keyNioBuffer, valueNioBuffer);
} finally {
if (mustCloseValue) {
value.close();
}
}
} finally {
if (mustCloseKey) {
key.close();
}
}
} else {
db.put(cfh, writeOptions, LLUtils.toArray(key), LLUtils.toArray(value));
}
this.keyBufferSize.record(key.size());
this.writeValueBufferSize.record(value.size());
db.put(cfh, writeOptions, LLUtils.asArray(key), LLUtils.asArray(value));
} finally {
closeLock.unlockRead(closeReadLock);
}
}
@Override
public boolean exists(@NotNull ReadOptions readOptions, Buffer key) throws RocksDBException {
public boolean exists(@NotNull ReadOptions readOptions, Buf key) throws RocksDBException {
var closeReadLock = closeLock.readLock();
try {
ensureOpen();
ensureOwned(readOptions);
if (nettyDirect) {
// Get the key nio buffer to pass to RocksDB
ByteBuffer keyNioBuffer;
boolean mustCloseKey;
{
if (!LLUtils.isReadOnlyDirect(key)) {
// If the nio buffer is not available, copy the netty buffer into a new direct buffer
mustCloseKey = true;
var directKey = DefaultBufferAllocators.offHeapAllocator().allocate(key.readableBytes());
key.copyInto(key.readerOffset(), directKey, 0, key.readableBytes());
key = directKey;
} else {
mustCloseKey = false;
}
keyNioBuffer = ((BufferComponent) key).readableBuffer();
assert keyNioBuffer.isDirect();
assert keyNioBuffer.limit() == key.readableBytes();
}
try {
if (db.keyMayExist(cfh, keyNioBuffer)) {
int size = db.get(cfh, readOptions, keyNioBuffer.position(0), LLUtils.EMPTY_BYTE_BUFFER);
boolean found = size != RocksDB.NOT_FOUND;
if (found) {
readValueFoundWithBloomSimpleBufferSize.record(size);
return true;
} else {
readValueNotFoundWithMayExistBloomBufferSize.record(0);
return false;
}
} else {
readValueNotFoundWithBloomBufferSize.record(0);
return false;
}
} finally {
if (mustCloseKey) {
key.close();
}
}
} else {
int size = RocksDB.NOT_FOUND;
byte[] keyBytes = LLUtils.toArray(key);
Holder<byte[]> data = new Holder<>();
boolean mayExistHit = false;
if (db.keyMayExist(cfh, readOptions, keyBytes, data)) {
mayExistHit = true;
if (data.getValue() != null) {
size = data.getValue().length;
} else {
size = db.get(cfh, readOptions, keyBytes, NO_DATA);
}
}
boolean found = size != RocksDB.NOT_FOUND;
if (found) {
readValueFoundWithBloomSimpleBufferSize.record(size);
int size = RocksDB.NOT_FOUND;
byte[] keyBytes = LLUtils.asArray(key);
Holder<byte[]> data = new Holder<>();
boolean mayExistHit = false;
if (db.keyMayExist(cfh, readOptions, keyBytes, data)) {
mayExistHit = true;
if (data.getValue() != null) {
size = data.getValue().length;
} else {
if (mayExistHit) {
readValueNotFoundWithMayExistBloomBufferSize.record(0);
} else {
readValueNotFoundWithBloomBufferSize.record(0);
}
size = db.get(cfh, readOptions, keyBytes, NO_DATA);
}
return found;
}
boolean found = size != RocksDB.NOT_FOUND;
if (found) {
readValueFoundWithBloomSimpleBufferSize.record(size);
} else {
if (mayExistHit) {
readValueNotFoundWithMayExistBloomBufferSize.record(0);
} else {
readValueNotFoundWithBloomBufferSize.record(0);
}
}
return found;
} finally {
closeLock.unlockRead(closeReadLock);
}
}
@Override
public boolean mayExists(@NotNull ReadOptions readOptions, Buffer key) throws RocksDBException {
public boolean mayExists(@NotNull ReadOptions readOptions, Buf key) throws RocksDBException {
var closeReadLock = closeLock.readLock();
try {
ensureOpen();
ensureOwned(readOptions);
if (nettyDirect) {
// Get the key nio buffer to pass to RocksDB
ByteBuffer keyNioBuffer;
boolean mustCloseKey;
{
if (!LLUtils.isReadOnlyDirect(key)) {
// If the nio buffer is not available, copy the netty buffer into a new direct buffer
mustCloseKey = true;
var directKey = DefaultBufferAllocators.offHeapAllocator().allocate(key.readableBytes());
key.copyInto(key.readerOffset(), directKey, 0, key.readableBytes());
key = directKey;
} else {
mustCloseKey = false;
}
keyNioBuffer = ((BufferComponent) key).readableBuffer();
assert keyNioBuffer.isDirect();
assert keyNioBuffer.limit() == key.readableBytes();
}
try {
return db.keyMayExist(cfh, readOptions, keyNioBuffer);
} finally {
if (mustCloseKey) {
key.close();
}
}
} else {
byte[] keyBytes = LLUtils.toArray(key);
return db.keyMayExist(cfh, readOptions, keyBytes, null);
}
byte[] keyBytes = LLUtils.asArray(key);
return db.keyMayExist(cfh, readOptions, keyBytes, null);
} finally {
closeLock.unlockRead(closeReadLock);
}
}
@Override
public void delete(WriteOptions writeOptions, Buffer key) throws RocksDBException {
public void delete(WriteOptions writeOptions, Buf key) throws RocksDBException {
var closeReadLock = closeLock.readLock();
try {
ensureOpen();
ensureOwned(writeOptions);
keyBufferSize.record(key.readableBytes());
if (nettyDirect) {
// Get the key nio buffer to pass to RocksDB
ByteBuffer keyNioBuffer;
boolean mustCloseKey;
{
if (!LLUtils.isReadOnlyDirect(key)) {
// If the nio buffer is not available, copy the netty buffer into a new direct buffer
mustCloseKey = true;
var directKey = DefaultBufferAllocators.offHeapAllocator().allocate(key.readableBytes());
key.copyInto(key.readerOffset(), directKey, 0, key.readableBytes());
key = directKey;
} else {
mustCloseKey = false;
}
keyNioBuffer = ((BufferComponent) key).readableBuffer();
assert keyNioBuffer.isDirect();
assert keyNioBuffer.limit() == key.readableBytes();
}
try {
db.delete(cfh, writeOptions, keyNioBuffer);
} finally {
if (mustCloseKey) {
key.close();
}
}
} else {
db.delete(cfh, writeOptions, LLUtils.toArray(key));
}
keyBufferSize.record(key.size());
db.delete(cfh, writeOptions, LLUtils.asArray(key));
} finally {
closeLock.unlockRead(closeReadLock);
}
@ -690,21 +502,17 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
@Override
public final @NotNull UpdateAtomicResult updateAtomic(@NotNull ReadOptions readOptions,
@NotNull WriteOptions writeOptions,
Buffer key,
Buf key,
BinarySerializationFunction updater,
UpdateAtomicResultMode returnMode) throws IOException {
UpdateAtomicResultMode returnMode) throws RocksDBException {
var closeReadLock = closeLock.readLock();
try {
ensureOpen();
ensureOwned(readOptions);
try {
keyBufferSize.record(key.readableBytes());
keyBufferSize.record(key.size());
startedUpdate.increment();
return updateAtomicImpl(readOptions, writeOptions, key, updater, returnMode);
} catch (IOException e) {
throw e;
} catch (Exception e) {
throw new IOException(e);
} finally {
endedUpdate.increment();
}
@ -733,31 +541,29 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
protected abstract @NotNull UpdateAtomicResult updateAtomicImpl(@NotNull ReadOptions readOptions,
@NotNull WriteOptions writeOptions,
Buffer key,
Buf key,
BinarySerializationFunction updater,
UpdateAtomicResultMode returnMode) throws IOException;
UpdateAtomicResultMode returnMode) throws RocksDBException;
@Override
@NotNull
public RocksIteratorObj newIterator(@NotNull ReadOptions readOptions,
@Nullable Buffer min,
@Nullable Buffer max) {
@Nullable Buf min,
@Nullable Buf max) {
var closeReadLock = closeLock.readLock();
try {
ensureOpen();
ensureOwned(readOptions);
ensureOwned(min);
ensureOwned(max);
AbstractSlice<?> sliceMin;
AbstractSlice<?> sliceMax;
if (min != null) {
sliceMin = setIterateBound(nettyDirect, readOptions, IterateBound.LOWER, min);
sliceMin = setIterateBound(readOptions, IterateBound.LOWER, min);
} else {
sliceMin = null;
}
try {
if (max != null) {
sliceMax = setIterateBound(nettyDirect, readOptions, IterateBound.UPPER, max);
sliceMax = setIterateBound(readOptions, IterateBound.UPPER, max);
} else {
sliceMax = null;
}
@ -769,7 +575,6 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
sliceMax,
min,
max,
nettyDirect,
this.startedIterSeek,
this.endedIterSeek,
this.iterSeekTime,
@ -814,6 +619,8 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
try {
ensureOpen();
RocksDBUtils.forceCompaction(db, db.getName(), cfh, volumeId, logger);
} catch (RocksDBException e) {
throw new RuntimeException(e);
} finally {
closeLock.unlockRead(closeReadLock);
}
@ -824,11 +631,6 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
return cfh;
}
@Override
public BufferAllocator getAllocator() {
return alloc;
}
public MeterRegistry getMeterRegistry() {
return meterRegistry;
}
@ -847,10 +649,6 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
private class DBColumnKeyMayExistGetter extends KeyMayExistGetter {
public DBColumnKeyMayExistGetter() {
super(alloc, nettyDirect);
}
@Override
protected KeyMayExist keyMayExist(ReadOptions readOptions, ByteBuffer key, ByteBuffer value) {
return db.keyMayExist(cfh, readOptions, key, value);

4
src/main/java/it/cavallium/dbengine/database/disk/BinarySerializationFunction.java
View File

@ -1,7 +1,7 @@
package it.cavallium.dbengine.database.disk;
import io.netty5.buffer.Buffer;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import org.jetbrains.annotations.Nullable;
public interface BinarySerializationFunction extends SerializationFunction<@Nullable Buffer, @Nullable Buffer> {}
public interface BinarySerializationFunction extends SerializationFunction<@Nullable Buf, @Nullable Buf> {}

125
src/main/java/it/cavallium/dbengine/database/disk/CachedIndexSearcherManager.java
View File

@ -1,23 +1,19 @@
package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.client.UninterruptibleScheduler.uninterruptibleScheduler;
import static it.cavallium.dbengine.lucene.LuceneUtils.luceneScheduler;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.lucene.LuceneCloseable;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.utils.SimpleResource;
import java.io.IOException;
import java.io.UncheckedIOException;
import it.cavallium.dbengine.utils.DBException;
import java.time.Duration;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.LockSupport;
import org.apache.logging.log4j.LogManager;
@ -30,11 +26,6 @@ import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.AlreadyClosedException;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import it.cavallium.dbengine.utils.ShortNamedThreadFactory;
import reactor.core.Disposable;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
// todo: deduplicate code between Cached and Simple searcher managers
public class CachedIndexSearcherManager extends SimpleResource implements IndexSearcherManager, LuceneCloseable {
@ -49,7 +40,7 @@ public class CachedIndexSearcherManager extends SimpleResource implements IndexS
@Nullable
private final SnapshotsManager snapshotsManager;
private final Scheduler luceneHeavyTasksScheduler;
private final ScheduledExecutorService luceneHeavyTasksScheduler;
private final Similarity similarity;
private final SearcherManager searcherManager;
private final Duration queryRefreshDebounceTime;
@ -57,17 +48,16 @@ public class CachedIndexSearcherManager extends SimpleResource implements IndexS
private final AtomicLong activeSearchers = new AtomicLong(0);
private final AtomicLong activeRefreshes = new AtomicLong(0);
private final LoadingCache<LLSnapshot, Mono<LLIndexSearcher>> cachedSnapshotSearchers;
private final Mono<LLIndexSearcher> cachedMainSearcher;
private final Disposable refreshSubscription;
private final LoadingCache<LLSnapshot, LLIndexSearcher> cachedSnapshotSearchers;
private final ScheduledFuture<?> refreshSubscription;
public CachedIndexSearcherManager(IndexWriter indexWriter,
@Nullable SnapshotsManager snapshotsManager,
Scheduler luceneHeavyTasksScheduler,
ScheduledExecutorService luceneHeavyTasksScheduler,
Similarity similarity,
boolean applyAllDeletes,
boolean writeAllDeletes,
Duration queryRefreshDebounceTime) throws IOException {
Duration queryRefreshDebounceTime) {
this.snapshotsManager = snapshotsManager;
this.luceneHeavyTasksScheduler = luceneHeavyTasksScheduler;
this.similarity = similarity;
@ -75,13 +65,17 @@ public class CachedIndexSearcherManager extends SimpleResource implements IndexS
this.searcherManager = new SearcherManager(indexWriter, applyAllDeletes, writeAllDeletes, SEARCHER_FACTORY);
refreshSubscription = LLUtils.scheduleRepeated(luceneHeavyTasksScheduler, () -> {
try {
maybeRefresh();
} catch (Exception ex) {
LOG.error("Failed to refresh the searcher manager", ex);
}
}, queryRefreshDebounceTime);
refreshSubscription = luceneHeavyTasksScheduler.scheduleAtFixedRate(() -> {
try {
maybeRefresh();
} catch (Exception ex) {
LOG.error("Failed to refresh the searcher manager", ex);
}
},
queryRefreshDebounceTime.toMillis(),
queryRefreshDebounceTime.toMillis(),
TimeUnit.MILLISECONDS
);
this.cachedSnapshotSearchers = CacheBuilder.newBuilder()
.expireAfterWrite(queryRefreshDebounceTime)
@ -89,44 +83,40 @@ public class CachedIndexSearcherManager extends SimpleResource implements IndexS
.maximumSize(3)
.build(new CacheLoader<>() {
@Override
public Mono<LLIndexSearcher> load(@NotNull LLSnapshot snapshot) {
public LLIndexSearcher load(@NotNull LLSnapshot snapshot) {
return CachedIndexSearcherManager.this.generateCachedSearcher(snapshot);
}
});
this.cachedMainSearcher = this.generateCachedSearcher(null);
}
private Mono<LLIndexSearcher> generateCachedSearcher(@Nullable LLSnapshot snapshot) {
return Mono.fromCallable(() -> {
if (isClosed()) {
return null;
}
activeSearchers.incrementAndGet();
try {
IndexSearcher indexSearcher;
boolean fromSnapshot;
if (snapshotsManager == null || snapshot == null) {
indexSearcher = searcherManager.acquire();
fromSnapshot = false;
} else {
indexSearcher = snapshotsManager.resolveSnapshot(snapshot).getIndexSearcher(SEARCH_EXECUTOR);
fromSnapshot = true;
}
indexSearcher.setSimilarity(similarity);
assert indexSearcher.getIndexReader().getRefCount() > 0;
LLIndexSearcher llIndexSearcher;
if (fromSnapshot) {
llIndexSearcher = new SnapshotIndexSearcher(indexSearcher);
} else {
llIndexSearcher = new MainIndexSearcher(indexSearcher, searcherManager);
}
return llIndexSearcher;
} catch (Throwable ex) {
activeSearchers.decrementAndGet();
throw ex;
}
})
.transform(LuceneUtils::scheduleLucene);
private LLIndexSearcher generateCachedSearcher(@Nullable LLSnapshot snapshot) {
if (isClosed()) {
return null;
}
activeSearchers.incrementAndGet();
try {
IndexSearcher indexSearcher;
boolean fromSnapshot;
if (snapshotsManager == null || snapshot == null) {
indexSearcher = searcherManager.acquire();
fromSnapshot = false;
} else {
indexSearcher = snapshotsManager.resolveSnapshot(snapshot).getIndexSearcher(SEARCH_EXECUTOR);
fromSnapshot = true;
}
indexSearcher.setSimilarity(similarity);
assert indexSearcher.getIndexReader().getRefCount() > 0;
LLIndexSearcher llIndexSearcher;
if (fromSnapshot) {
llIndexSearcher = new SnapshotIndexSearcher(indexSearcher);
} else {
llIndexSearcher = new MainIndexSearcher(indexSearcher, searcherManager);
}
return llIndexSearcher;
} catch (Throwable ex) {
activeSearchers.decrementAndGet();
throw ex;
}
}
private void dropCachedIndexSearcher() {
@ -135,7 +125,7 @@ public class CachedIndexSearcherManager extends SimpleResource implements IndexS
}
@Override
public void maybeRefreshBlocking() throws IOException {
public void maybeRefreshBlocking() {
try {
activeRefreshes.incrementAndGet();
searcherManager.maybeRefreshBlocking();
@ -147,7 +137,7 @@ public class CachedIndexSearcherManager extends SimpleResource implements IndexS
}
@Override
public void maybeRefresh() throws IOException {
public void maybeRefresh() {
try {
activeRefreshes.incrementAndGet();
searcherManager.maybeRefresh();
@ -159,9 +149,9 @@ public class CachedIndexSearcherManager extends SimpleResource implements IndexS
}
@Override
public Mono<LLIndexSearcher> retrieveSearcher(@Nullable LLSnapshot snapshot) {
public LLIndexSearcher retrieveSearcher(@Nullable LLSnapshot snapshot) {
if (snapshot == null) {
return this.cachedMainSearcher;
return this.generateCachedSearcher(null);
} else {
return this.cachedSnapshotSearchers.getUnchecked(snapshot);
}
@ -170,10 +160,15 @@ public class CachedIndexSearcherManager extends SimpleResource implements IndexS
@Override
protected void onClose() {
LOG.debug("Closing IndexSearcherManager...");
refreshSubscription.dispose();
long initTime = System.nanoTime();
refreshSubscription.cancel(false);
while (!refreshSubscription.isDone() && (System.nanoTime() - initTime) <= 240000000000L) {
LockSupport.parkNanos(50000000);
}
refreshSubscription.cancel(true);
LOG.debug("Closed IndexSearcherManager");
LOG.debug("Closing refreshes...");
long initTime = System.nanoTime();
initTime = System.nanoTime();
while (activeRefreshes.get() > 0 && (System.nanoTime() - initTime) <= 15000000000L) {
LockSupport.parkNanos(50000000);
}
@ -227,7 +222,7 @@ public class CachedIndexSearcherManager extends SimpleResource implements IndexS
try {
searcherManager.release(indexSearcher);
} catch (IOException ex) {
throw new UncheckedIOException(ex);
throw new DBException(ex);
}
}
}

112
src/main/java/it/cavallium/dbengine/database/disk/CappedWriteBatch.java
View File

@ -1,19 +1,9 @@
package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.database.LLUtils.isDirect;
import static it.cavallium.dbengine.database.LLUtils.isReadOnlyDirect;
import static it.cavallium.dbengine.database.LLUtils.asArray;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.BufferAllocator;
import io.netty5.buffer.BufferComponent;
import io.netty5.util.Send;
import io.netty5.util.internal.PlatformDependent;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.RocksDBColumn;
import java.io.Closeable;
import it.cavallium.dbengine.buffers.Buf;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.RocksDBException;
import org.rocksdb.WriteBatch;
@ -21,61 +11,30 @@ import org.rocksdb.WriteOptions;
public class CappedWriteBatch extends WriteBatch {
/**
* Default: true, Use false to debug problems with direct buffers
*/
private static final boolean USE_FAST_DIRECT_BUFFERS = true;
private final RocksDBColumn db;
private final BufferAllocator alloc;
private final int cap;
private final WriteOptions writeOptions;
private final List<Buffer> buffersToRelease;
private final List<ByteBuffer> byteBuffersToRelease;
/**
* @param db
* @param cap The limit of operations
*/
public CappedWriteBatch(RocksDBColumn db,
BufferAllocator alloc,
int cap,
int reservedWriteBatchSize,
long maxWriteBatchSize,
WriteOptions writeOptions) {
super(reservedWriteBatchSize);
this.db = db;
this.alloc = alloc;
this.cap = cap;
this.writeOptions = writeOptions;
this.setMaxBytes(maxWriteBatchSize);
this.buffersToRelease = new ArrayList<>();
this.byteBuffersToRelease = new ArrayList<>();
}
private synchronized void flushIfNeeded(boolean force) throws RocksDBException {
if (this.count() >= (force ? 1 : cap)) {
try {
db.write(writeOptions, this.getWriteBatch());
this.clear();
} finally {
releaseAllBuffers();
}
}
}
public synchronized void releaseAllBuffers() {
if (!buffersToRelease.isEmpty()) {
for (Buffer byteBuffer : buffersToRelease) {
byteBuffer.close();
}
buffersToRelease.clear();
}
if (!byteBuffersToRelease.isEmpty()) {
for (var byteBuffer : byteBuffersToRelease) {
PlatformDependent.freeDirectBuffer(byteBuffer);
}
byteBuffersToRelease.clear();
db.write(writeOptions, this.getWriteBatch());
this.clear();
}
}
@ -109,29 +68,9 @@ public class CappedWriteBatch extends WriteBatch {
}
public synchronized void put(ColumnFamilyHandle columnFamilyHandle,
Send<Buffer> keyToReceive,
Send<Buffer> valueToReceive) throws RocksDBException {
var key = keyToReceive.receive();
var value = valueToReceive.receive();
if (USE_FAST_DIRECT_BUFFERS
&& (isReadOnlyDirect(key))
&& (isReadOnlyDirect(value))) {
ByteBuffer keyNioBuffer = ((BufferComponent) key).readableBuffer();
ByteBuffer valueNioBuffer = ((BufferComponent) value).readableBuffer();
buffersToRelease.add(value);
buffersToRelease.add(key);
super.put(columnFamilyHandle, keyNioBuffer, valueNioBuffer);
} else {
try {
byte[] keyArray = LLUtils.toArray(key);
byte[] valueArray = LLUtils.toArray(value);
super.put(columnFamilyHandle, keyArray, valueArray);
} finally {
key.close();
value.close();
}
}
Buf keyToReceive,
Buf valueToReceive) throws RocksDBException {
super.put(columnFamilyHandle, asArray(keyToReceive), asArray(valueToReceive));
flushIfNeeded(false);
}
@ -159,19 +98,8 @@ public class CappedWriteBatch extends WriteBatch {
flushIfNeeded(false);
}
public synchronized void delete(ColumnFamilyHandle columnFamilyHandle, Send<Buffer> keyToReceive) throws RocksDBException {
var key = keyToReceive.receive();
if (USE_FAST_DIRECT_BUFFERS && isReadOnlyDirect(key)) {
ByteBuffer keyNioBuffer = ((BufferComponent) key).readableBuffer();
buffersToRelease.add(key);
delete(columnFamilyHandle, keyNioBuffer);
} else {
try {
super.delete(columnFamilyHandle, LLUtils.toArray(key));
} finally {
key.close();
}
}
public synchronized void delete(ColumnFamilyHandle columnFamilyHandle, Buf keyToDelete) throws RocksDBException {
super.delete(columnFamilyHandle, asArray(keyToDelete));
flushIfNeeded(false);
}
@ -221,7 +149,6 @@ public class CappedWriteBatch extends WriteBatch {
@Override
public synchronized void clear() {
super.clear();
releaseAllBuffers();
}
@Override
@ -250,26 +177,11 @@ public class CappedWriteBatch extends WriteBatch {
}
public synchronized void writeToDbAndClose() throws RocksDBException {
try {
flushIfNeeded(true);
super.close();
} finally {
releaseAllBuffers();
}
flushIfNeeded(true);
super.close();
}
public void flush() throws RocksDBException {
try {
flushIfNeeded(true);
} finally {
releaseAllBuffers();
}
flushIfNeeded(true);
}
/*
protected void disposeInternal(boolean owningHandle) {
super.disposeInternal(owningHandle);
releaseAllBuffers();
}
*/
}

84
src/main/java/it/cavallium/dbengine/database/disk/HugePqEnv.java
View File

@ -1,84 +0,0 @@
package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.database.disk.LLTempHugePqEnv.getColumnOptions;
import com.google.common.primitives.Ints;
import io.micrometer.core.instrument.composite.CompositeMeterRegistry;
import io.netty5.buffer.BufferAllocator;
import it.cavallium.dbengine.utils.SimpleResource;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import java.io.Closeable;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Objects;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.StampedLock;
import org.rocksdb.AbstractComparator;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
public class HugePqEnv extends SimpleResource {
private final RocksDB db;
private final ArrayList<ColumnFamilyHandle> defaultCfh;
private final Int2ObjectMap<ColumnFamilyHandle> cfhs = new Int2ObjectOpenHashMap<>();
public HugePqEnv(RocksDB db, ArrayList<ColumnFamilyHandle> defaultCfh) {
this.db = db;
this.defaultCfh = defaultCfh;
}
@Override
protected void onClose() {
for (var cfh : defaultCfh) {
db.destroyColumnFamilyHandle(cfh);
}
try {
db.closeE();
} catch (RocksDBException e) {
throw new IllegalStateException(e);
}
}
public int createColumnFamily(int name, AbstractComparator comparator) throws RocksDBException {
var cfh = db.createColumnFamily(new ColumnFamilyDescriptor(Ints.toByteArray(name), getColumnOptions(comparator)));
synchronized (cfhs) {
var prev = cfhs.put(name, cfh);
if (prev != null) {
throw new UnsupportedOperationException("Db " + name + " already exists");
}
return name;
}
}
public void deleteColumnFamily(int db) throws RocksDBException {
ColumnFamilyHandle cfh;
synchronized (cfhs) {
cfh = cfhs.remove(db);
}
if (cfh != null) {
this.db.dropColumnFamily(cfh);
this.db.destroyColumnFamilyHandle(cfh);
}
}
public StandardRocksDBColumn openDb(int hugePqId) {
ColumnFamilyHandle cfh;
synchronized (cfhs) {
cfh = Objects.requireNonNull(cfhs.get(hugePqId), () -> "column " + hugePqId + " does not exist");
}
return new StandardRocksDBColumn(db,
true,
BufferAllocator.offHeapPooled(),
db.getName(),
cfh,
new CompositeMeterRegistry(),
new StampedLock()
);
}
}

12
src/main/java/it/cavallium/dbengine/database/disk/IndexSearcherManager.java
View File

@ -1,20 +1,16 @@
package it.cavallium.dbengine.database.disk;
import io.netty5.util.Send;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.SafeCloseable;
import java.io.IOException;
import java.util.function.Function;
import org.apache.lucene.search.IndexSearcher;
import java.util.function.Supplier;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public interface IndexSearcherManager extends SafeCloseable {
void maybeRefreshBlocking() throws IOException;
void maybeRefreshBlocking();
void maybeRefresh() throws IOException;
void maybeRefresh();
Mono<LLIndexSearcher> retrieveSearcher(@Nullable LLSnapshot snapshot);
LLIndexSearcher retrieveSearcher(@Nullable LLSnapshot snapshot);
}

191
src/main/java/it/cavallium/dbengine/database/disk/KeyMayExistGetter.java
View File

@ -1,31 +1,19 @@
package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.database.LLUtils.INITIAL_DIRECT_READ_BYTE_BUF_SIZE_BYTES;
import static java.lang.Boolean.parseBoolean;
import static java.lang.System.getProperty;
import static java.util.Objects.requireNonNull;
import static org.rocksdb.KeyMayExist.KeyMayExistEnum.kExistsWithValue;
import static org.rocksdb.KeyMayExist.KeyMayExistEnum.kExistsWithoutValue;
import static org.rocksdb.KeyMayExist.KeyMayExistEnum.kNotExist;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.BufferAllocator;
import io.netty5.buffer.BufferComponent;
import io.netty5.buffer.DefaultBufferAllocators;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.LLUtils;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.HexFormat;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.Holder;
import org.rocksdb.KeyMayExist;
import org.rocksdb.KeyMayExist.KeyMayExistEnum;
import org.rocksdb.ReadOptions;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
public abstract class KeyMayExistGetter {
@ -40,180 +28,19 @@ public abstract class KeyMayExistGetter {
"it.cavallium.dbengine.mayexist.strict_no_value",
"false"
));
private static final BufferAllocator OFF_HEAP_ALLOCATOR = DefaultBufferAllocators.offHeapAllocator();
private final BufferAllocator bufferAllocator;
private final boolean nettyDirect;
public KeyMayExistGetter(BufferAllocator bufferAllocator, boolean nettyDirect) {
this.bufferAllocator = bufferAllocator;
this.nettyDirect = nettyDirect;
public KeyMayExistGetter() {
}
public final @Nullable Buffer get(@NotNull ReadOptions readOptions, Buffer key) throws RocksDBException {
recordKeyBufferSize(key.readableBytes());
if (nettyDirect) {
return getDirect(readOptions, key);
} else {
return getHeap(readOptions, key);
}
public final @Nullable Buf get(@NotNull ReadOptions readOptions, Buf key) throws RocksDBException {
recordKeyBufferSize(key.size());
return getHeap(readOptions, key);
}
private Buffer getDirect(ReadOptions readOptions, Buffer key) throws RocksDBException {
int readAttemptsCount = 0;
// Get the key nio buffer to pass to RocksDB
ByteBuffer keyNioBuffer;
boolean mustCloseKey;
{
if (!LLUtils.isReadOnlyDirect(key)) {
// If the nio buffer is not available, copy the netty buffer into a new direct buffer
mustCloseKey = true;
var directKey = OFF_HEAP_ALLOCATOR.allocate(key.readableBytes());
key.copyInto(key.readerOffset(), directKey, 0, key.readableBytes());
key = directKey;
} else {
mustCloseKey = false;
}
keyNioBuffer = ((BufferComponent) key).readableBuffer();
assert keyNioBuffer.isDirect();
assert keyNioBuffer.limit() == key.readableBytes();
}
try {
// Create a direct result buffer because RocksDB works only with direct buffers
var resultBuffer = bufferAllocator.allocate(INITIAL_DIRECT_READ_BYTE_BUF_SIZE_BYTES);
try {
assert resultBuffer.readerOffset() == 0;
assert resultBuffer.writerOffset() == 0;
var resultWritable = ((BufferComponent) resultBuffer).writableBuffer();
var keyMayExist = keyMayExist(readOptions, keyNioBuffer.rewind(), resultWritable.clear());
if (STRICT_MAYEXIST_NO_VALUE && keyMayExist.exists != kExistsWithValue && keyMayExist.valueLength != 0) {
// Create a direct result buffer because RocksDB works only with direct buffers
try (var realResultBuffer = bufferAllocator.allocate(INITIAL_DIRECT_READ_BYTE_BUF_SIZE_BYTES)) {
var resultWritableF = resultWritable;
var realResultWritable = ((BufferComponent) realResultBuffer).writableBuffer();
var realSize = get(readOptions, keyNioBuffer.rewind(), realResultWritable);
var hexFormat = HexFormat.ofDelimiter(" ");
LOG.error(
"KeyMayExist is {}, but value length is non-zero: {}! Disk value size is {}\nBytes from bloom cache:\n{}\nBytes from db:\n{}",
() -> keyMayExist.exists,
() -> keyMayExist.valueLength,
() -> realSize,
() -> {
resultBuffer.writerOffset(resultWritableF.limit());
return hexFormat.formatHex(LLUtils.toArray(resultBuffer));
},
() -> {
realResultBuffer.writerOffset(realResultWritable.limit());
return hexFormat.formatHex(LLUtils.toArray(realResultBuffer));
}
);
var sliceKME = LLUtils.toArray(resultBuffer.copy(0, Math.min(resultWritableF.limit(), realSize)));
var sliceDB = LLUtils.toArray(realResultBuffer.copy(0, Math.min(realResultWritable.limit(), realSize)));
throw new RocksDBException(
"KeyMayExist is " + keyMayExist.exists + ", but value length is non-zero: " + keyMayExist.valueLength
+ "! Disk value size is " + realSize + ". The bloom cache partial value is "
+ (Arrays.equals(sliceKME, sliceDB) ? "correct" : "corrupted"));
}
}
KeyMayExistEnum keyMayExistState = keyMayExist.exists;
int keyMayExistValueLength = keyMayExist.valueLength;
// At the beginning, size reflects the expected size, then it becomes the real data size
//noinspection SwitchStatementWithTooFewBranches
int size = switch (keyMayExistState) {
case kExistsWithValue -> keyMayExistValueLength;
default -> -1;
};
boolean isKExistsWithoutValue = false;
switch (keyMayExistState) {
case kNotExist: {
recordReadValueNotFoundWithBloomBufferSize(0);
resultBuffer.close();
return null;
}
// todo: kExistsWithValue is not reliable (read below),
// in some cases it should be treated as kExistsWithoutValue
case kExistsWithValue:
case kExistsWithoutValue: {
if (keyMayExistState == kExistsWithoutValue) {
isKExistsWithoutValue = true;
} else if (WORKAROUND_MAY_EXIST_FAKE_ZERO) {
// todo: "size == 0 || resultWritable.limit() == 0" is checked because keyMayExist is broken,
// and sometimes it returns an empty array, as if it exists
if (size == 0 || resultWritable.limit() == 0) {
isKExistsWithoutValue = true;
}
}
if (isKExistsWithoutValue) {
assert
!STRICT_MAYEXIST_NO_VALUE || keyMayExistValueLength == 0 :
"keyMayExist value length is " + keyMayExistValueLength + " instead of 0";
resultWritable.clear();
readAttemptsCount++;
// real data size
size = get(readOptions, keyNioBuffer.rewind(), resultWritable.clear());
if (keyMayExistState == kExistsWithValue && size != keyMayExistValueLength) {
throw new IllegalStateException("Bloom filter data is corrupted."
+ " Bloom value size=" + keyMayExistState + ", Real value size=" + size);
}
if (size == RocksDB.NOT_FOUND) {
resultBuffer.close();
recordReadValueNotFoundWithMayExistBloomBufferSize(0);
return null;
}
}
}
default: {
// real data size
assert size >= 0;
if (size <= resultWritable.limit()) {
if (isKExistsWithoutValue) {
recordReadValueFoundWithBloomUncachedBufferSize(size);
} else {
recordReadValueFoundWithBloomCacheBufferSize(size);
}
assert size == resultWritable.limit();
return resultBuffer.writerOffset(resultWritable.limit());
} else {
resultBuffer.ensureWritable(size);
resultWritable = ((BufferComponent) resultBuffer).writableBuffer();
assert resultBuffer.readerOffset() == 0;
assert resultBuffer.writerOffset() == 0;
readAttemptsCount++;
size = get(readOptions, keyNioBuffer.rewind(), resultWritable.clear());
if (size == RocksDB.NOT_FOUND) {
recordReadValueNotFoundWithMayExistBloomBufferSize(0);
resultBuffer.close();
return null;
}
assert size == resultWritable.limit();
if (isKExistsWithoutValue) {
recordReadValueFoundWithBloomUncachedBufferSize(size);
} else {
recordReadValueFoundWithBloomCacheBufferSize(size);
}
return resultBuffer.writerOffset(resultWritable.limit());
}
}
}
} catch (Throwable t) {
resultBuffer.close();
throw t;
}
} finally {
if (mustCloseKey) {
key.close();
}
recordReadAttempts(readAttemptsCount);
}
}
private Buffer getHeap(ReadOptions readOptions, Buffer key) throws RocksDBException {
private Buf getHeap(ReadOptions readOptions, Buf key) throws RocksDBException {
int readAttemptsCount = 0;
try {
byte[] keyArray = LLUtils.toArray(key);
byte[] keyArray = LLUtils.asArray(key);
requireNonNull(keyArray);
Holder<byte[]> data = new Holder<>();
if (keyMayExist(readOptions, keyArray, data)) {
@ -221,7 +48,7 @@ public abstract class KeyMayExistGetter {
// returns an empty array, as if it exists
if (data.getValue() != null && (!WORKAROUND_MAY_EXIST_FAKE_ZERO || data.getValue().length > 0)) {
recordReadValueFoundWithBloomCacheBufferSize(data.getValue().length);
return LLUtils.fromByteArray(bufferAllocator, data.getValue());
return LLUtils.asByteList(data.getValue());
} else {
readAttemptsCount++;
byte[] result = get(readOptions, keyArray);
@ -234,7 +61,7 @@ public abstract class KeyMayExistGetter {
return null;
} else {
recordReadValueFoundWithBloomUncachedBufferSize(0);
return LLUtils.fromByteArray(bufferAllocator, result);
return LLUtils.asByteList(result);
}
}
} else {

7
src/main/java/it/cavallium/dbengine/database/disk/LLIndexSearcher.java
View File

@ -1,17 +1,10 @@
package it.cavallium.dbengine.database.disk;
import io.netty5.buffer.Drop;
import io.netty5.buffer.Owned;
import io.netty5.buffer.internal.ResourceSupport;
import it.cavallium.dbengine.database.DiscardingCloseable;
import it.cavallium.dbengine.database.SafeCloseable;
import it.cavallium.dbengine.utils.SimpleResource;
import java.io.Closeable;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
public abstract class LLIndexSearcher extends SimpleResource implements DiscardingCloseable {

3
src/main/java/it/cavallium/dbengine/database/disk/LLIndexSearcherImpl.java
View File

@ -1,11 +1,8 @@
package it.cavallium.dbengine.database.disk;
import it.cavallium.dbengine.database.DiscardingCloseable;
import it.cavallium.dbengine.utils.SimpleResource;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
public abstract class LLIndexSearcherImpl extends LLIndexSearcher {

149
src/main/java/it/cavallium/dbengine/database/disk/LLLocalDatabaseConnection.java
View File

@ -1,56 +1,38 @@
package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.lucene.LuceneUtils.luceneScheduler;
import io.micrometer.core.instrument.MeterRegistry;
import io.netty5.buffer.BufferAllocator;
import it.cavallium.dbengine.database.LLDatabaseConnection;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneRocksDBManager;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.rpc.current.data.Column;
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
import it.cavallium.dbengine.rpc.current.data.IndicizerAnalyzers;
import it.cavallium.dbengine.rpc.current.data.IndicizerSimilarities;
import it.cavallium.dbengine.rpc.current.data.LuceneIndexStructure;
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
import it.cavallium.dbengine.utils.DBException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
public class LLLocalDatabaseConnection implements LLDatabaseConnection {
private final AtomicBoolean connected = new AtomicBoolean();
private final BufferAllocator allocator;
private final MeterRegistry meterRegistry;
private final Path basePath;
private final boolean inMemory;
private final LuceneRocksDBManager rocksDBManager;
private final AtomicReference<LLTempHugePqEnv> env = new AtomicReference<>();
public LLLocalDatabaseConnection(BufferAllocator allocator,
public LLLocalDatabaseConnection(
MeterRegistry meterRegistry,
Path basePath,
boolean inMemory,
LuceneRocksDBManager rocksDBManager) {
this.allocator = allocator;
boolean inMemory) {
this.meterRegistry = meterRegistry;
this.basePath = basePath;
this.inMemory = inMemory;
this.rocksDBManager = rocksDBManager;
}
@Override
public BufferAllocator getAllocator() {
return allocator;
}
public MeterRegistry getMeterRegistry() {
@ -58,94 +40,67 @@ public class LLLocalDatabaseConnection implements LLDatabaseConnection {
}
@Override
public Mono<LLDatabaseConnection> connect() {
return Mono
.<LLDatabaseConnection>fromCallable(() -> {
if (!connected.compareAndSet(false, true)) {
throw new IllegalStateException("Already connected");
}
if (Files.notExists(basePath)) {
Files.createDirectories(basePath);
}
var prev = env.getAndSet(new LLTempHugePqEnv());
if (prev != null) {
throw new IllegalStateException("Env was already set");
}
return this;
})
.subscribeOn(Schedulers.boundedElastic());
public LLDatabaseConnection connect() {
if (!connected.compareAndSet(false, true)) {
throw new IllegalStateException("Already connected");
}
if (Files.notExists(basePath)) {
try {
Files.createDirectories(basePath);
} catch (IOException e) {
throw new DBException(e);
}
}
return this;
}
@Override
public Mono<LLLocalKeyValueDatabase> getDatabase(String name,
List<Column> columns,
DatabaseOptions databaseOptions) {
return Mono
.fromCallable(() -> new LLLocalKeyValueDatabase(
allocator,
meterRegistry,
name,
inMemory,
basePath.resolve("database_" + name),
columns,
new LinkedList<>(),
databaseOptions
))
.subscribeOn(Schedulers.boundedElastic());
public LLLocalKeyValueDatabase getDatabase(String name, List<Column> columns, DatabaseOptions databaseOptions) {
return new LLLocalKeyValueDatabase(meterRegistry,
name,
inMemory,
basePath.resolve("database_" + name),
columns,
new LinkedList<>(),
databaseOptions
);
}
@Override
public Mono<? extends LLLuceneIndex> getLuceneIndex(String clusterName,
public LLLuceneIndex getLuceneIndex(String clusterName,
LuceneIndexStructure indexStructure,
IndicizerAnalyzers indicizerAnalyzers,
IndicizerSimilarities indicizerSimilarities,
LuceneOptions luceneOptions,
@Nullable LuceneHacks luceneHacks) {
return Mono
.fromCallable(() -> {
var env = this.env.get();
if (clusterName == null) {
throw new IllegalArgumentException("Cluster name must be set");
}
if (indexStructure.activeShards().size() != 1) {
Objects.requireNonNull(env, "Environment not set");
return new LLLocalMultiLuceneIndex(env,
meterRegistry,
clusterName,
indexStructure.activeShards(),
indexStructure.totalShards(),
indicizerAnalyzers,
indicizerSimilarities,
luceneOptions,
luceneHacks,
rocksDBManager
);
} else {
return new LLLocalLuceneIndex(env,
meterRegistry,
clusterName,
indexStructure.activeShards().getInt(0),
indicizerAnalyzers,
indicizerSimilarities,
luceneOptions,
luceneHacks,
rocksDBManager
);
}
})
.transform(LuceneUtils::scheduleLucene);
if (clusterName == null) {
throw new IllegalArgumentException("Cluster name must be set");
}
if (indexStructure.activeShards().size() != 1) {
return new LLLocalMultiLuceneIndex(meterRegistry,
clusterName,
indexStructure.activeShards(),
indexStructure.totalShards(),
indicizerAnalyzers,
indicizerSimilarities,
luceneOptions,
luceneHacks
);
} else {
return new LLLocalLuceneIndex(meterRegistry,
clusterName,
indexStructure.activeShards().getInt(0),
indicizerAnalyzers,
indicizerSimilarities,
luceneOptions,
luceneHacks
);
}
}
@Override
public Mono<Void> disconnect() {
return Mono.<Void>fromCallable(() -> {
if (connected.compareAndSet(true, false)) {
var env = this.env.get();
if (env != null) {
env.close();
}
}
return null;
}).subscribeOn(Schedulers.boundedElastic());
public void disconnect() {
if (connected.compareAndSet(true, false)) {
}
}
}

1494
src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java
View File

File diff suppressed because it is too large Load Diff

16
src/main/java/it/cavallium/dbengine/database/disk/LLLocalEntryReactiveRocksIterator.java
View File

@ -1,31 +1,25 @@
package it.cavallium.dbengine.database.disk;
import io.netty5.buffer.Buffer;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.LLEntry;
import it.cavallium.dbengine.database.LLRange;
import java.util.function.Supplier;
import org.rocksdb.ReadOptions;
import reactor.core.publisher.Mono;
public class LLLocalEntryReactiveRocksIterator extends LLLocalReactiveRocksIterator<LLEntry> {
public LLLocalEntryReactiveRocksIterator(RocksDBColumn db,
Mono<LLRange> rangeMono,
boolean allowNettyDirect,
LLRange range,
Supplier<ReadOptions> readOptions,
boolean reverse,
boolean smallRange) {
super(db, rangeMono, allowNettyDirect, readOptions, true, reverse, smallRange);
super(db, range, readOptions, true, reverse, smallRange);
}
@Override
public LLEntry getEntry(Buffer key, Buffer value) {
public LLEntry getEntry(Buf key, Buf value) {
assert key != null;
assert value != null;
return LLEntry.of(
key.touch("iteration entry key"),
value.touch("iteration entry value")
);
return LLEntry.of(key.copy(), value.copy());
}
}

19
src/main/java/it/cavallium/dbengine/database/disk/LLLocalGroupedEntryReactiveRocksIterator.java
View File

@ -1,32 +1,25 @@
package it.cavallium.dbengine.database.disk;
import io.netty5.buffer.Buffer;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.LLEntry;
import it.cavallium.dbengine.database.LLRange;
import java.util.function.Supplier;
import org.rocksdb.ReadOptions;
import reactor.core.publisher.Mono;
public class LLLocalGroupedEntryReactiveRocksIterator extends
LLLocalGroupedReactiveRocksIterator<LLEntry> {
public class LLLocalGroupedEntryReactiveRocksIterator extends LLLocalGroupedReactiveRocksIterator<LLEntry> {
public LLLocalGroupedEntryReactiveRocksIterator(RocksDBColumn db,
int prefixLength,
Mono<LLRange> rangeMono,
boolean allowNettyDirect,
LLRange range,
Supplier<ReadOptions> readOptions,
boolean smallRange) {
super(db, prefixLength, rangeMono, allowNettyDirect, readOptions, false, true, smallRange);
super(db, prefixLength, range, readOptions, true, true, smallRange);
}
@Override
public LLEntry getEntry(Buffer key, Buffer value) {
public LLEntry getEntry(Buf key, Buf value) {
assert key != null;
assert value != null;
return LLEntry.of(
key.touch("iteration entry key"),
value.touch("iteration entry value")
);
return LLEntry.of(key.copy(), value.copy());
}
}

19
src/main/java/it/cavallium/dbengine/database/disk/LLLocalGroupedKeyReactiveRocksIterator.java
View File

@ -1,28 +1,23 @@
package it.cavallium.dbengine.database.disk;
import io.netty5.buffer.Buffer;
import io.netty5.util.Send;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.LLRange;
import java.util.function.Supplier;
import org.rocksdb.ReadOptions;
import reactor.core.publisher.Mono;
public class LLLocalGroupedKeyReactiveRocksIterator extends LLLocalGroupedReactiveRocksIterator<Buffer> {
public class LLLocalGroupedKeyReactiveRocksIterator extends LLLocalGroupedReactiveRocksIterator<Buf> {
public LLLocalGroupedKeyReactiveRocksIterator(RocksDBColumn db,
int prefixLength,
Mono<LLRange> rangeMono,
boolean allowNettyDirect,
LLRange range,
Supplier<ReadOptions> readOptions,
boolean smallRange) {
super(db, prefixLength, rangeMono, allowNettyDirect, readOptions, true, false, smallRange);
super(db, prefixLength, range, readOptions, true, false, smallRange);
}
@Override
public Buffer getEntry(Buffer key, Buffer value) {
if (value != null) {
value.close();
}
return key;
public Buf getEntry(Buf key, Buf value) {
assert key != null;
return key.copy();
}
}

Some files were not shown because too many files have changed in this diff Show More