andreacavalli
/
CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Update dependencies

This commit is contained in:
Andrea Cavalli 2022-05-10 00:31:16 +02:00
parent 2c308f0f63
commit f89c3ff707
20 changed files with 970 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

254
pom.xml
View File

@ -14,8 +14,9 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<revision>0-SNAPSHOT</revision>
<dbengine.ci>false</dbengine.ci>
<micrometer.version>1.7.4</micrometer.version>
<micrometer.version>1.8.5</micrometer.version>
<lucene.version>9.1.0</lucene.version>
<junit.jupiter.version>5.8.2</junit.jupiter.version>
</properties>
<repositories>
<repository>
@ -82,162 +83,16 @@
</scm>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>31.1-jre</version>
</dependency>
<dependency>
<groupId>org.warp</groupId>
<artifactId>common-utils</artifactId>
<version>1.1.8</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty5-buffer</artifactId>
<version>5.0.0.Alpha1</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.30</version>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>8.5.8</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>5.8.2</version>
<exclusions>
<exclusion>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.8.2</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<version>5.8.2</version>
</dependency>
<!-- This will get hamcrest-core automatically -->
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
<version>2.2</version>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-join</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-common</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-icu</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-codecs</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queries</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-misc</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-facet</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-test-framework</artifactId>
<version>${lucene.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
<version>23.0.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.novasearch</groupId>
<artifactId>lucene-relevance</artifactId>
<version>9.0.0.0.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.soabase.record-builder</groupId>
<artifactId>record-builder-core</artifactId>
<version>33</version>
</dependency>
<dependency>
<groupId>it.cavallium</groupId>
<artifactId>data-generator-runtime</artifactId>
<version>1.0.63</version>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
<version>${micrometer.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-jmx</artifactId>
<version>${micrometer.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-bom</artifactId>
<version>2020.0.17</version>
<version>2020.0.18</version>
<type>pom</type>
<scope>import</scope>
</dependency>
@ -247,27 +102,28 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.warp</groupId>
<artifactId>common-utils</artifactId>
<version>31.1-jre</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty5-buffer</artifactId>
<version>5.0.0.Alpha1</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.30</version>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>8.5.8</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
<version>${junit.jupiter.version}</version>
<exclusions>
<exclusion>
<groupId>org.hamcrest</groupId>
@ -278,11 +134,13 @@
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>${junit.jupiter.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<version>${junit.jupiter.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@ -327,6 +185,7 @@
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<version>3.4.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.rocksdb</groupId>
@ -336,71 +195,78 @@
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-join</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-common</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-icu</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-codecs</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queries</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-misc</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-facet</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-test-framework</artifactId>
<version>${lucene.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor.addons</groupId>
<artifactId>reactor-extra</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-tools</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-test</artifactId>
<version>23.0.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty-core</artifactId>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-epoll</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-transport-classes-epoll</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.netty.incubator</groupId>
@ -430,10 +296,36 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.squareup.moshi</groupId>
<artifactId>moshi</artifactId>
<version>1.13.0</version>
<exclusions>
<exclusion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
</exclusion>
<exclusion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>dev.zacsweers.moshix</groupId>
<artifactId>moshi-records-reflect</artifactId>
<version>0.14.1</version>
<exclusions>
<exclusion>
<groupId>com.squareup.moshi</groupId>
<artifactId>moshi</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.projectreactor.netty.incubator</groupId>
<artifactId>reactor-netty-incubator-quic</artifactId>
<version>0.0.5</version>
<version>0.0.7</version>
<exclusions>
<exclusion>
<groupId>io.netty.incubator</groupId>
@ -468,15 +360,23 @@
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
<version>1.69</version>
<version>1.70</version>
</dependency>
<dependency>
<groupId>org.novasearch</groupId>
<artifactId>lucene-relevance</artifactId>
<version>9.0.0.0.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>it.cavallium</groupId>
<artifactId>data-generator-runtime</artifactId>
<version>1.0.66</version>
<exclusions>
<exclusion>
<groupId>org.jetbrains</groupId>
@ -487,6 +387,8 @@
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
<version>${micrometer.version}</version>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@ -497,6 +399,7 @@
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-jmx</artifactId>
<version>${micrometer.version}</version>
<optional>true</optional>
<exclusions>
<exclusion>
@ -508,8 +411,15 @@
<dependency>
<groupId>io.soabase.record-builder</groupId>
<artifactId>record-builder-core</artifactId>
<version>33</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<testSourceDirectory>src/test/java</testSourceDirectory>
@ -557,7 +467,7 @@
<plugin>
<groupId>it.cavallium</groupId>
<artifactId>data-generator</artifactId>
<version>0.9.126</version>
<version>0.9.131</version>
<executions>
<execution>
<id>generate-lucene-query-sources</id>
@ -593,7 +503,7 @@
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.8.2</version>
<version>${junit.jupiter.version}</version>
</dependency>
</dependencies>
<configuration>

16
src/main/java/it/cavallium/dbengine/client/query/QueryMoshi.java
View File

@ -15,18 +15,16 @@ import it.unimi.dsi.fastutil.objects.Object2ObjectMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectMaps;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.shorts.ShortList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.warp.commonutils.moshi.BooleanListJsonAdapter;
import org.warp.commonutils.moshi.ByteListJsonAdapter;
import org.warp.commonutils.moshi.CharListJsonAdapter;
import org.warp.commonutils.moshi.IntListJsonAdapter;
import org.warp.commonutils.moshi.LongListJsonAdapter;
import org.warp.commonutils.moshi.MoshiPolymorphic;
import org.warp.commonutils.moshi.ShortListJsonAdapter;
import it.cavallium.dbengine.utils.BooleanListJsonAdapter;
import it.cavallium.dbengine.utils.ByteListJsonAdapter;
import it.cavallium.dbengine.utils.CharListJsonAdapter;
import it.cavallium.dbengine.utils.IntListJsonAdapter;
import it.cavallium.dbengine.utils.LongListJsonAdapter;
import it.cavallium.dbengine.utils.MoshiPolymorphic;
import it.cavallium.dbengine.utils.ShortListJsonAdapter;
public class QueryMoshi extends MoshiPolymorphic<IType> {

2
src/main/java/it/cavallium/dbengine/database/disk/CachedIndexSearcherManager.java
View File

@ -25,7 +25,7 @@ import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.AlreadyClosedException;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.warp.commonutils.type.ShortNamedThreadFactory;
import it.cavallium.dbengine.utils.ShortNamedThreadFactory;
import reactor.core.publisher.Mono;
import reactor.core.publisher.Sinks;
import reactor.core.publisher.Sinks.Empty;

4
src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
View File

@ -87,7 +87,7 @@ import org.rocksdb.TxnDBWritePolicy;
import org.rocksdb.WALRecoveryMode;
import org.rocksdb.WriteBufferManager;
import org.rocksdb.util.SizeUnit;
import org.warp.commonutils.type.ShortNamedThreadFactory;
import it.cavallium.dbengine.utils.ShortNamedThreadFactory;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
@ -838,7 +838,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
options.setCreateIfMissing(true);
options.setSkipStatsUpdateOnDbOpen(true);
options.setCreateMissingColumnFamilies(true);
options.setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL);
options.setInfoLogLevel(InfoLogLevel.WARN_LEVEL);
options.setAvoidFlushDuringShutdown(false); // Flush all WALs during shutdown
options.setAvoidFlushDuringRecovery(true); // Flush all WALs during startup
options.setWalRecoveryMode(databaseOptions.absoluteConsistency()

2
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
View File

@ -70,7 +70,7 @@ import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.util.InfoStream;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.warp.commonutils.type.ShortNamedThreadFactory;
import it.cavallium.dbengine.utils.ShortNamedThreadFactory;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.publisher.SignalType;

1
src/main/java/it/cavallium/dbengine/database/disk/UnmodifiableReadOptions.java
View File

@ -5,7 +5,6 @@ import org.rocksdb.AbstractTableFilter;
import org.rocksdb.ReadOptions;
import org.rocksdb.ReadTier;
import org.rocksdb.Snapshot;
import org.warp.commonutils.range.UnmodifiableRange;
public class UnmodifiableReadOptions extends ReadOptions {

3
src/main/java/it/cavallium/dbengine/database/serialization/CodecSerializer.java
View File

@ -7,7 +7,6 @@ import java.io.IOError;
import java.io.IOException;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.warp.commonutils.error.IndexOutOfBoundsException;
public class CodecSerializer<A> implements Serializer<A> {
@ -33,7 +32,7 @@ public class CodecSerializer<A> implements Serializer<A> {
this.serializationCodecId = serializationCodecId;
this.microCodecs = microCodecs;
if (microCodecs && (serializationCodecId > 255 || serializationCodecId < 0)) {
throw new IndexOutOfBoundsException(serializationCodecId, 0, 255);
throw new IndexOutOfBoundsException(serializationCodecId);
}
if (serializedSizeHint != -1) {
this.serializedSizeHint = (microCodecs ? Byte.BYTES : Integer.BYTES) + serializedSizeHint;

2
src/main/java/it/cavallium/dbengine/lucene/RandomFieldComparator.java
View File

@ -8,7 +8,7 @@ import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.ScoreCachingWrappingScorer;
import org.jetbrains.annotations.NotNull;
import org.warp.commonutils.random.LFSR.LFSRIterator;
import it.cavallium.dbengine.utils.LFSR.LFSRIterator;
//todo: fix
public class RandomFieldComparator extends FieldComparator<Float> implements LeafFieldComparator {

2
src/main/java/it/cavallium/dbengine/lucene/RandomFieldComparatorSource.java
View File

@ -3,7 +3,7 @@ package it.cavallium.dbengine.lucene;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.FieldComparatorSource;
import org.warp.commonutils.random.LFSR;
import it.cavallium.dbengine.utils.LFSR;
public class RandomFieldComparatorSource extends FieldComparatorSource {

40
src/main/java/it/cavallium/dbengine/utils/BooleanListJsonAdapter.java Normal file
View File

@ -0,0 +1,40 @@
package it.cavallium.dbengine.utils;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
import it.unimi.dsi.fastutil.booleans.BooleanList;
import it.unimi.dsi.fastutil.booleans.BooleanLists;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import java.io.IOException;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public class BooleanListJsonAdapter extends JsonAdapter<BooleanList> {
@Override
public @NotNull BooleanList fromJson(@NotNull JsonReader reader) throws IOException {
reader.beginArray();
BooleanArrayList modifiableOutput = new BooleanArrayList();
while (reader.hasNext()) {
modifiableOutput.add(reader.nextBoolean());
}
reader.endArray();
return BooleanLists.unmodifiable(modifiableOutput);
}
@Override
public void toJson(@NotNull JsonWriter writer, @Nullable BooleanList value) throws IOException {
if (value == null) {
writer.nullValue();
return;
}
writer.beginArray();
for (int i = 0; i < value.size(); i++) {
writer.value(value.getBoolean(i));
}
writer.endArray();
}
}

39
src/main/java/it/cavallium/dbengine/utils/ByteListJsonAdapter.java Normal file
View File

@ -0,0 +1,39 @@
package it.cavallium.dbengine.utils;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.bytes.ByteList;
import it.unimi.dsi.fastutil.bytes.ByteLists;
import java.io.IOException;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public class ByteListJsonAdapter extends JsonAdapter<ByteList> {
@Override
public @NotNull ByteList fromJson(@NotNull JsonReader reader) throws IOException {
reader.beginArray();
ByteArrayList modifiableOutput = new ByteArrayList();
while (reader.hasNext()) {
modifiableOutput.add((byte) reader.nextInt());
}
reader.endArray();
return ByteLists.unmodifiable(modifiableOutput);
}
@Override
public void toJson(@NotNull JsonWriter writer, @Nullable ByteList value) throws IOException {
if (value == null) {
writer.nullValue();
return;
}
writer.beginArray();
for (int i = 0; i < value.size(); i++) {
writer.value((long) value.getByte(i));
}
writer.endArray();
}
}

39
src/main/java/it/cavallium/dbengine/utils/CharListJsonAdapter.java Normal file
View File

@ -0,0 +1,39 @@
package it.cavallium.dbengine.utils;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
import it.unimi.dsi.fastutil.chars.CharArrayList;
import it.unimi.dsi.fastutil.chars.CharList;
import it.unimi.dsi.fastutil.chars.CharLists;
import java.io.IOException;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public class CharListJsonAdapter extends JsonAdapter<CharList> {
@Override
public @NotNull CharList fromJson(@NotNull JsonReader reader) throws IOException {
reader.beginArray();
CharArrayList modifiableOutput = new CharArrayList();
while (reader.hasNext()) {
modifiableOutput.add((char) reader.nextInt());
}
reader.endArray();
return CharLists.unmodifiable(modifiableOutput);
}
@Override
public void toJson(@NotNull JsonWriter writer, @Nullable CharList value) throws IOException {
if (value == null) {
writer.nullValue();
return;
}
writer.beginArray();
for (int i = 0; i < value.size(); i++) {
writer.value((long) value.getChar(i));
}
writer.endArray();
}
}

39
src/main/java/it/cavallium/dbengine/utils/IntListJsonAdapter.java Normal file
View File

@ -0,0 +1,39 @@
package it.cavallium.dbengine.utils;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntList;
import it.unimi.dsi.fastutil.ints.IntLists;
import java.io.IOException;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public class IntListJsonAdapter extends JsonAdapter<IntList> {
@Override
public @NotNull IntList fromJson(@NotNull JsonReader reader) throws IOException {
reader.beginArray();
IntArrayList modifiableOutput = new IntArrayList();
while (reader.hasNext()) {
modifiableOutput.add(reader.nextInt());
}
reader.endArray();
return IntLists.unmodifiable(modifiableOutput);
}
@Override
public void toJson(@NotNull JsonWriter writer, @Nullable IntList value) throws IOException {
if (value == null) {
writer.nullValue();
return;
}
writer.beginArray();
for (int i = 0; i < value.size(); i++) {
writer.value((long) value.getInt(i));
}
writer.endArray();
}
}

137
src/main/java/it/cavallium/dbengine/utils/LFSR.java Normal file
View File

@ -0,0 +1,137 @@
package it.cavallium.dbengine.utils;
import java.math.BigInteger;
import java.util.Iterator;
import java.util.Random;
import org.jetbrains.annotations.NotNull;
/**
* Linear feedback shift register
* <p>
* Taps can be found at: See http://www.xilinx.com/support/documentation/application_notes/xapp052.pdf See
* http://mathoverflow.net/questions/46961/how-are-taps-proven-to-work-for-lfsrs/46983#46983 See
* http://www.newwaveinstruments.com/resources/articles/m_sequence_linear_feedback_shift_register_lfsr.htm See
* http://www.yikes.com/~ptolemy/lfsr_web/index.htm See http://seanerikoconnor.freeservers.com/Mathematics/AbstractAlgebra/PrimitivePolynomials/overview.html
*
* @author OldCurmudgeon
*/
public class LFSR implements Iterable<BigInteger> {
private static final Random random = new Random();
// Bit pattern for taps.
private final BigInteger taps;
// Where to start (and end).
private final BigInteger start;
public static LFSR randomInt() {
return random(32, random.nextInt());
}
public static LFSR randomLong() {
return random(64, random.nextLong());
}
public static LFSR randomPositiveLong() {
return random(50, Math.abs(random.nextInt()));
}
public static BigInteger randomPrimitive(int bitsSize) {
// Build the BigInteger.
BigInteger primitive = BigInteger.ZERO;
for (int bitNumber = 0; bitNumber <= bitsSize; bitNumber++) {
if (random.nextBoolean() || bitNumber == 0 || bitNumber == bitsSize) {
primitive = primitive.or(BigInteger.ONE.shiftLeft(bitNumber));
}
}
return primitive;
}
public static LFSR random(int bitsSize, long startNumber) {
return new LFSR(randomPrimitive(bitsSize), BigInteger.valueOf(startNumber));
}
// The poly must be primitive to span the full sequence.
public LFSR(BigInteger primitivePoly, BigInteger start) {
// Where to start from (and stop).
this.start = start.equals(BigInteger.ZERO) ? BigInteger.ONE : start;
// Knock off the 2^0 coefficient of the polynomial for the TAP.
this.taps = primitivePoly.shiftRight(1);
}
@NotNull
@Override
public LFSRIterator iterator() {
return new LFSRIterator(start);
}
public class LFSRIterator implements Iterator<BigInteger> {
// The last one we returned.
private BigInteger last = null;
// The next one to return.
private BigInteger next = null;
public LFSRIterator(BigInteger start) {
// Do not return the seed.
last = start;
}
@Override
public boolean hasNext() {
if (next == null) {
/*
* Uses the Galois form.
*
* Shift last right one.
*
* If the bit shifted out was a 1 - xor with the tap mask.
*/
boolean shiftedOutA1 = last.testBit(0);
// Shift right.
next = last.shiftRight(1);
if (shiftedOutA1) {
// Tap!
next = next.xor(taps);
}
// Never give them `start` again.
if (next.equals(start)) {
// Could set a finished flag here too.
next = null;
}
}
return next != null;
}
@Override
public BigInteger next() {
// Remember this one.
last = hasNext() ? next : null;
// Don't deliver it again.
next = null;
return last;
}
public BigInteger next(BigInteger last) {
this.last = last;
next = null;
return next();
}
@Override
public void remove() {
throw new UnsupportedOperationException("Not supported.");
}
@Override
public String toString() {
return LFSR.this.toString() + "[" + (last != null ? last.toString(16) : "") + "-" + (next != null ? next
.toString(16) : "") + "]";
}
}
@Override
public String toString() {
return "(" + taps.toString(32) + ")-" + start.toString(32);
}
}

39
src/main/java/it/cavallium/dbengine/utils/LongListJsonAdapter.java Normal file
View File

@ -0,0 +1,39 @@
package it.cavallium.dbengine.utils;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongList;
import it.unimi.dsi.fastutil.longs.LongLists;
import java.io.IOException;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public class LongListJsonAdapter extends JsonAdapter<LongList> {
@Override
public @NotNull LongList fromJson(@NotNull JsonReader reader) throws IOException {
reader.beginArray();
LongArrayList modifiableOutput = new LongArrayList();
while (reader.hasNext()) {
modifiableOutput.add(reader.nextLong());
}
reader.endArray();
return LongLists.unmodifiable(modifiableOutput);
}
@Override
public void toJson(@NotNull JsonWriter writer, @Nullable LongList value) throws IOException {
if (value == null) {
writer.nullValue();
return;
}
writer.beginArray();
for (int i = 0; i < value.size(); i++) {
writer.value(value.getLong(i));
}
writer.endArray();
}
}

372
src/main/java/it/cavallium/dbengine/utils/MoshiPolymorphic.java Normal file
View File

@ -0,0 +1,372 @@
package it.cavallium.dbengine.utils;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.JsonDataException;
import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonReader.Options;
import com.squareup.moshi.JsonWriter;
import com.squareup.moshi.Moshi;
import com.squareup.moshi.Types;
import dev.zacsweers.moshix.records.RecordsJsonAdapterFactory;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Modifier;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public abstract class MoshiPolymorphic<OBJ> {
public enum GetterStyle {
FIELDS,
RECORDS_GETTERS,
STANDARD_GETTERS
}
private final boolean instantiateUsingStaticOf;
private final GetterStyle getterStyle;
private boolean initialized = false;
private Moshi abstractMoshi;
private final Map<Type, JsonAdapter<OBJ>> abstractClassesSerializers = new ConcurrentHashMap<>();
private final Map<Type, JsonAdapter<List<OBJ>>> abstractListClassesSerializers = new ConcurrentHashMap<>();
private final Map<Type, JsonAdapter<OBJ>> concreteClassesSerializers = new ConcurrentHashMap<>();
private final Map<Type, JsonAdapter<List<OBJ>>> concreteListClassesSerializers = new ConcurrentHashMap<>();
private final Map<Type, JsonAdapter<?>> extraClassesSerializers = new ConcurrentHashMap<>();
private final Map<String, JsonAdapter<OBJ>> customAdapters = new ConcurrentHashMap<>();
public MoshiPolymorphic() {
this(false, GetterStyle.FIELDS);
}
public MoshiPolymorphic(boolean instantiateUsingStaticOf, GetterStyle getterStyle) {
this.instantiateUsingStaticOf = instantiateUsingStaticOf;
this.getterStyle = getterStyle;
}
private synchronized void initialize() {
if (!this.initialized) {
this.initialized = true;
var abstractMoshiBuilder = new Moshi.Builder();
var abstractClasses = getAbstractClasses();
var concreteClasses = getConcreteClasses();
var extraAdapters = getExtraAdapters();
extraAdapters.forEach((extraClass, jsonAdapter) -> {
extraClassesSerializers.put(extraClass, jsonAdapter);
abstractMoshiBuilder.add(extraClass, jsonAdapter);
});
for (Class<?> declaredClass : abstractClasses) {
var name = fixType(declaredClass.getSimpleName());
JsonAdapter<OBJ> adapter = new PolymorphicAdapter<>(name);
if (!extraClassesSerializers.containsKey(declaredClass)) {
abstractMoshiBuilder.add(declaredClass, adapter);
abstractClassesSerializers.put(declaredClass, adapter);
abstractListClassesSerializers.put(Types.newParameterizedType(List.class, declaredClass),
new ListValueAdapter<>(adapter)
);
}
customAdapters.put(name, adapter);
}
for (Class<?> declaredClass : concreteClasses) {
var name = fixType(declaredClass.getSimpleName());
JsonAdapter<OBJ> adapter = new NormalValueAdapter<>(name, declaredClass);
if (!extraClassesSerializers.containsKey(declaredClass)
&& !abstractClassesSerializers.containsKey(declaredClass)) {
concreteClassesSerializers.put(declaredClass, adapter);
concreteListClassesSerializers.put(Types.newParameterizedType(List.class, declaredClass),
new ListValueAdapter<>(adapter)
);
abstractMoshiBuilder.add(declaredClass, adapter);
}
customAdapters.put(name, adapter);
}
abstractMoshiBuilder.addLast(new RecordsJsonAdapterFactory());
abstractMoshi = abstractMoshiBuilder.build();
}
}
protected abstract Set<Class<OBJ>> getAbstractClasses();
protected abstract Set<Class<OBJ>> getConcreteClasses();
protected Map<Class<?>, JsonAdapter<?>> getExtraAdapters() {
return Map.of();
}
protected abstract boolean shouldIgnoreField(String fieldName);
public Moshi.Builder registerAdapters(Moshi.Builder moshiBuilder) {
initialize();
extraClassesSerializers.forEach(moshiBuilder::add);
abstractClassesSerializers.forEach(moshiBuilder::add);
abstractListClassesSerializers.forEach(moshiBuilder::add);
concreteClassesSerializers.forEach(moshiBuilder::add);
concreteListClassesSerializers.forEach(moshiBuilder::add);
return moshiBuilder;
}
private class PolymorphicAdapter<T> extends JsonAdapter<T> {
private final String adapterName;
private PolymorphicAdapter(String adapterName) {
this.adapterName = adapterName;
}
private final Options NAMES = Options.of("type", "properties");
@Nullable
@Override
public T fromJson(@NotNull JsonReader jsonReader) throws IOException {
String type = null;
jsonReader.beginObject();
iterate: while (jsonReader.hasNext()) {
switch (jsonReader.selectName(NAMES)) {
case 0:
type = fixType(jsonReader.nextString());
break;
case 1:
if (type == null) {
throw new JsonDataException("Type must be defined before properties");
}
break iterate;
default:
String name = jsonReader.nextName();
throw new JsonDataException("Key \"" + name + "\" is invalid");
}
}
JsonAdapter<? extends OBJ> propertiesAdapter = customAdapters.get(type);
if (propertiesAdapter == null) {
throw new JsonDataException("Type \"" + type + "\" is unknown");
}
//noinspection unchecked
var result = (T) propertiesAdapter.fromJson(jsonReader);
jsonReader.endObject();
return result;
}
@Override
public void toJson(@NotNull JsonWriter jsonWriter, @Nullable T t) throws IOException {
if (t == null) {
jsonWriter.nullValue();
} else {
String type = fixType(t.getClass().getSimpleName());
JsonAdapter<OBJ> propertiesAdapter = customAdapters.get(type);
if (propertiesAdapter == null) {
abstractMoshi.adapter(Object.class).toJson(jsonWriter, t);
} else {
jsonWriter.beginObject();
jsonWriter.name("type").value(type);
jsonWriter.name("properties");
//noinspection unchecked
propertiesAdapter.toJson(jsonWriter, (OBJ) t);
jsonWriter.endObject();
}
}
}
}
private class NormalValueAdapter<T> extends JsonAdapter<T> {
private final String adapterName;
private final Options names;
private final Class<?> declaredClass;
private final List<Field> declaredFields;
private final Function<T, Object>[] fieldGetters;
private NormalValueAdapter(String adapterName, Class<?> declaredClass) {
try {
this.adapterName = adapterName;
this.declaredClass = declaredClass;
this.declaredFields = Arrays
.stream(declaredClass.getDeclaredFields())
.filter(field -> {
var modifiers = field.getModifiers();
return !Modifier.isStatic(modifiers)
&& !Modifier.isTransient(modifiers)
&& !shouldIgnoreField(field.getName());
})
.collect(Collectors.toList());
String[] fieldNames = new String[this.declaredFields.size()];
//noinspection unchecked
this.fieldGetters = new Function[this.declaredFields.size()];
int i = 0;
for (Field declaredField : this.declaredFields) {
fieldNames[i] = declaredField.getName();
switch (getterStyle) {
case STANDARD_GETTERS:
var getterMethod = declaredField
.getDeclaringClass()
.getMethod("get" + StringUtils.capitalize(declaredField.getName()));
fieldGetters[i] = obj -> {
try {
return getterMethod.invoke(obj);
} catch (InvocationTargetException | IllegalAccessException e) {
throw new RuntimeException(e);
}
};
break;
case RECORDS_GETTERS:
var getterMethod2 = declaredField
.getDeclaringClass()
.getMethod(declaredField.getName());
fieldGetters[i] = obj -> {
try {
return getterMethod2.invoke(obj);
} catch (InvocationTargetException | IllegalAccessException e) {
throw new RuntimeException(e);
}
};
break;
case FIELDS:
fieldGetters[i] = t -> {
try {
return declaredField.get(t);
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
};
break;
}
i++;
} this.names = Options.of(fieldNames);
} catch (NoSuchMethodException e) {
throw new RuntimeException(e);
}
}
@Nullable
@Override
public T fromJson(@NotNull JsonReader jsonReader) throws IOException {
try {
Object instance;
Object[] fields;
if (instantiateUsingStaticOf) {
fields = new Object[declaredFields.size()];
instance = null;
} else {
fields = null;
instance = declaredClass.getConstructor().newInstance();
}
jsonReader.beginObject();
while (jsonReader.hasNext()) {
var nameId = jsonReader.selectName(names);
if (nameId >= 0 && nameId < this.declaredFields.size()) {
var fieldValue = abstractMoshi.adapter(declaredFields.get(nameId).getGenericType()).fromJson(jsonReader);
if (instantiateUsingStaticOf) {
fields[nameId] = fieldValue;
} else {
declaredFields.get(nameId).set(instance, fieldValue);
}
} else {
String keyName = jsonReader.nextName();
throw new JsonDataException("Key \"" + keyName + "\" is invalid");
}
}
jsonReader.endObject();
if (instantiateUsingStaticOf) {
Class[] params = new Class[declaredFields.size()];
for (int i = 0; i < declaredFields.size(); i++) {
params[i] = declaredFields.get(i).getType();
}
instance = declaredClass.getMethod("of", params).invoke(null, fields);
}
//noinspection unchecked
return (T) instance;
} catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
throw new JsonDataException(e);
}
}
@Override
public void toJson(@NotNull JsonWriter jsonWriter, @Nullable T t) throws IOException {
if (t == null) {
jsonWriter.nullValue();
} else {
jsonWriter.beginObject();
int i = 0;
for (Field declaredField : declaredFields) {
jsonWriter.name(declaredField.getName());
Class<?> fieldType = declaredField.getType();
if (abstractClassesSerializers.containsKey(fieldType)) {
//noinspection unchecked
abstractClassesSerializers.<OBJ>get(fieldType).toJson(jsonWriter, (OBJ) fieldGetters[i].apply(t));
} else if (concreteClassesSerializers.containsKey(fieldType)) {
//noinspection unchecked
concreteClassesSerializers.<OBJ>get(fieldType).toJson(jsonWriter, (OBJ) fieldGetters[i].apply(t));
} else {
abstractMoshi.<Object>adapter(fieldType).toJson(jsonWriter, fieldGetters[i].apply(t));
}
i++;
}
jsonWriter.endObject();
}
}
}
private static class ListValueAdapter<T> extends JsonAdapter<List<T>> {
private final JsonAdapter<T> valueAdapter;
public ListValueAdapter(JsonAdapter<T> valueAdapter) {
this.valueAdapter = valueAdapter;
}
@Nullable
@Override
public List<T> fromJson(@NotNull JsonReader jsonReader) throws IOException {
jsonReader.beginArray();
var result = new ArrayList<T>();
while (jsonReader.hasNext()) {
result.add(valueAdapter.fromJson(jsonReader));
}
jsonReader.endArray();
return Collections.unmodifiableList(result);
}
@Override
public void toJson(@NotNull JsonWriter jsonWriter, @Nullable List<T> ts) throws IOException {
if (ts == null) {
jsonWriter.nullValue();
} else {
jsonWriter.beginArray();
for (T value : ts) {
valueAdapter.toJson(jsonWriter.valueSink(), value);
}
jsonWriter.endArray();
}
}
}
private static String fixType(String nextString) {
if (nextString.length() > 512) {
throw new IllegalArgumentException("Input too long: " + nextString.length());
}
return UTFUtils.keepOnlyASCII(nextString);
}
}

39
src/main/java/it/cavallium/dbengine/utils/ShortListJsonAdapter.java Normal file
View File

@ -0,0 +1,39 @@
package it.cavallium.dbengine.utils;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
import it.unimi.dsi.fastutil.shorts.ShortList;
import it.unimi.dsi.fastutil.shorts.ShortLists;
import java.io.IOException;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public class ShortListJsonAdapter extends JsonAdapter<ShortList> {
@Override
public @NotNull ShortList fromJson(@NotNull JsonReader reader) throws IOException {
reader.beginArray();
ShortArrayList modifiableOutput = new ShortArrayList();
while (reader.hasNext()) {
modifiableOutput.add((short) reader.nextInt());
}
reader.endArray();
return ShortLists.unmodifiable(modifiableOutput);
}
@Override
public void toJson(@NotNull JsonWriter writer, @Nullable ShortList value) throws IOException {
if (value == null) {
writer.nullValue();
return;
}
writer.beginArray();
for (int i = 0; i < value.size(); i++) {
writer.value((long) value.getShort(i));
}
writer.endArray();
}
}

86
src/main/java/it/cavallium/dbengine/utils/ShortNamedThreadFactory.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package it.cavallium.dbengine.utils;
import java.util.Locale;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
import org.jetbrains.annotations.NotNull;
/**
* A default {@link ThreadFactory} implementation that accepts the name prefix
* of the created threads as a constructor argument. Otherwise, this factory
* yields the same semantics as the thread factory returned by
* {@link Executors#defaultThreadFactory()}.
*/
public class ShortNamedThreadFactory implements ThreadFactory {
private static int POOL_NUMBERS_COUNT = 50;
private static final AtomicInteger[] threadPoolNumber = new AtomicInteger[POOL_NUMBERS_COUNT];
static {
for (int i = 0; i < threadPoolNumber.length; i++) {
threadPoolNumber[i] = new AtomicInteger(1);
}
}
private ThreadGroup group;
private boolean daemon;
private final AtomicInteger threadNumber = new AtomicInteger(1);
private static final String NAME_PATTERN = "%s-%d";
private final String threadNamePrefix;
/**
* Creates a new {@link ShortNamedThreadFactory} instance
*
* @param threadNamePrefix the name prefix assigned to each thread created.
*/
public ShortNamedThreadFactory(String threadNamePrefix) {
group = Thread.currentThread().getThreadGroup();
this.threadNamePrefix = String.format(Locale.ROOT, NAME_PATTERN,
checkPrefix(threadNamePrefix), threadPoolNumber[(threadNamePrefix.hashCode() % POOL_NUMBERS_COUNT / 2) + POOL_NUMBERS_COUNT / 2].getAndIncrement());
}
public ShortNamedThreadFactory withGroup(ThreadGroup threadGroup) {
this.group = threadGroup;
return this;
}
public ShortNamedThreadFactory setDaemon(boolean daemon) {
this.daemon = daemon;
return this;
}
private static String checkPrefix(String prefix) {
return prefix == null || prefix.length() == 0 ? "Unnamed" : prefix;
}
/**
* Creates a new {@link Thread}
*
* @see ThreadFactory#newThread(Runnable)
*/
@Override
public Thread newThread(@NotNull Runnable r) {
final Thread t = new Thread(group, r, String.format(Locale.ROOT, "%s-%d",
this.threadNamePrefix, threadNumber.getAndIncrement()), 0);
t.setDaemon(daemon);
t.setPriority(Thread.NORM_PRIORITY);
return t;
}
}

42
src/main/java/it/cavallium/dbengine/utils/UTFUtils.java Normal file
View File

@ -0,0 +1,42 @@
package it.cavallium.dbengine.utils;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
public class UTFUtils {
public static void writeUTF(DataOutput out, String utf) throws IOException {
byte[] bytes = utf.getBytes(StandardCharsets.UTF_8);
out.writeInt(bytes.length);
out.write(bytes);
}
public static String readUTF(DataInput in) throws IOException {
int len = in.readInt();
byte[] data = new byte[len];
in.readFully(data, 0, len);
return new String(data, StandardCharsets.UTF_8);
}
/**
* Keep only ascii alphanumeric letters
*/
public static String keepOnlyASCII(String nextString) {
char[] chars = nextString.toCharArray();
//noinspection UnusedAssignment
nextString = null;
int writeIndex = 0;
char c;
for (int checkIndex = 0; checkIndex < chars.length; checkIndex++) {
c = chars[checkIndex];
if ((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')) {
if (writeIndex != checkIndex) {
chars[writeIndex] = c;
}
writeIndex++;
}
}
return new String(chars, 0, writeIndex);
}
}

3
src/main/java/module-info.java
View File

@ -17,6 +17,7 @@ module dbengine {
exports it.cavallium.dbengine.netty;
opens it.cavallium.dbengine.database.remote;
exports it.cavallium.dbengine;
exports it.cavallium.dbengine.utils;
requires org.jetbrains.annotations;
requires reactor.core;
requires com.google.common;
@ -29,7 +30,6 @@ module dbengine {
requires moshi;
requires io.netty5.common;
requires it.unimi.dsi.fastutil;
requires common.utils;
requires data.generator.runtime;
requires java.logging;
requires org.apache.lucene.core;
@ -54,4 +54,5 @@ module dbengine {
requires org.apache.lucene.queryparser;
requires reactor.netty.incubator.quic;
requires okio;
requires moshi.records.reflect;
}