Store resulting events and events into a disk queue

This commit is contained in:
Andrea Cavalli 2022-10-10 01:05:53 +02:00
parent 59027e8e62
commit 708fcbd1e4
15 changed files with 362 additions and 30 deletions

35
pom.xml
View File

@ -74,6 +74,11 @@
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>it.cavallium</groupId>
<artifactId>filequeue</artifactId>
<version>3.0.0</version>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-tools</artifactId>
@ -138,22 +143,52 @@
<dependency>
<groupId>io.projectreactor.kafka</groupId>
<artifactId>reactor-kafka</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.rsocket</groupId>
<artifactId>rsocket-core</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.rsocket</groupId>
<artifactId>rsocket-load-balancer</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.rsocket</groupId>
<artifactId>rsocket-transport-local</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.rsocket</groupId>
<artifactId>rsocket-transport-netty</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>

View File

@ -21,17 +21,17 @@ import java.util.concurrent.CompletionException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.LockSupport;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import reactor.core.Disposable;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
public class AtomixReactiveApi implements ReactiveApi {
private static final Logger LOG = LoggerFactory.getLogger(AtomixReactiveApi.class);
private static final Logger LOG = LogManager.getLogger(AtomixReactiveApi.class);
private final AtomixReactiveApiMode mode;

View File

@ -29,9 +29,9 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import reactor.core.Disposable;
import reactor.core.publisher.Mono;
import reactor.core.publisher.MonoSink;
@ -42,7 +42,7 @@ import reactor.core.scheduler.Schedulers;
abstract class BaseAtomixReactiveApiClient implements ReactiveApiMultiClient {
private static final Logger LOG = LoggerFactory.getLogger(BaseAtomixReactiveApiClient.class);
private static final Logger LOG = LogManager.getLogger(BaseAtomixReactiveApiClient.class);
private static final long EMPTY_USER_ID = 0;
// Temporary id used to make requests

View File

@ -12,15 +12,15 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import net.minecrell.terminalconsole.SimpleTerminalConsole;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jline.reader.LineReader;
import org.jline.reader.LineReaderBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import reactor.core.scheduler.Schedulers;
public class Cli {
private static final Logger LOG = LoggerFactory.getLogger(Cli.class);
private static final Logger LOG = LogManager.getLogger(Cli.class);
private static final Object parameterLock = new Object();
private static boolean askedParameter = false;

View File

@ -0,0 +1,7 @@
package it.tdlight.reactiveapi;
import it.tdlight.reactiveapi.ResultingEvent.ClientBoundResultingEvent;
public class ClientBoundResultingEventSerializer implements Serializer<ClientBoundResultingEvent> {
}

View File

@ -11,13 +11,13 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashSet;
import java.util.Set;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Entrypoint {
private static final Logger LOG = LoggerFactory.getLogger(Entrypoint.class);
private static final Logger LOG = LogManager.getLogger(Entrypoint.class);
public record ValidEntrypointArgs(String clusterPath, String instancePath, String diskSessionsPath) {}

View File

@ -5,6 +5,7 @@ import static it.tdlight.reactiveapi.AuthPhase.LOGGED_OUT;
import static it.tdlight.reactiveapi.Event.SERIAL_VERSION;
import static java.util.Objects.requireNonNull;
import it.cavallium.filequeue.DiskQueueToConsumer;
import it.tdlight.common.Init;
import it.tdlight.common.ReactiveTelegramClient;
import it.tdlight.common.Response;
@ -36,41 +37,53 @@ import it.tdlight.reactiveapi.ResultingEvent.ClientBoundResultingEvent;
import it.tdlight.reactiveapi.ResultingEvent.ClusterBoundResultingEvent;
import it.tdlight.reactiveapi.ResultingEvent.ResultingEventPublisherClosed;
import it.tdlight.reactiveapi.ResultingEvent.TDLibBoundResultingEvent;
import it.tdlight.reactiveapi.rsocket.FileQueueUtils;
import it.tdlight.tdlight.ClientManager;
import java.io.ByteArrayOutputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.reactivestreams.Publisher;
import org.reactivestreams.Subscriber;
import org.reactivestreams.Subscription;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import reactor.core.CoreSubscriber;
import reactor.core.Disposable;
import reactor.core.publisher.BaseSubscriber;
import reactor.core.publisher.EmitterProcessor;
import reactor.core.publisher.Flux;
import reactor.core.publisher.FluxSink.OverflowStrategy;
import reactor.core.publisher.Mono;
import reactor.core.publisher.SignalType;
import reactor.core.publisher.Sinks.EmitFailureHandler;
import reactor.core.publisher.Sinks.Many;
import reactor.core.scheduler.Schedulers;
public abstract class ReactiveApiPublisher {
private static final Logger LOG = LoggerFactory.getLogger(ReactiveApiPublisher.class);
private static final Logger LOG = LogManager.getLogger(ReactiveApiPublisher.class);
private static final Duration SPECIAL_RAW_TIMEOUT_DURATION = Duration.ofMinutes(5);
private static final Duration HUNDRED_MS = Duration.ofMillis(100);
private final TdlibChannelsSharedHost sharedTdlibServers;
private final Set<ResultingEventTransformer> resultingEventTransformerSet;
private final ReactiveTelegramClient rawTelegramClient;
@ -100,18 +113,68 @@ public abstract class ReactiveApiPublisher {
throw new RuntimeException("Can't load TDLight", e);
}
this.telegramClient = Flux.<Signal>create(sink -> {
var path = this.path.get();
if (path == null) {
sink.error(new IllegalStateException("Path not set!"));
return;
}
DiskQueueToConsumer<Signal> queue;
try {
var queuePath = path.resolve(".queue");
if (Files.notExists(queuePath)) {
Files.createDirectories(queuePath);
}
queue = new DiskQueueToConsumer<>(queuePath.resolve("tdlib-events.tape2"),
FileQueueUtils.convert(SignalUtils.serializer()),
FileQueueUtils.convert(SignalUtils.deserializer()),
signal -> {
if (sink.requestedFromDownstream() > 0) {
if (signal != null) {
sink.next(signal);
}
return true;
} else {
return false;
}
}
);
} catch (Throwable ex) {
LOG.error("Failed to initialize queue {}", userId, ex);
sink.error(ex);
return;
}
try {
queue.startQueue();
} catch (Throwable ex) {
LOG.error("Failed to initialize queue {}", userId, ex);
sink.error(ex);
return;
}
try {
rawTelegramClient.createAndRegisterClient();
} catch (Throwable ex) {
LOG.error("Failed to initialize client {}", userId, ex);
sink.error(ex);
return;
}
rawTelegramClient.setListener(sink::next);
sink.onCancel(rawTelegramClient::cancel);
rawTelegramClient.setListener(value -> {
if (!sink.isCancelled()) {
queue.add(value);
}
});
sink.onDispose(() -> {
rawTelegramClient.dispose();
try {
queue.close();
} catch (Exception e) {
LOG.error("Unexpected error while closing the queue", e);
}
});
});
sink.onCancel(rawTelegramClient::cancel);
}, OverflowStrategy.ERROR).subscribeOn(Schedulers.boundedElastic());
}
public static ReactiveApiPublisher fromToken(TdlibChannelsSharedHost sharedTdlibServers,
@ -180,7 +243,7 @@ public abstract class ReactiveApiPublisher {
.doOnError(ex -> LOG.error("Failed to receive the response for special request {}\n"
+ " The instance will be closed", req.action(), ex))
.onErrorResume(ex -> Mono.just(new OnUpdateError(userId, new TdApi.Error(500, ex.getMessage()))))
, Integer.MAX_VALUE)
, Integer.MAX_VALUE, Integer.MAX_VALUE)
.doOnError(ex -> LOG.error("Failed to receive resulting events. The instance will be closed", ex))
.onErrorResume(ex -> Mono.just(new OnUpdateError(userId, new TdApi.Error(500, ex.getMessage()))))
@ -198,7 +261,56 @@ public abstract class ReactiveApiPublisher {
// Obtain only client-bound events
.filter(s -> s instanceof ClientBoundResultingEvent)
.cast(ClientBoundResultingEvent.class)
.map(ClientBoundResultingEvent::event);
.map(ClientBoundResultingEvent::event)
.transform(flux -> Flux.<ClientBoundEvent>create(sink -> {
try {
var queuePath = path.resolve(".queue");
if (Files.notExists(queuePath)) {
Files.createDirectories(queuePath);
}
var queue = new DiskQueueToConsumer<>(queuePath.resolve("client-bound-resulting-events.tape2"),
FileQueueUtils.convert(new ClientBoundEventSerializer()),
FileQueueUtils.convert(new ClientBoundEventDeserializer()),
signal -> {
if (sink.requestedFromDownstream() > 0) {
if (signal != null) {
sink.next(signal);
}
return true;
} else {
return false;
}
}
);
sink.onDispose(queue::close);
flux.subscribeOn(Schedulers.parallel()).subscribe(new CoreSubscriber<>() {
@Override
public void onSubscribe(@NotNull Subscription s) {
sink.onCancel(s::cancel);
s.request(Long.MAX_VALUE);
}
@Override
public void onNext(ClientBoundEvent clientBoundEvent) {
if (!sink.isCancelled()) {
queue.add(clientBoundEvent);
}
}
@Override
public void onError(Throwable throwable) {
sink.error(throwable);
}
@Override
public void onComplete() {
}
});
} catch (IOException ex) {
sink.error(ex);
}
}, OverflowStrategy.ERROR).subscribeOn(Schedulers.boundedElastic()))
.as(ReactorUtils::subscribeOnceUntilUnsubscribe);
sharedTdlibServers.events(lane, messagesToSend);

View File

@ -0,0 +1,76 @@
package it.tdlight.reactiveapi;
import it.tdlight.common.Signal;
import it.tdlight.jni.TdApi;
import it.unimi.dsi.fastutil.io.FastByteArrayInputStream;
import it.unimi.dsi.fastutil.io.FastByteArrayOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
public class SignalUtils {
private static final Serializer<Signal> SERIALIZER = new Serializer<>() {
@Override
public byte[] serialize(Signal data) {
return SignalUtils.serialize(data);
}
};
private static final Deserializer<Signal> DESERIALIZER = new Deserializer<>() {
@Override
public Signal deserialize(byte[] data) {
return SignalUtils.deserialize(data);
}
};
public static Serializer<Signal> serializer() {
return SERIALIZER;
}
public static Deserializer<Signal> deserializer() {
return DESERIALIZER;
}
public static Signal deserialize(byte[] bytes) {
var dis = new DataInputStream(new FastByteArrayInputStream(bytes));
try {
byte type = dis.readByte();
return switch (type) {
case 0 -> Signal.ofUpdate(TdApi.Deserializer.deserialize(dis));
case 1 -> Signal.ofUpdateException(new Exception(dis.readUTF()));
case 2 -> Signal.ofClosed();
default -> throw new IllegalStateException("Unexpected value: " + type);
};
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
public static byte[] serialize(Signal signal) {
var baos = new FastByteArrayOutputStream();
try (var daos = new DataOutputStream(baos)) {
if (signal.isUpdate()) {
daos.writeByte(0);
var up = signal.getUpdate();
up.serialize(daos);
} else if (signal.isException()) {
daos.writeByte(1);
var ex = signal.getException();
var exMsg = ex.getMessage();
daos.writeUTF(exMsg);
} else if (signal.isClosed()) {
daos.writeByte(2);
} else {
throw new IllegalStateException();
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
baos.trim();
return baos.array;
}
}

View File

@ -12,13 +12,13 @@ import io.soabase.recordbuilder.core.RecordBuilder;
import it.tdlight.common.Signal;
import it.tdlight.jni.TdApi;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@RecordBuilder
public record State(AuthPhase authPhase) implements StateBuilder.With {
private static final Logger LOG = LoggerFactory.getLogger(State.class);
private static final Logger LOG = LogManager.getLogger(State.class);
public State withSignal(Signal signal) {
var newState = this;

View File

@ -0,0 +1,39 @@
package it.tdlight.reactiveapi.rsocket;
import it.cavallium.filequeue.Deserializer;
import it.cavallium.filequeue.Serializer;
import it.tdlight.reactiveapi.ClientBoundEventDeserializer;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
public class FileQueueUtils {
public static <T> Serializer<T> convert(it.tdlight.reactiveapi.Serializer<T> serializer) {
return new Serializer<T>() {
@Override
public byte[] serialize(T data) throws IOException {
return serializer.serialize(data);
}
@Override
public void serialize(T data, DataOutput output) throws IOException {
serializer.serialize(data, output);
}
};
}
public static <T> Deserializer<T> convert(it.tdlight.reactiveapi.Deserializer<T> deserializer) {
return new Deserializer<T>() {
@Override
public T deserialize(byte[] data) throws IOException {
return deserializer.deserialize(data);
}
@Override
public T deserialize(int length, DataInput dataInput) throws IOException {
return deserializer.deserialize(length, dataInput);
}
};
}
}

View File

@ -41,7 +41,7 @@ public class MyRSocketClient implements RSocketChannelManager {
var transport = TcpClientTransport.create(baseHost.getHost(), baseHost.getPort());
this.nextClient = RSocketConnector.create()
//.setupPayload(DefaultPayload.create("client", "setup-info"))
.setupPayload(DefaultPayload.create("client", "setup-info"))
.payloadDecoder(PayloadDecoder.ZERO_COPY)
//.reconnect(retryStrategy)
.connect(transport)

View File

@ -16,6 +16,7 @@ import it.tdlight.reactiveapi.EventConsumer;
import it.tdlight.reactiveapi.EventProducer;
import it.tdlight.reactiveapi.Serializer;
import it.tdlight.reactiveapi.Timestamped;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
@ -39,7 +40,16 @@ public class MyRSocketServer implements RSocketChannelManager, RSocket {
public MyRSocketServer(HostAndPort baseHost) {
var serverMono = RSocketServer
.create(SocketAcceptor.with(this))
.create(new SocketAcceptor() {
@Override
public @NotNull Mono<RSocket> accept(@NotNull ConnectionSetupPayload setup, @NotNull RSocket sendingSocket) {
if (setup.getMetadataUtf8().equals("setup-info") && setup.getDataUtf8().equals("client")) {
return Mono.just(MyRSocketServer.this);
} else {
return Mono.error(new IOException("Invalid credentials"));
}
}
})
.payloadDecoder(PayloadDecoder.ZERO_COPY)
.bind(TcpServerTransport.create(baseHost.getHost(), baseHost.getPort()))
.doOnNext(d -> logger.debug("Server up"))

View File

@ -5,13 +5,12 @@ module tdlib.reactive.api {
exports it.tdlight.reactiveapi.kafka;
requires com.fasterxml.jackson.annotation;
requires org.jetbrains.annotations;
requires org.slf4j;
requires tdlight.java;
requires org.reactivestreams;
requires tdlight.api;
requires com.google.common;
requires java.logging;
requires static kafka.clients;
requires kafka.clients;
requires org.apache.logging.log4j;
requires reactor.kafka;
requires com.fasterxml.jackson.databind;
@ -28,4 +27,5 @@ module tdlib.reactive.api {
requires rsocket.transport.local;
requires rsocket.transport.netty;
requires io.netty.buffer;
requires filequeue;
}

View File

@ -0,0 +1,53 @@
package it.tdlight.reactiveapi.test;
import it.cavallium.filequeue.Deserializer;
import it.cavallium.filequeue.Serializer;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.time.Duration;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.LongAdder;
import org.apache.logging.log4j.core.util.FileUtils;
public class InfiniteQueueBench {
public static void main(String[] args) throws IOException {
var SECOND = Duration.ofSeconds(1).toNanos();
var tmpFile = Files.createTempFile("tmp", "");
long startTime = System.nanoTime();
LongAdder totalCount = new LongAdder();
AtomicLong internalQueueSize = new AtomicLong();
AtomicLong lastPrint = new AtomicLong();
AtomicInteger status = new AtomicInteger();
tmpFile.toFile().deleteOnExit();
Files.delete(tmpFile);
try (var queue = new it.cavallium.filequeue.DiskQueueToConsumer<String>(tmpFile, new Serializer<String>() {
@Override
public byte[] serialize(String data) throws IOException {
return data.getBytes(StandardCharsets.US_ASCII);
}
}, new Deserializer<String>() {
@Override
public String deserialize(byte[] data) throws IOException {
return new String(data, StandardCharsets.US_ASCII);
}
}, text -> {
var s = internalQueueSize.decrementAndGet();
var now = System.nanoTime();
if (lastPrint.updateAndGet(prev -> prev + SECOND <= now ? now : prev) == now) {
System.out.println(s + " currently queued elements, " + (totalCount.longValue() * SECOND / (now - startTime)) + " ops/s");
}
return status.incrementAndGet() % 10 == 0;
})) {
queue.startQueue();
final var text = "a".repeat(32);
while (true) {
internalQueueSize.incrementAndGet();
totalCount.increment();
queue.add(text);
}
}
}
}

View File

@ -1,7 +1,6 @@
module tdlib.reactive.api.test {
exports it.tdlight.reactiveapi.test;
requires org.apache.logging.log4j.core;
requires org.slf4j;
requires tdlib.reactive.api;
requires org.junit.jupiter.api;
requires reactor.core;
@ -19,4 +18,5 @@ module tdlib.reactive.api.test {
requires reactor.test;
requires reactor.netty.core;
requires org.apache.logging.log4j;
requires filequeue;
}