2020-10-14 01:38:44 +02:00
|
|
|
package it.tdlight.tdlibsession.td.middle;
|
|
|
|
|
|
|
|
import com.hazelcast.config.Config;
|
2020-12-20 12:13:25 +01:00
|
|
|
import com.hazelcast.config.EvictionConfig;
|
2020-10-14 01:38:44 +02:00
|
|
|
import com.hazelcast.config.EvictionPolicy;
|
|
|
|
import com.hazelcast.config.MapConfig;
|
2020-12-20 12:13:25 +01:00
|
|
|
import com.hazelcast.config.MaxSizePolicy;
|
2020-10-14 01:38:44 +02:00
|
|
|
import com.hazelcast.config.MergePolicyConfig;
|
2020-12-20 12:13:25 +01:00
|
|
|
import com.hazelcast.config.cp.SemaphoreConfig;
|
2020-10-19 00:50:27 +02:00
|
|
|
import io.vertx.core.DeploymentOptions;
|
2020-10-14 01:38:44 +02:00
|
|
|
import io.vertx.core.Handler;
|
|
|
|
import io.vertx.core.VertxOptions;
|
|
|
|
import io.vertx.core.eventbus.DeliveryOptions;
|
|
|
|
import io.vertx.core.eventbus.MessageCodec;
|
|
|
|
import io.vertx.core.http.ClientAuth;
|
|
|
|
import io.vertx.core.net.JksOptions;
|
|
|
|
import io.vertx.core.spi.cluster.ClusterManager;
|
2021-01-22 17:31:09 +01:00
|
|
|
import io.vertx.reactivex.core.Vertx;
|
|
|
|
import io.vertx.reactivex.core.eventbus.EventBus;
|
|
|
|
import io.vertx.reactivex.core.eventbus.Message;
|
|
|
|
import io.vertx.reactivex.core.eventbus.MessageConsumer;
|
|
|
|
import io.vertx.reactivex.core.shareddata.SharedData;
|
2020-10-14 01:38:44 +02:00
|
|
|
import io.vertx.spi.cluster.hazelcast.HazelcastClusterManager;
|
2021-01-22 17:31:09 +01:00
|
|
|
import it.tdlight.common.ConstructorDetector;
|
|
|
|
import it.tdlight.tdlibsession.td.TdResultMessage;
|
2020-10-28 12:04:42 +01:00
|
|
|
import it.tdlight.utils.MonoUtils;
|
2020-10-14 01:38:44 +02:00
|
|
|
import java.nio.channels.AlreadyBoundException;
|
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.Collections;
|
|
|
|
import java.util.Map;
|
|
|
|
import java.util.Random;
|
|
|
|
import java.util.Set;
|
|
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
import java.util.concurrent.atomic.AtomicBoolean;
|
|
|
|
import org.jetbrains.annotations.Nullable;
|
|
|
|
import reactor.core.publisher.Mono;
|
2021-01-25 01:20:33 +01:00
|
|
|
import reactor.core.scheduler.Schedulers;
|
2020-10-14 01:38:44 +02:00
|
|
|
|
|
|
|
public class TdClusterManager {
|
|
|
|
|
|
|
|
private static final AtomicBoolean definedMasterCluster = new AtomicBoolean(false);
|
|
|
|
private static final AtomicBoolean definedNodesCluster = new AtomicBoolean(false);
|
|
|
|
private final ClusterManager mgr;
|
|
|
|
private final VertxOptions vertxOptions;
|
|
|
|
private final Vertx vertx;
|
|
|
|
|
2021-01-23 18:49:21 +01:00
|
|
|
@SuppressWarnings({"unchecked", "rawtypes"})
|
2021-01-13 17:22:14 +01:00
|
|
|
public TdClusterManager(ClusterManager mgr, VertxOptions vertxOptions, Vertx vertx) {
|
2020-10-14 01:38:44 +02:00
|
|
|
this.mgr = mgr;
|
|
|
|
this.vertxOptions = vertxOptions;
|
|
|
|
this.vertx = vertx;
|
2021-01-22 17:31:09 +01:00
|
|
|
|
|
|
|
if (vertx != null && vertx.eventBus() != null) {
|
|
|
|
vertx
|
|
|
|
.eventBus()
|
|
|
|
.getDelegate()
|
|
|
|
.registerDefaultCodec(TdResultList.class, new TdResultListMessageCodec())
|
|
|
|
.registerDefaultCodec(ExecuteObject.class, new TdExecuteObjectMessageCodec())
|
2021-01-23 18:49:21 +01:00
|
|
|
.registerDefaultCodec(TdResultMessage.class, new TdResultMessageCodec())
|
|
|
|
.registerDefaultCodec(StartSessionMessage.class, new StartSessionMessageCodec())
|
|
|
|
.registerDefaultCodec(EndSessionMessage.class, new EndSessionMessageCodec());
|
2021-01-22 17:31:09 +01:00
|
|
|
for (Class<?> value : ConstructorDetector.getTDConstructorsUnsafe().values()) {
|
|
|
|
vertx.eventBus().getDelegate().registerDefaultCodec(value, new TdMessageCodec(value));
|
|
|
|
}
|
|
|
|
}
|
2020-10-14 01:38:44 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
public static Mono<TdClusterManager> ofMaster(JksOptions keyStoreOptions, JksOptions trustStoreOptions, boolean onlyLocal, String masterHostname, String netInterface, int port, Set<String> nodesAddresses) {
|
|
|
|
if (definedMasterCluster.compareAndSet(false, true)) {
|
|
|
|
var vertxOptions = new VertxOptions();
|
|
|
|
netInterface = onlyLocal ? "127.0.0.1" : netInterface;
|
|
|
|
Config cfg;
|
|
|
|
if (!onlyLocal) {
|
|
|
|
cfg = new Config();
|
|
|
|
cfg.setInstanceName("Master");
|
|
|
|
} else {
|
|
|
|
cfg = null;
|
|
|
|
}
|
|
|
|
return of(cfg,
|
|
|
|
vertxOptions,
|
|
|
|
keyStoreOptions, trustStoreOptions, masterHostname, netInterface, port, nodesAddresses);
|
|
|
|
} else {
|
|
|
|
return Mono.error(new AlreadyBoundException());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
public static Mono<TdClusterManager> ofNodes(JksOptions keyStoreOptions, JksOptions trustStoreOptions, boolean onlyLocal, String masterHostname, String netInterface, int port, Set<String> nodesAddresses) {
|
2021-01-24 03:15:45 +01:00
|
|
|
return Mono.defer(() -> {
|
|
|
|
if (definedNodesCluster.compareAndSet(false, true)) {
|
|
|
|
var vertxOptions = new VertxOptions();
|
|
|
|
var netInterfaceF = onlyLocal ? "127.0.0.1" : netInterface;
|
|
|
|
Config cfg;
|
|
|
|
if (!onlyLocal) {
|
|
|
|
cfg = new Config();
|
|
|
|
cfg.setInstanceName("Node-" + new Random().nextLong());
|
|
|
|
} else {
|
|
|
|
cfg = null;
|
|
|
|
}
|
|
|
|
return of(cfg, vertxOptions, keyStoreOptions, trustStoreOptions, masterHostname, netInterfaceF, port, nodesAddresses);
|
2020-10-14 01:38:44 +02:00
|
|
|
} else {
|
2021-01-24 03:15:45 +01:00
|
|
|
return Mono.error(new AlreadyBoundException());
|
2020-10-14 01:38:44 +02:00
|
|
|
}
|
2021-01-24 03:15:45 +01:00
|
|
|
});
|
2020-10-14 01:38:44 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
public static Mono<TdClusterManager> of(@Nullable Config cfg,
|
|
|
|
VertxOptions vertxOptions,
|
|
|
|
JksOptions keyStoreOptions,
|
|
|
|
JksOptions trustStoreOptions,
|
|
|
|
String masterHostname,
|
|
|
|
String netInterface,
|
|
|
|
int port,
|
|
|
|
Set<String> nodesAddresses) {
|
|
|
|
ClusterManager mgr;
|
|
|
|
if (cfg != null) {
|
|
|
|
cfg.getNetworkConfig().setPortCount(1);
|
|
|
|
cfg.getNetworkConfig().setPort(port);
|
|
|
|
cfg.getNetworkConfig().setPortAutoIncrement(false);
|
|
|
|
cfg.getPartitionGroupConfig().setEnabled(false);
|
|
|
|
cfg.addMapConfig(new MapConfig()
|
|
|
|
.setName("__vertx.subs")
|
|
|
|
.setBackupCount(1)
|
|
|
|
.setTimeToLiveSeconds(0)
|
|
|
|
.setMaxIdleSeconds(0)
|
2020-12-20 12:13:25 +01:00
|
|
|
.setEvictionConfig(new EvictionConfig()
|
|
|
|
.setMaxSizePolicy(MaxSizePolicy.PER_NODE)
|
|
|
|
.setEvictionPolicy(EvictionPolicy.NONE)
|
|
|
|
.setSize(0))
|
2020-10-14 01:38:44 +02:00
|
|
|
.setMergePolicyConfig(new MergePolicyConfig().setPolicy("com.hazelcast.map.merge.LatestUpdateMapMergePolicy")));
|
2020-12-20 12:13:25 +01:00
|
|
|
cfg.getCPSubsystemConfig().setSemaphoreConfigs(Map.of("__vertx.*", new SemaphoreConfig().setInitialPermits(1)));
|
2020-10-14 01:38:44 +02:00
|
|
|
cfg.getNetworkConfig().getJoin().getMulticastConfig().setEnabled(false);
|
|
|
|
cfg.getNetworkConfig().getJoin().getAwsConfig().setEnabled(false);
|
|
|
|
cfg.getNetworkConfig().getJoin().getTcpIpConfig().setEnabled(true);
|
|
|
|
var addresses = new ArrayList<>(nodesAddresses);
|
|
|
|
cfg.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(addresses);
|
|
|
|
cfg.getNetworkConfig().getInterfaces().clear();
|
|
|
|
cfg.getNetworkConfig().getInterfaces().setInterfaces(Collections.singleton(netInterface)).setEnabled(true);
|
|
|
|
cfg.getNetworkConfig().setOutboundPorts(Collections.singleton(0));
|
|
|
|
|
|
|
|
cfg.setProperty("hazelcast.logging.type", "slf4j");
|
|
|
|
cfg.setProperty("hazelcast.wait.seconds.before.join", "0");
|
|
|
|
cfg.setProperty("hazelcast.tcp.join.port.try.count", "5");
|
|
|
|
cfg.setProperty("hazelcast.socket.bind.any", "false");
|
2020-12-20 12:13:25 +01:00
|
|
|
cfg.setClusterName("tdlib-session-container");
|
2020-10-14 01:38:44 +02:00
|
|
|
mgr = new HazelcastClusterManager(cfg);
|
|
|
|
vertxOptions.setClusterManager(mgr);
|
|
|
|
vertxOptions.getEventBusOptions().setConnectTimeout(120000);
|
|
|
|
//vertxOptions.getEventBusOptions().setIdleTimeout(60);
|
|
|
|
//vertxOptions.getEventBusOptions().setSsl(false);
|
|
|
|
|
|
|
|
vertxOptions.getEventBusOptions().setSslHandshakeTimeout(120000).setSslHandshakeTimeoutUnit(TimeUnit.MILLISECONDS);
|
|
|
|
vertxOptions.getEventBusOptions().setKeyStoreOptions(keyStoreOptions);
|
|
|
|
vertxOptions.getEventBusOptions().setTrustStoreOptions(trustStoreOptions);
|
|
|
|
vertxOptions.getEventBusOptions().setHost(masterHostname);
|
|
|
|
vertxOptions.getEventBusOptions().setPort(port + 1);
|
|
|
|
vertxOptions.getEventBusOptions().setSsl(true).setEnabledSecureTransportProtocols(Set.of("TLSv1.3", "TLSv1.2"));
|
|
|
|
vertxOptions.getEventBusOptions().setClientAuth(ClientAuth.REQUIRED);
|
|
|
|
} else {
|
|
|
|
mgr = null;
|
|
|
|
vertxOptions.setClusterManager(null);
|
|
|
|
}
|
|
|
|
|
2021-01-24 03:15:45 +01:00
|
|
|
vertxOptions.setPreferNativeTransport(true);
|
2021-01-24 19:13:46 +01:00
|
|
|
// check for blocked threads every 5s
|
|
|
|
vertxOptions.setBlockedThreadCheckInterval(5);
|
|
|
|
vertxOptions.setBlockedThreadCheckIntervalUnit(TimeUnit.SECONDS);
|
|
|
|
// warn if an event loop thread handler took more than 100ms to execute
|
|
|
|
vertxOptions.setMaxEventLoopExecuteTime(50);
|
|
|
|
vertxOptions.setMaxEventLoopExecuteTimeUnit(TimeUnit.MILLISECONDS);
|
|
|
|
// warn if an worker thread handler took more than 10s to execute
|
|
|
|
vertxOptions.setMaxWorkerExecuteTime(10);
|
|
|
|
vertxOptions.setMaxWorkerExecuteTimeUnit(TimeUnit.SECONDS);
|
|
|
|
// log the stack trace if an event loop or worker handler took more than 20s to execute
|
|
|
|
vertxOptions.setWarningExceptionTime(100);
|
|
|
|
vertxOptions.setWarningExceptionTimeUnit(TimeUnit.MILLISECONDS);
|
2021-01-24 03:15:45 +01:00
|
|
|
|
2020-10-14 01:38:44 +02:00
|
|
|
return Mono
|
|
|
|
.<Vertx>create(sink -> {
|
|
|
|
if (mgr != null) {
|
|
|
|
Vertx.clusteredVertx(vertxOptions, MonoUtils.toHandler(sink));
|
|
|
|
} else {
|
|
|
|
sink.success(Vertx.vertx(vertxOptions));
|
|
|
|
}
|
2021-01-25 02:31:17 +01:00
|
|
|
})
|
|
|
|
.publishOn(Schedulers.boundedElastic())
|
|
|
|
.flatMap(vertx -> Mono
|
2021-01-25 01:20:33 +01:00
|
|
|
.fromCallable(() -> new TdClusterManager(mgr, vertxOptions, vertx))
|
|
|
|
.subscribeOn(Schedulers.boundedElastic())
|
|
|
|
);
|
2020-10-14 01:38:44 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
public Vertx getVertx() {
|
|
|
|
return vertx;
|
|
|
|
}
|
|
|
|
|
|
|
|
public EventBus getEventBus() {
|
2021-01-13 17:22:14 +01:00
|
|
|
return vertx.eventBus();
|
2020-10-14 01:38:44 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
public VertxOptions getVertxOptions() {
|
|
|
|
return vertxOptions;
|
|
|
|
}
|
|
|
|
|
|
|
|
public DeliveryOptions newDeliveryOpts() {
|
|
|
|
return new DeliveryOptions().setSendTimeout(120000);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
*
|
|
|
|
* @param messageCodec
|
|
|
|
* @param <T>
|
|
|
|
* @return true if registered, false if already registered
|
|
|
|
*/
|
2021-01-22 17:31:09 +01:00
|
|
|
public <T> boolean registerCodec(MessageCodec<T, T> messageCodec) {
|
2020-10-14 01:38:44 +02:00
|
|
|
try {
|
2021-01-22 17:31:09 +01:00
|
|
|
vertx.eventBus().registerCodec(messageCodec);
|
2020-10-14 01:38:44 +02:00
|
|
|
return true;
|
|
|
|
} catch (IllegalStateException ex) {
|
|
|
|
if (ex.getMessage().startsWith("Already a default codec registered for class")) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (ex.getMessage().startsWith("Already a codec registered with name")) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
throw ex;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Create a message consumer against the specified address.
|
|
|
|
* <p>
|
|
|
|
* The returned consumer is not yet registered
|
|
|
|
* at the address, registration will be effective when {@link MessageConsumer#handler(io.vertx.core.Handler)}
|
|
|
|
* is called.
|
|
|
|
*
|
|
|
|
* @param address the address that it will register it at
|
|
|
|
* @param localOnly if you want to receive only local messages
|
|
|
|
* @return the event bus message consumer
|
|
|
|
*/
|
|
|
|
public <T> MessageConsumer<T> consumer(String address, boolean localOnly) {
|
|
|
|
if (localOnly) {
|
2021-01-13 17:22:14 +01:00
|
|
|
return vertx.eventBus().localConsumer(address);
|
2020-10-14 01:38:44 +02:00
|
|
|
} else {
|
2021-01-13 17:22:14 +01:00
|
|
|
return vertx.eventBus().consumer(address);
|
2020-10-14 01:38:44 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Create a consumer and register it against the specified address.
|
|
|
|
*
|
|
|
|
* @param address the address that will register it at
|
|
|
|
* @param localOnly if you want to receive only local messages
|
|
|
|
* @param handler the handler that will process the received messages
|
|
|
|
*
|
|
|
|
* @return the event bus message consumer
|
|
|
|
*/
|
|
|
|
public <T> MessageConsumer<T> consumer(String address, boolean localOnly, Handler<Message<T>> handler) {
|
|
|
|
if (localOnly) {
|
2021-01-13 17:22:14 +01:00
|
|
|
return vertx.eventBus().localConsumer(address, handler);
|
2020-10-14 01:38:44 +02:00
|
|
|
} else {
|
2021-01-13 17:22:14 +01:00
|
|
|
return vertx.eventBus().consumer(address, handler);
|
2020-10-14 01:38:44 +02:00
|
|
|
}
|
|
|
|
}
|
2020-10-19 00:50:27 +02:00
|
|
|
|
|
|
|
public DeploymentOptions newDeploymentOpts() {
|
|
|
|
return new DeploymentOptions().setWorkerPoolName("td-main-pool");
|
|
|
|
}
|
2020-10-28 12:04:42 +01:00
|
|
|
|
|
|
|
public SharedData getSharedData() {
|
|
|
|
return vertx.sharedData();
|
|
|
|
}
|
2020-10-14 01:38:44 +02:00
|
|
|
}
|