Restart jitter to avoid crashes

This commit is contained in:
Andrea Cavalli 2022-01-20 19:57:43 +01:00
parent 79bfd5d95c
commit f903035643

View File

@ -17,6 +17,7 @@ import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -30,6 +31,8 @@ public class PeriodicRestarter {
private static final Logger LOG = LoggerFactory.getLogger(PeriodicRestarter.class); private static final Logger LOG = LoggerFactory.getLogger(PeriodicRestarter.class);
private static Duration JITTER_MAX_DELAY = Duration.ofMinutes(5);
private final ReactiveApi api; private final ReactiveApi api;
private final Duration interval; private final Duration interval;
private final Set<Long> restartUserIds; private final Set<Long> restartUserIds;
@ -123,10 +126,20 @@ public class PeriodicRestarter {
} }
private void onSessionReady(long liveId, long userId) { private void onSessionReady(long liveId, long userId) {
Duration maxRandomDelay;
if (JITTER_MAX_DELAY.compareTo(interval) < 0) {
maxRandomDelay = JITTER_MAX_DELAY;
} else {
maxRandomDelay = interval;
}
var randomDelay = randomTime(maxRandomDelay);
var totalDelay = interval.plus(randomDelay);
LOG.info("The session #IDU{} (liveId: {}) will be restarted at {}", LOG.info("The session #IDU{} (liveId: {}) will be restarted at {}",
userId, userId,
liveId, liveId,
Instant.now().plus(interval) Instant.now().plus(totalDelay)
); );
// Restart after x time // Restart after x time
@ -153,12 +166,19 @@ public class PeriodicRestarter {
.subscribe(); .subscribe();
} }
}, interval.toMillis(), TimeUnit.MILLISECONDS); }, totalDelay.toMillis(), TimeUnit.MILLISECONDS);
disposableRef.set(disposable); disposableRef.set(disposable);
var prev = closeManagedByPeriodicRestarter.put(liveId, disposable); var prev = closeManagedByPeriodicRestarter.put(liveId, disposable);
if (prev != null) prev.dispose(); if (prev != null) prev.dispose();
} }
/**
* @return random duration from 0 to n
*/
private Duration randomTime(Duration max) {
return Duration.ofMillis(ThreadLocalRandom.current().nextInt(0, Math.toIntExact(max.toMillis())));
}
public Mono<Void> stop() { public Mono<Void> stop() {
return Mono.fromRunnable(() -> { return Mono.fromRunnable(() -> {
LOG.info("Stopping periodic restarter..."); LOG.info("Stopping periodic restarter...");