Restart jitter to avoid crashes

This commit is contained in:
Andrea Cavalli 2022-01-20 19:57:43 +01:00
parent 79bfd5d95c
commit f903035643
1 changed files with 22 additions and 2 deletions

View File

@ -17,6 +17,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import org.slf4j.Logger;
@ -30,6 +31,8 @@ public class PeriodicRestarter {
private static final Logger LOG = LoggerFactory.getLogger(PeriodicRestarter.class);
private static Duration JITTER_MAX_DELAY = Duration.ofMinutes(5);
private final ReactiveApi api;
private final Duration interval;
private final Set<Long> restartUserIds;
@ -123,10 +126,20 @@ public class PeriodicRestarter {
}
private void onSessionReady(long liveId, long userId) {
Duration maxRandomDelay;
if (JITTER_MAX_DELAY.compareTo(interval) < 0) {
maxRandomDelay = JITTER_MAX_DELAY;
} else {
maxRandomDelay = interval;
}
var randomDelay = randomTime(maxRandomDelay);
var totalDelay = interval.plus(randomDelay);
LOG.info("The session #IDU{} (liveId: {}) will be restarted at {}",
userId,
liveId,
Instant.now().plus(interval)
Instant.now().plus(totalDelay)
);
// Restart after x time
@ -153,12 +166,19 @@ public class PeriodicRestarter {
.subscribe();
}
}, interval.toMillis(), TimeUnit.MILLISECONDS);
}, totalDelay.toMillis(), TimeUnit.MILLISECONDS);
disposableRef.set(disposable);
var prev = closeManagedByPeriodicRestarter.put(liveId, disposable);
if (prev != null) prev.dispose();
}
/**
* @return random duration from 0 to n
*/
private Duration randomTime(Duration max) {
return Duration.ofMillis(ThreadLocalRandom.current().nextInt(0, Math.toIntExact(max.toMillis())));
}
public Mono<Void> stop() {
return Mono.fromRunnable(() -> {
LOG.info("Stopping periodic restarter...");