2012-06-04 22:31:44 +02:00
|
|
|
/*
|
|
|
|
* Copyright 2012 The Netty Project
|
|
|
|
*
|
|
|
|
* The Netty Project licenses this file to you under the Apache License,
|
|
|
|
* version 2.0 (the "License"); you may not use this file except in compliance
|
|
|
|
* with the License. You may obtain a copy of the License at:
|
|
|
|
*
|
2020-10-23 14:44:18 +02:00
|
|
|
* https://www.apache.org/licenses/LICENSE-2.0
|
2012-06-04 22:31:44 +02:00
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
|
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
|
|
* License for the specific language governing permissions and limitations
|
|
|
|
* under the License.
|
|
|
|
*/
|
2013-03-05 21:41:19 +01:00
|
|
|
package io.netty.util.concurrent;
|
2012-06-02 02:51:19 +02:00
|
|
|
|
2019-01-23 08:32:05 +01:00
|
|
|
import io.netty.util.internal.EmptyArrays;
|
|
|
|
|
|
|
|
import java.util.Arrays;
|
2012-12-14 03:43:04 +01:00
|
|
|
import java.util.Collections;
|
2013-04-09 07:15:49 +02:00
|
|
|
import java.util.Iterator;
|
2019-01-23 08:32:05 +01:00
|
|
|
import java.util.List;
|
2013-08-20 18:42:56 +02:00
|
|
|
import java.util.concurrent.Executor;
|
2012-06-02 02:51:19 +02:00
|
|
|
import java.util.concurrent.ThreadFactory;
|
|
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
import java.util.concurrent.atomic.AtomicInteger;
|
2020-08-12 11:22:24 +02:00
|
|
|
import java.util.concurrent.atomic.AtomicLong;
|
2012-06-02 02:51:19 +02:00
|
|
|
|
2021-04-22 08:03:24 +02:00
|
|
|
import static io.netty.util.internal.ObjectUtil.checkPositive;
|
|
|
|
|
2012-12-18 07:23:42 +01:00
|
|
|
/**
|
2019-01-23 08:32:05 +01:00
|
|
|
* {@link EventExecutorGroup} implementation that handles their tasks with multiple threads at
|
2012-12-18 07:23:42 +01:00
|
|
|
* the same time.
|
|
|
|
*/
|
2021-09-02 20:12:56 +02:00
|
|
|
public class MultithreadEventExecutorGroup implements EventExecutorGroup {
|
2012-06-02 02:51:19 +02:00
|
|
|
|
|
|
|
private final EventExecutor[] children;
|
2019-01-23 08:32:05 +01:00
|
|
|
private final List<EventExecutor> readonlyChildren;
|
2013-06-12 01:00:54 +02:00
|
|
|
private final AtomicInteger terminatedChildren = new AtomicInteger();
|
2021-09-02 20:12:56 +02:00
|
|
|
private final Promise<?> terminationFuture = GlobalEventExecutor.INSTANCE.newPromise();
|
2019-01-23 08:32:05 +01:00
|
|
|
private final boolean powerOfTwo;
|
2012-06-02 02:51:19 +02:00
|
|
|
|
2012-12-18 07:23:42 +01:00
|
|
|
/**
|
2012-12-18 15:27:52 +01:00
|
|
|
* Create a new instance.
|
2012-12-18 07:23:42 +01:00
|
|
|
*
|
2013-04-03 10:15:25 +02:00
|
|
|
* @param nThreads the number of threads that will be used by this instance.
|
2012-12-18 07:23:42 +01:00
|
|
|
* @param threadFactory the ThreadFactory to use, or {@code null} if the default should be used.
|
|
|
|
*/
|
2019-01-23 08:32:05 +01:00
|
|
|
public MultithreadEventExecutorGroup(int nThreads, ThreadFactory threadFactory) {
|
|
|
|
this(nThreads, threadFactory, SingleThreadEventExecutor.DEFAULT_MAX_PENDING_EXECUTOR_TASKS,
|
|
|
|
RejectedExecutionHandlers.reject());
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Create a new instance.
|
|
|
|
*
|
|
|
|
* @param nThreads the number of threads that will be used by this instance.
|
|
|
|
* @param executor the {@link Executor} to use, or {@code null} if the default should be used.
|
|
|
|
*/
|
|
|
|
public MultithreadEventExecutorGroup(int nThreads, Executor executor) {
|
|
|
|
this(nThreads, executor, SingleThreadEventExecutor.DEFAULT_MAX_PENDING_EXECUTOR_TASKS,
|
|
|
|
RejectedExecutionHandlers.reject());
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Create a new instance.
|
|
|
|
*
|
|
|
|
* @param nThreads the number of threads that will be used by this instance.
|
|
|
|
* @param threadFactory the {@link ThreadFactory} to use, or {@code null} if the default should be used.
|
|
|
|
* @param maxPendingTasks the maximum number of pending tasks before new tasks will be rejected.
|
|
|
|
* @param rejectedHandler the {@link RejectedExecutionHandler} to use.
|
|
|
|
*/
|
|
|
|
public MultithreadEventExecutorGroup(int nThreads, ThreadFactory threadFactory,
|
|
|
|
int maxPendingTasks, RejectedExecutionHandler rejectedHandler) {
|
|
|
|
this(nThreads, threadFactory, maxPendingTasks, rejectedHandler, EmptyArrays.EMPTY_OBJECTS);
|
2013-08-20 18:42:56 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Create a new instance.
|
|
|
|
*
|
|
|
|
* @param nThreads the number of threads that will be used by this instance.
|
|
|
|
* @param executor the Executor to use, or {@code null} if the default should be used.
|
2019-01-23 08:32:05 +01:00
|
|
|
* @param maxPendingTasks the maximum number of pending tasks before new tasks will be rejected.
|
|
|
|
* @param rejectedHandler the {@link RejectedExecutionHandler} to use.
|
2013-08-20 18:42:56 +02:00
|
|
|
*/
|
2019-01-23 08:32:05 +01:00
|
|
|
public MultithreadEventExecutorGroup(int nThreads, Executor executor,
|
|
|
|
int maxPendingTasks, RejectedExecutionHandler rejectedHandler) {
|
|
|
|
this(nThreads, executor, maxPendingTasks, rejectedHandler, EmptyArrays.EMPTY_OBJECTS);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Create a new instance.
|
|
|
|
*
|
|
|
|
* @param nThreads the number of threads that will be used by this instance.
|
|
|
|
* @param threadFactory the ThreadFactory to use, or {@code null} if the default should be used.
|
|
|
|
* @param maxPendingTasks the maximum number of pending tasks before new tasks will be rejected.
|
|
|
|
* @param rejectedHandler the {@link RejectedExecutionHandler} to use.
|
|
|
|
* @param args arguments which will passed to each {@link #newChild(Executor, int,
|
|
|
|
* RejectedExecutionHandler, Object...)} call
|
|
|
|
*/
|
|
|
|
protected MultithreadEventExecutorGroup(int nThreads, ThreadFactory threadFactory, int maxPendingTasks,
|
|
|
|
RejectedExecutionHandler rejectedHandler, Object... args) {
|
|
|
|
this(nThreads, threadFactory == null ? null : new ThreadPerTaskExecutor(threadFactory),
|
|
|
|
maxPendingTasks, rejectedHandler, args);
|
2016-04-12 22:00:38 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Create a new instance.
|
|
|
|
*
|
|
|
|
* @param nThreads the number of threads that will be used by this instance.
|
|
|
|
* @param executor the Executor to use, or {@code null} if the default should be used.
|
2019-01-23 08:32:05 +01:00
|
|
|
* @param maxPendingTasks the maximum number of pending tasks before new tasks will be rejected.
|
|
|
|
* @param rejectedHandler the {@link RejectedExecutionHandler} to use.
|
|
|
|
* @param args arguments which will passed to each {@link #newChild(Executor, int,
|
|
|
|
* RejectedExecutionHandler, Object...)} call
|
2016-04-12 22:00:38 +02:00
|
|
|
*/
|
2019-01-23 08:32:05 +01:00
|
|
|
protected MultithreadEventExecutorGroup(int nThreads, Executor executor, int maxPendingTasks,
|
|
|
|
RejectedExecutionHandler rejectedHandler, Object... args) {
|
2021-04-22 08:03:24 +02:00
|
|
|
checkPositive(nThreads, "nThreads");
|
2012-06-02 07:57:54 +02:00
|
|
|
|
2013-08-20 18:42:56 +02:00
|
|
|
if (executor == null) {
|
2019-01-23 08:32:05 +01:00
|
|
|
executor = new ThreadPerTaskExecutor(new DefaultThreadFactory(getClass()));
|
2012-06-02 02:51:19 +02:00
|
|
|
}
|
|
|
|
|
2014-03-24 04:32:55 +01:00
|
|
|
children = new EventExecutor[nThreads];
|
2019-01-23 08:32:05 +01:00
|
|
|
powerOfTwo = isPowerOfTwo(children.length);
|
2012-06-02 02:51:19 +02:00
|
|
|
for (int i = 0; i < nThreads; i ++) {
|
|
|
|
boolean success = false;
|
|
|
|
try {
|
2019-01-23 08:32:05 +01:00
|
|
|
children[i] = newChild(executor, maxPendingTasks, rejectedHandler, args);
|
2012-06-02 02:51:19 +02:00
|
|
|
success = true;
|
|
|
|
} catch (Exception e) {
|
2013-03-05 21:41:19 +01:00
|
|
|
// TODO: Think about if this is a good exception type
|
2019-01-23 08:32:05 +01:00
|
|
|
throw new IllegalStateException("failed to create a child event executor", e);
|
2012-06-02 02:51:19 +02:00
|
|
|
} finally {
|
|
|
|
if (!success) {
|
|
|
|
for (int j = 0; j < i; j ++) {
|
2013-04-27 09:39:19 +02:00
|
|
|
children[j].shutdownGracefully();
|
|
|
|
}
|
|
|
|
|
|
|
|
for (int j = 0; j < i; j ++) {
|
|
|
|
EventExecutor e = children[j];
|
|
|
|
try {
|
|
|
|
while (!e.isTerminated()) {
|
|
|
|
e.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS);
|
|
|
|
}
|
|
|
|
} catch (InterruptedException interrupted) {
|
2014-03-24 04:32:55 +01:00
|
|
|
// Let the caller handle the interruption.
|
2013-04-27 09:39:19 +02:00
|
|
|
Thread.currentThread().interrupt();
|
|
|
|
break;
|
|
|
|
}
|
2012-06-02 02:51:19 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2013-06-12 01:00:54 +02:00
|
|
|
|
2019-01-29 14:06:05 +01:00
|
|
|
final FutureListener<Object> terminationListener = future -> {
|
|
|
|
if (terminatedChildren.incrementAndGet() == children.length) {
|
|
|
|
terminationFuture.setSuccess(null);
|
2013-06-12 01:00:54 +02:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
for (EventExecutor e: children) {
|
|
|
|
e.terminationFuture().addListener(terminationListener);
|
|
|
|
}
|
2019-01-23 08:32:05 +01:00
|
|
|
readonlyChildren = Collections.unmodifiableList(Arrays.asList(children));
|
2012-06-02 02:51:19 +02:00
|
|
|
}
|
|
|
|
|
2020-08-12 11:22:24 +02:00
|
|
|
// Use a 'long' counter to avoid non-round-robin behaviour at the 32-bit overflow boundary.
|
|
|
|
// The 64-bit long solves this by placing the overflow so far into the future, that no system
|
|
|
|
// will encounter this in practice.
|
|
|
|
private final AtomicLong idx = new AtomicLong();
|
2019-01-23 08:32:05 +01:00
|
|
|
|
|
|
|
/**
|
|
|
|
* The {@link EventExecutor}s that are used by this {@link MultithreadEventExecutorGroup}.
|
|
|
|
*/
|
|
|
|
protected final List<EventExecutor> executors() {
|
|
|
|
return readonlyChildren;
|
2013-04-03 10:49:30 +02:00
|
|
|
}
|
|
|
|
|
2019-01-23 08:32:05 +01:00
|
|
|
/**
|
|
|
|
* Returns the next {@link EventExecutor} to use. The default implementation will use round-robin, but you may
|
|
|
|
* override this to change the selection algorithm.
|
|
|
|
*/
|
2012-06-02 02:51:19 +02:00
|
|
|
@Override
|
2012-08-10 13:17:18 +02:00
|
|
|
public EventExecutor next() {
|
2019-01-23 08:32:05 +01:00
|
|
|
if (powerOfTwo) {
|
2020-08-12 11:22:24 +02:00
|
|
|
return children[(int) idx.getAndIncrement() & children.length - 1];
|
2019-01-23 08:32:05 +01:00
|
|
|
}
|
2020-08-12 11:22:24 +02:00
|
|
|
return children[(int) Math.abs(idx.getAndIncrement() % children.length)];
|
2019-01-23 08:32:05 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
private static boolean isPowerOfTwo(int val) {
|
|
|
|
return (val & -val) == val;
|
2012-06-02 02:51:19 +02:00
|
|
|
}
|
|
|
|
|
2013-04-09 07:15:49 +02:00
|
|
|
@Override
|
|
|
|
public Iterator<EventExecutor> iterator() {
|
2019-01-23 08:32:05 +01:00
|
|
|
return executors().iterator();
|
2013-04-09 07:15:49 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Return the number of {@link EventExecutor} this implementation uses. This number is the maps
|
|
|
|
* 1:1 to the threads it use.
|
|
|
|
*/
|
|
|
|
public final int executorCount() {
|
2019-01-23 08:32:05 +01:00
|
|
|
return executors().size();
|
2013-04-09 07:15:49 +02:00
|
|
|
}
|
|
|
|
|
2012-12-18 07:23:42 +01:00
|
|
|
/**
|
2019-01-23 08:32:05 +01:00
|
|
|
* Create a new EventExecutor which will later then accessible via the {@link #next()} method. This method will be
|
2012-12-18 07:23:42 +01:00
|
|
|
* called for each thread that will serve this {@link MultithreadEventExecutorGroup}.
|
|
|
|
*
|
2019-01-23 08:32:05 +01:00
|
|
|
* As this method is called from within the constructor you can only use the parameters passed into the method when
|
|
|
|
* overriding this method.
|
2012-12-18 07:23:42 +01:00
|
|
|
*/
|
2019-01-23 08:32:05 +01:00
|
|
|
protected EventExecutor newChild(Executor executor, int maxPendingTasks,
|
|
|
|
RejectedExecutionHandler rejectedExecutionHandler,
|
|
|
|
Object... args) {
|
|
|
|
assert args.length == 0;
|
|
|
|
return new SingleThreadEventExecutor(executor, maxPendingTasks, rejectedExecutionHandler);
|
|
|
|
}
|
2012-08-10 13:17:18 +02:00
|
|
|
|
2012-06-02 02:51:19 +02:00
|
|
|
@Override
|
2019-01-23 08:32:05 +01:00
|
|
|
public final Future<?> shutdownGracefully(long quietPeriod, long timeout, TimeUnit unit) {
|
2013-04-27 09:39:19 +02:00
|
|
|
for (EventExecutor l: children) {
|
|
|
|
l.shutdownGracefully(quietPeriod, timeout, unit);
|
Reduce the chance of RejectedExecutionException
When a Netty application shuts down, a user often sees a REE
(RejectedExecutionException).
A REE is raised due to various reasons we don't have control over, such
as:
- A client connects to a server while the server is shutting down.
- An event is triggered for a closed Channel while its event loop is
also shutting down. Some of them are:
- channelDeregistered (triggered after a channel is closed)
- freeIn/OutboundBuffer (triggered after channelDeregistered)
- userEventTriggered (triggered anytime)
To address this issue, a new method called confirmShutdown() has been
added to SingleThreadEventExecutor. After a user calls shutdown(),
confirmShutdown() runs any remaining tasks in the task queue and ensures
no events are triggered for last 2 seconds. If any task are added to
the task queue before 2 seconds passes, confirmShutdown() prevents the
event loop from terminating by returning false.
Now that SingleThreadEventExecutor needs to accept tasks even after
shutdown(), its execute() method only rejects the task after the event
loop is terminated (i.e. isTerminated() returns true.) Except that,
there's no change in semantics.
SingleThreadEventExecutor also checks if its subclass called
confirmShutdown() in its run() implementation, so that Netty developers
can make sure they shut down their event loop impementation correctly.
It also fixes a bug in AioSocketChannel, revealed by delayed shutdown,
where an inboundBufferUpdated() event is triggered on a closed Channel
with deallocated buffers.
Caveats:
Because SingleThreadEventExecutor.takeTask() does not have a notion of
timeout, confirmShutdown() adds a dummy task (WAKEUP_TASK) to wake up
takeTask() immediately and instead sleeps hard-coded 100ms. I'll
address this issue later by modifying takeTask() times out dynamically.
Miscellaneous changes:
SingleThreadEventExecutor.wakeup() now has the default implementation.
Instead of interrupting the current thread, it simply adds a dummy task
(WAKEUP_TASK) to the task queue, which is more elegant and efficient.
NioEventLoop is the only implementation that overrides it. All other
implementations' wakeup()s were removed thanks to this change.
2012-11-22 12:45:49 +01:00
|
|
|
}
|
2013-06-12 01:00:54 +02:00
|
|
|
return terminationFuture();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2019-01-23 08:32:05 +01:00
|
|
|
public final Future<?> terminationFuture() {
|
2021-09-02 10:46:54 +02:00
|
|
|
return terminationFuture.asFuture();
|
2013-04-27 09:39:19 +02:00
|
|
|
}
|
Reduce the chance of RejectedExecutionException
When a Netty application shuts down, a user often sees a REE
(RejectedExecutionException).
A REE is raised due to various reasons we don't have control over, such
as:
- A client connects to a server while the server is shutting down.
- An event is triggered for a closed Channel while its event loop is
also shutting down. Some of them are:
- channelDeregistered (triggered after a channel is closed)
- freeIn/OutboundBuffer (triggered after channelDeregistered)
- userEventTriggered (triggered anytime)
To address this issue, a new method called confirmShutdown() has been
added to SingleThreadEventExecutor. After a user calls shutdown(),
confirmShutdown() runs any remaining tasks in the task queue and ensures
no events are triggered for last 2 seconds. If any task are added to
the task queue before 2 seconds passes, confirmShutdown() prevents the
event loop from terminating by returning false.
Now that SingleThreadEventExecutor needs to accept tasks even after
shutdown(), its execute() method only rejects the task after the event
loop is terminated (i.e. isTerminated() returns true.) Except that,
there's no change in semantics.
SingleThreadEventExecutor also checks if its subclass called
confirmShutdown() in its run() implementation, so that Netty developers
can make sure they shut down their event loop impementation correctly.
It also fixes a bug in AioSocketChannel, revealed by delayed shutdown,
where an inboundBufferUpdated() event is triggered on a closed Channel
with deallocated buffers.
Caveats:
Because SingleThreadEventExecutor.takeTask() does not have a notion of
timeout, confirmShutdown() adds a dummy task (WAKEUP_TASK) to wake up
takeTask() immediately and instead sleeps hard-coded 100ms. I'll
address this issue later by modifying takeTask() times out dynamically.
Miscellaneous changes:
SingleThreadEventExecutor.wakeup() now has the default implementation.
Instead of interrupting the current thread, it simply adds a dummy task
(WAKEUP_TASK) to the task queue, which is more elegant and efficient.
NioEventLoop is the only implementation that overrides it. All other
implementations' wakeup()s were removed thanks to this change.
2012-11-22 12:45:49 +01:00
|
|
|
|
2013-04-27 09:39:19 +02:00
|
|
|
@Override
|
|
|
|
@Deprecated
|
2019-01-23 08:32:05 +01:00
|
|
|
public final void shutdown() {
|
2012-06-02 02:51:19 +02:00
|
|
|
for (EventExecutor l: children) {
|
|
|
|
l.shutdown();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-04-27 09:39:19 +02:00
|
|
|
@Override
|
2019-01-23 08:32:05 +01:00
|
|
|
public final boolean isShuttingDown() {
|
2013-04-27 09:39:19 +02:00
|
|
|
for (EventExecutor l: children) {
|
|
|
|
if (!l.isShuttingDown()) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2012-06-02 02:51:19 +02:00
|
|
|
@Override
|
2019-01-23 08:32:05 +01:00
|
|
|
public final boolean isShutdown() {
|
2012-06-02 02:51:19 +02:00
|
|
|
for (EventExecutor l: children) {
|
|
|
|
if (!l.isShutdown()) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2019-01-23 08:32:05 +01:00
|
|
|
public final boolean isTerminated() {
|
2012-06-02 02:51:19 +02:00
|
|
|
for (EventExecutor l: children) {
|
|
|
|
if (!l.isTerminated()) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2019-01-23 08:32:05 +01:00
|
|
|
public final boolean awaitTermination(long timeout, TimeUnit unit)
|
2012-06-02 02:51:19 +02:00
|
|
|
throws InterruptedException {
|
|
|
|
long deadline = System.nanoTime() + unit.toNanos(timeout);
|
|
|
|
loop: for (EventExecutor l: children) {
|
|
|
|
for (;;) {
|
|
|
|
long timeLeft = deadline - System.nanoTime();
|
|
|
|
if (timeLeft <= 0) {
|
|
|
|
break loop;
|
|
|
|
}
|
|
|
|
if (l.awaitTermination(timeLeft, TimeUnit.NANOSECONDS)) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return isTerminated();
|
|
|
|
}
|
|
|
|
}
|