From 98beb777f81f092aa0fddf49ed08b426b2c72f01 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 16 Jan 2017 12:36:32 -0500 Subject: [PATCH] Enable configuring available processors Motivation: In cases when an application is running in a container or is otherwise constrained to the number of processors that it is using, the JVM invocation Runtime#availableProcessors will not return the constrained value but rather the number of processors available to the virtual machine. Netty uses this number in sizing various resources. Additionally, some applications will constrain the number of threads that they are using independenly of the number of processors available on the system. Thus, applications should have a way to globally configure the number of processors. Modifications: Rather than invoking Runtime#availableProcessors, Netty should rely on a method that enables configuration when the JVM is started or by the application. This commit exposes a new class NettyRuntime for enabling such configuraiton. This value can only be set once. Its default value is Runtime#availableProcessors so that there is no visible change to existing applications, but enables configuring either a system property or configuring during application startup (e.g., based on settings used to configure the application). Additionally, we introduce the usage of forbidden-apis to prevent future uses of Runtime#availableProcessors from creeping. Future work should enable the bundled signatures and clean up uses of deprecated and other forbidden methods. Result: Netty can be configured to not use the underlying number of processors, but rather the constrained number of processors. --- .../netty/buffer/PooledByteBufAllocator.java | 14 +- .../main/java/io/netty/util/NettyRuntime.java | 106 +++++++++ .../src/main/java/io/netty/util/Recycler.java | 2 +- .../java/io/netty/util/SuppressForbidden.java | 32 +++ .../java/io/netty/util/NettyRuntimeTests.java | 206 ++++++++++++++++++ .../NonStickyEventExecutorGroupTest.java | 3 +- dev-tools/pom.xml | 54 +++++ .../main/resources/forbidden/signatures.txt | 17 ++ pom.xml | 101 ++++++++- .../channel/MultithreadEventLoopGroup.java | 3 +- 10 files changed, 528 insertions(+), 10 deletions(-) create mode 100644 common/src/main/java/io/netty/util/NettyRuntime.java create mode 100644 common/src/main/java/io/netty/util/SuppressForbidden.java create mode 100644 common/src/test/java/io/netty/util/NettyRuntimeTests.java create mode 100644 dev-tools/pom.xml create mode 100644 dev-tools/src/main/resources/forbidden/signatures.txt diff --git a/buffer/src/main/java/io/netty/buffer/PooledByteBufAllocator.java b/buffer/src/main/java/io/netty/buffer/PooledByteBufAllocator.java index 4bc4f182af..6e9c06cc80 100644 --- a/buffer/src/main/java/io/netty/buffer/PooledByteBufAllocator.java +++ b/buffer/src/main/java/io/netty/buffer/PooledByteBufAllocator.java @@ -16,6 +16,7 @@ package io.netty.buffer; +import io.netty.util.NettyRuntime; import io.netty.util.concurrent.FastThreadLocal; import io.netty.util.concurrent.FastThreadLocalThread; import io.netty.util.internal.PlatformDependent; @@ -73,11 +74,14 @@ public class PooledByteBufAllocator extends AbstractByteBufAllocator implements // Assuming each arena has 3 chunks, the pool should not consume more than 50% of max memory. final Runtime runtime = Runtime.getRuntime(); - // Use 2 * cores by default to reduce condition as we use 2 * cores for the number of EventLoops - // in NIO and EPOLL as well. If we choose a smaller number we will run into hotspots as allocation and - // deallocation needs to be synchronized on the PoolArena. - // See https://github.com/netty/netty/issues/3888 - final int defaultMinNumArena = runtime.availableProcessors() * 2; + /* + * We use 2 * available processors by default to reduce contention as we use 2 * available processors for the + * number of EventLoops in NIO and EPOLL as well. If we choose a smaller number we will run into hot spots as + * allocation and de-allocation needs to be synchronized on the PoolArena. + * + * See https://github.com/netty/netty/issues/3888. + */ + final int defaultMinNumArena = NettyRuntime.availableProcessors() * 2; final int defaultChunkSize = DEFAULT_PAGE_SIZE << DEFAULT_MAX_ORDER; DEFAULT_NUM_HEAP_ARENA = Math.max(0, SystemPropertyUtil.getInt( diff --git a/common/src/main/java/io/netty/util/NettyRuntime.java b/common/src/main/java/io/netty/util/NettyRuntime.java new file mode 100644 index 0000000000..362416151a --- /dev/null +++ b/common/src/main/java/io/netty/util/NettyRuntime.java @@ -0,0 +1,106 @@ +/* + * Copyright 2017 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.netty.util; + +import io.netty.util.internal.ObjectUtil; +import io.netty.util.internal.SystemPropertyUtil; + +import java.util.Locale; + +/** + * A utility class for wrapping calls to {@link Runtime}. + */ +public final class NettyRuntime { + + /** + * Holder class for available processors to enable testing. + */ + static class AvailableProcessorsHolder { + + private int availableProcessors; + + /** + * Set the number of available processors. + * + * @param availableProcessors the number of available processors + * @throws IllegalArgumentException if the specified number of available processors is non-positive + * @throws IllegalStateException if the number of available processors is already configured + */ + synchronized void setAvailableProcessors(final int availableProcessors) { + ObjectUtil.checkPositive(availableProcessors, "availableProcessors"); + if (this.availableProcessors != 0) { + final String message = String.format( + Locale.ROOT, + "availableProcessors is already set to [%d], rejecting [%d]", + this.availableProcessors, + availableProcessors); + throw new IllegalStateException(message); + } + this.availableProcessors = availableProcessors; + } + + /** + * Get the configured number of available processors. The default is {@link Runtime#availableProcessors()}. + * This can be overridden by setting the system property "io.netty.availableProcessors" or by invoking + * {@link #setAvailableProcessors(int)} before any calls to this method. + * + * @return the configured number of available processors + */ + @SuppressForbidden(reason = "to obtain default number of available processors") + synchronized int availableProcessors() { + if (this.availableProcessors == 0) { + final int availableProcessors = + SystemPropertyUtil.getInt( + "io.netty.availableProcessors", + Runtime.getRuntime().availableProcessors()); + setAvailableProcessors(availableProcessors); + } + return this.availableProcessors; + } + } + + private static final AvailableProcessorsHolder holder = new AvailableProcessorsHolder(); + + /** + * Set the number of available processors. + * + * @param availableProcessors the number of available processors + * @throws IllegalArgumentException if the specified number of available processors is non-positive + * @throws IllegalStateException if the number of available processors is already configured + */ + @SuppressWarnings("unused,WeakerAccess") // this method is part of the public API + public static void setAvailableProcessors(final int availableProcessors) { + holder.setAvailableProcessors(availableProcessors); + } + + /** + * Get the configured number of available processors. The default is {@link Runtime#availableProcessors()}. This + * can be overridden by setting the system property "io.netty.availableProcessors" or by invoking + * {@link #setAvailableProcessors(int)} before any calls to this method. + * + * @return the configured number of available processors + */ + public static int availableProcessors() { + return holder.availableProcessors(); + } + + /** + * No public constructor to prevent instances from being created. + */ + private NettyRuntime() { + } +} diff --git a/common/src/main/java/io/netty/util/Recycler.java b/common/src/main/java/io/netty/util/Recycler.java index 6a30f0efc1..4a9d44d4ba 100644 --- a/common/src/main/java/io/netty/util/Recycler.java +++ b/common/src/main/java/io/netty/util/Recycler.java @@ -76,7 +76,7 @@ public abstract class Recycler { MAX_DELAYED_QUEUES_PER_THREAD = max(0, SystemPropertyUtil.getInt("io.netty.recycler.maxDelayedQueuesPerThread", // We use the same value as default EventLoop number - Runtime.getRuntime().availableProcessors() * 2)); + NettyRuntime.availableProcessors() * 2)); LINK_CAPACITY = safeFindNextPositivePowerOfTwo( max(SystemPropertyUtil.getInt("io.netty.recycler.linkCapacity", 16), 16)); diff --git a/common/src/main/java/io/netty/util/SuppressForbidden.java b/common/src/main/java/io/netty/util/SuppressForbidden.java new file mode 100644 index 0000000000..318cf92065 --- /dev/null +++ b/common/src/main/java/io/netty/util/SuppressForbidden.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.netty.util; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation to suppress forbidden-apis errors inside a whole class, a method, or a field. + */ +@Retention(RetentionPolicy.CLASS) +@Target({ ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE }) +public @interface SuppressForbidden { + + String reason(); +} diff --git a/common/src/test/java/io/netty/util/NettyRuntimeTests.java b/common/src/test/java/io/netty/util/NettyRuntimeTests.java new file mode 100644 index 0000000000..123ffe5f57 --- /dev/null +++ b/common/src/test/java/io/netty/util/NettyRuntimeTests.java @@ -0,0 +1,206 @@ +/* + * Copyright 2017 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.netty.util; + +import io.netty.util.internal.SystemPropertyUtil; +import org.junit.Test; + +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasToString; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +public class NettyRuntimeTests { + + @Test + public void testIllegalSet() { + final NettyRuntime.AvailableProcessorsHolder holder = new NettyRuntime.AvailableProcessorsHolder(); + for (final int i : new int[] { -1, 0 }) { + try { + holder.setAvailableProcessors(i); + fail(); + } catch (final IllegalArgumentException e) { + assertThat(e, hasToString(containsString("(expected: > 0)"))); + } + } + } + + @Test + public void testMultipleSets() { + final NettyRuntime.AvailableProcessorsHolder holder = new NettyRuntime.AvailableProcessorsHolder(); + holder.setAvailableProcessors(1); + try { + holder.setAvailableProcessors(2); + fail(); + } catch (final IllegalStateException e) { + assertThat(e, hasToString(containsString("availableProcessors is already set to [1], rejecting [2]"))); + } + } + + @Test + public void testSetAfterGet() { + final NettyRuntime.AvailableProcessorsHolder holder = new NettyRuntime.AvailableProcessorsHolder(); + holder.availableProcessors(); + try { + holder.setAvailableProcessors(1); + fail(); + } catch (final IllegalStateException e) { + assertThat(e, hasToString(containsString("availableProcessors is already set"))); + } + } + + @Test + public void testRacingGetAndGet() throws InterruptedException { + final NettyRuntime.AvailableProcessorsHolder holder = new NettyRuntime.AvailableProcessorsHolder(); + final CyclicBarrier barrier = new CyclicBarrier(3); + + final AtomicReference firstReference = new AtomicReference(); + final Runnable firstTarget = getRunnable(holder, barrier, firstReference); + final Thread firstGet = new Thread(firstTarget); + firstGet.start(); + + final AtomicReference secondRefernce = new AtomicReference(); + final Runnable secondTarget = getRunnable(holder, barrier, secondRefernce); + final Thread secondGet = new Thread(secondTarget); + secondGet.start(); + + // release the hounds + await(barrier); + + // wait for the hounds + await(barrier); + + firstGet.join(); + secondGet.join(); + + assertNull(firstReference.get()); + assertNull(secondRefernce.get()); + } + + private Runnable getRunnable( + final NettyRuntime.AvailableProcessorsHolder holder, + final CyclicBarrier barrier, + final AtomicReference reference) { + return new Runnable() { + @Override + public void run() { + await(barrier); + try { + holder.availableProcessors(); + } catch (final IllegalStateException e) { + reference.set(e); + } + await(barrier); + } + }; + } + + @Test + public void testRacingGetAndSet() throws InterruptedException { + final NettyRuntime.AvailableProcessorsHolder holder = new NettyRuntime.AvailableProcessorsHolder(); + final CyclicBarrier barrier = new CyclicBarrier(3); + final Thread get = new Thread(new Runnable() { + @Override + public void run() { + await(barrier); + holder.availableProcessors(); + await(barrier); + } + }); + get.start(); + + final AtomicReference setException = new AtomicReference(); + final Thread set = new Thread(new Runnable() { + @Override + public void run() { + await(barrier); + try { + holder.setAvailableProcessors(2048); + } catch (final IllegalStateException e) { + setException.set(e); + } + await(barrier); + } + }); + set.start(); + + // release the hounds + await(barrier); + + // wait for the hounds + await(barrier); + + get.join(); + set.join(); + + if (setException.get() == null) { + assertThat(holder.availableProcessors(), equalTo(2048)); + } else { + assertNotNull(setException.get()); + } + } + + @Test + public void testGetWithSystemProperty() { + final String availableProcessorsSystemProperty = SystemPropertyUtil.get("io.netty.availableProcessors"); + try { + System.setProperty("io.netty.availableProcessors", "2048"); + final NettyRuntime.AvailableProcessorsHolder holder = new NettyRuntime.AvailableProcessorsHolder(); + assertThat(holder.availableProcessors(), equalTo(2048)); + } finally { + if (availableProcessorsSystemProperty != null) { + System.setProperty("io.netty.availableProcessors", availableProcessorsSystemProperty); + } else { + System.clearProperty("io.netty.availableProcessors"); + } + } + } + + @Test + @SuppressForbidden(reason = "testing fallback to Runtime#availableProcessors") + public void testGet() { + final String availableProcessorsSystemProperty = SystemPropertyUtil.get("io.netty.availableProcessors"); + try { + System.clearProperty("io.netty.availableProcessors"); + final NettyRuntime.AvailableProcessorsHolder holder = new NettyRuntime.AvailableProcessorsHolder(); + assertThat(holder.availableProcessors(), equalTo(Runtime.getRuntime().availableProcessors())); + } finally { + if (availableProcessorsSystemProperty != null) { + System.setProperty("io.netty.availableProcessors", availableProcessorsSystemProperty); + } else { + System.clearProperty("io.netty.availableProcessors"); + } + } + } + + private static void await(final CyclicBarrier barrier) { + try { + barrier.await(); + } catch (final InterruptedException e) { + fail(e.toString()); + } catch (final BrokenBarrierException e) { + fail(e.toString()); + } + } +} diff --git a/common/src/test/java/io/netty/util/concurrent/NonStickyEventExecutorGroupTest.java b/common/src/test/java/io/netty/util/concurrent/NonStickyEventExecutorGroupTest.java index 5582a049da..16035505e8 100644 --- a/common/src/test/java/io/netty/util/concurrent/NonStickyEventExecutorGroupTest.java +++ b/common/src/test/java/io/netty/util/concurrent/NonStickyEventExecutorGroupTest.java @@ -15,6 +15,7 @@ */ package io.netty.util.concurrent; +import io.netty.util.NettyRuntime; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -58,7 +59,7 @@ public class NonStickyEventExecutorGroupTest { @Test(timeout = 10000) public void testOrdering() throws Throwable { - final int threads = Runtime.getRuntime().availableProcessors() * 2; + final int threads = NettyRuntime.availableProcessors() * 2; final EventExecutorGroup group = new UnorderedThreadPoolEventExecutor(threads); final NonStickyEventExecutorGroup nonStickyGroup = new NonStickyEventExecutorGroup(group, maxTaskExecutePerRun); try { diff --git a/dev-tools/pom.xml b/dev-tools/pom.xml new file mode 100644 index 0000000000..88f560b49e --- /dev/null +++ b/dev-tools/pom.xml @@ -0,0 +1,54 @@ + + + + 4.0.0 + + org.sonatype.oss + oss-parent + 7 + + + + io.netty + netty-dev-tools + 4.1.10.Final-SNAPSHOT + + Netty/Dev-Tools + + + + + org.apache.maven.plugins + maven-remote-resources-plugin + 1.5 + + + + bundle + + + + + + **/* + + + + + + diff --git a/dev-tools/src/main/resources/forbidden/signatures.txt b/dev-tools/src/main/resources/forbidden/signatures.txt new file mode 100644 index 0000000000..e4e90ed250 --- /dev/null +++ b/dev-tools/src/main/resources/forbidden/signatures.txt @@ -0,0 +1,17 @@ +/* + * Copyright 2017 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +java.lang.Runtime#availableProcessors() @ use NettyRuntime#availableProcessors() diff --git a/pom.xml b/pom.xml index bacc5b5b95..40c3b0751f 100644 --- a/pom.xml +++ b/pom.xml @@ -215,6 +215,9 @@ + 1.6 + 1.6 + ${project.build.directory}/dev-tools UTF-8 UTF-8 22 @@ -259,6 +262,7 @@ + dev-tools common buffer codec @@ -293,6 +297,12 @@ + + ${project.groupId} + netty-dev-tools + ${project.version} + + org.javassist @@ -653,8 +663,8 @@ 1.8 true - 1.6 - 1.6 + ${maven.compiler.source} + ${maven.compiler.target} true true true @@ -1085,6 +1095,16 @@ + + org.apache.maven.plugins + maven-remote-resources-plugin + 1.5 + + + de.thetaphi + forbiddenapis + 2.2 + @@ -1300,6 +1320,83 @@ + + org.apache.maven.plugins + maven-remote-resources-plugin + 1.5 + + + io.netty:netty-dev-tools:${project.version} + + ${netty.dev.tools.directory} + + false + false + + + + + process + + + + + + de.thetaphi + forbiddenapis + 2.2 + + + check-forbidden-apis + + ${maven.compiler.target} + + false + + false + + + + + + + + + ${netty.dev.tools.directory}/forbidden/signatures.txt + + **.SuppressForbidden + + compile + + check + + + + check-forbidden-test-apis + + ${maven.compiler.target} + + true + + false + + + + + + + + ${netty.dev.tools.directory}/forbidden/signatures.txt + + **.SuppressForbidden + + test-compile + + testCheck + + + + diff --git a/transport/src/main/java/io/netty/channel/MultithreadEventLoopGroup.java b/transport/src/main/java/io/netty/channel/MultithreadEventLoopGroup.java index 9395f5687e..a9bc23dfe9 100644 --- a/transport/src/main/java/io/netty/channel/MultithreadEventLoopGroup.java +++ b/transport/src/main/java/io/netty/channel/MultithreadEventLoopGroup.java @@ -15,6 +15,7 @@ */ package io.netty.channel; +import io.netty.util.NettyRuntime; import io.netty.util.concurrent.DefaultThreadFactory; import io.netty.util.concurrent.EventExecutorChooserFactory; import io.netty.util.concurrent.MultithreadEventExecutorGroup; @@ -37,7 +38,7 @@ public abstract class MultithreadEventLoopGroup extends MultithreadEventExecutor static { DEFAULT_EVENT_LOOP_THREADS = Math.max(1, SystemPropertyUtil.getInt( - "io.netty.eventLoopThreads", Runtime.getRuntime().availableProcessors() * 2)); + "io.netty.eventLoopThreads", NettyRuntime.availableProcessors() * 2)); if (logger.isDebugEnabled()) { logger.debug("-Dio.netty.eventLoopThreads: {}", DEFAULT_EVENT_LOOP_THREADS);