From 03b0099b6305a3c9d7fe9df70b9d47d0ebfdf9c0 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 23 Dec 2013 09:29:14 +0100 Subject: [PATCH] microbench: move from Caliper to JMH --- microbench/README.md | 0 microbench/pom.xml | 18 +- .../java/com/google/caliper/StandardVm.java | 73 ------ .../java/com/google/caliper/package-info.java | 20 -- .../buffer/ByteBufAllocatorBenchmark.java | 215 ++++++++++++------ .../netty/microbench/buffer/package-info.java | 20 -- .../util/AbstractMicrobenchmark.java | 90 ++++++++ .../microbench/util/DefaultBenchmark.java | 129 ----------- .../netty/microbench/util/package-info.java | 20 -- microbench/src/test/resources/logback.xml | 5 + 10 files changed, 259 insertions(+), 331 deletions(-) create mode 100644 microbench/README.md delete mode 100644 microbench/src/test/java/com/google/caliper/StandardVm.java delete mode 100644 microbench/src/test/java/com/google/caliper/package-info.java delete mode 100644 microbench/src/test/java/io/netty/microbench/buffer/package-info.java create mode 100644 microbench/src/test/java/io/netty/microbench/util/AbstractMicrobenchmark.java delete mode 100644 microbench/src/test/java/io/netty/microbench/util/DefaultBenchmark.java delete mode 100644 microbench/src/test/java/io/netty/microbench/util/package-info.java create mode 100644 microbench/src/test/resources/logback.xml diff --git a/microbench/README.md b/microbench/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/microbench/pom.xml b/microbench/pom.xml index e8ce2a626f..8d5a38cdfa 100644 --- a/microbench/pom.xml +++ b/microbench/pom.xml @@ -45,8 +45,9 @@ ${project.version} - com.google.caliper - caliper + org.openjdk.jmh + jmh-core + 0.2.1 @@ -58,6 +59,19 @@ true + + org.apache.maven.plugins + maven-surefire-plugin + + + **/AbstractMicrobenchmark.java + **/*$*.class + + + ${project.build.directory}/reports/performance/ + + + diff --git a/microbench/src/test/java/com/google/caliper/StandardVm.java b/microbench/src/test/java/com/google/caliper/StandardVm.java deleted file mode 100644 index 211c7a423b..0000000000 --- a/microbench/src/test/java/com/google/caliper/StandardVm.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2012 The Netty Project - * - * The Netty Project licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -/* - * Copyright (C) 2010 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.caliper; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; - -import java.util.Collections; -import java.util.List; - -final class StandardVm extends Vm { - - @Override public List getVmSpecificOptions(MeasurementType type, Arguments arguments) { - if (!arguments.getCaptureVmLog()) { - return ImmutableList.of(); - } - - List result = Lists.newArrayList( - "-server", "-dsa", "-da", "-ea:io.netty...", - "-Xms768m", "-Xmx768m", "-XX:MaxDirectMemorySize=768m", - "-XX:+AggressiveOpts", "-XX:+UseBiasedLocking", "-XX:+UseFastAccessorMethods", - "-XX:+UseStringCache", "-XX:+OptimizeStringConcat", - "-XX:+HeapDumpOnOutOfMemoryError", "-Dio.netty.noResourceLeakDetection"); - - if (type == MeasurementType.TIME) { - Collections.addAll( - result, - "-XX:+PrintCompilation"); - } else { - Collections.addAll( - result, - "-verbose:gc", - "-Xbatch", - "-XX:+UseSerialGC", - "-XX:+TieredCompilation"); - } - - return result; - } - - public static String defaultVmName() { - return "java"; - } -} diff --git a/microbench/src/test/java/com/google/caliper/package-info.java b/microbench/src/test/java/com/google/caliper/package-info.java deleted file mode 100644 index 5931441937..0000000000 --- a/microbench/src/test/java/com/google/caliper/package-info.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2013 The Netty Project - * - * The Netty Project licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -/** - * Caliper VM arguments - */ -package com.google.caliper; diff --git a/microbench/src/test/java/io/netty/microbench/buffer/ByteBufAllocatorBenchmark.java b/microbench/src/test/java/io/netty/microbench/buffer/ByteBufAllocatorBenchmark.java index 9482e9310a..538af6168a 100644 --- a/microbench/src/test/java/io/netty/microbench/buffer/ByteBufAllocatorBenchmark.java +++ b/microbench/src/test/java/io/netty/microbench/buffer/ByteBufAllocatorBenchmark.java @@ -13,88 +13,169 @@ * License for the specific language governing permissions and limitations * under the License. */ - package io.netty.microbench.buffer; -import com.google.caliper.Param; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; import io.netty.buffer.PooledByteBufAllocator; import io.netty.buffer.UnpooledByteBufAllocator; -import io.netty.microbench.util.DefaultBenchmark; +import io.netty.microbench.util.AbstractMicrobenchmark; +import org.openjdk.jmh.annotations.GenerateMicroBenchmark; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; -import java.util.ArrayDeque; -import java.util.Deque; +/** + * This class benchmarks different allocators with different allocation sizes. + */ +@State(Scope.Thread) +public class ByteBufAllocatorBenchmark extends AbstractMicrobenchmark { -public class ByteBufAllocatorBenchmark extends DefaultBenchmark { + private final ByteBufAllocator unpooledHeapAllocator = new UnpooledByteBufAllocator(false); + private final ByteBufAllocator unpooledDirectAllocator = new UnpooledByteBufAllocator(true); + private final ByteBufAllocator pooledHeapAllocator = new PooledByteBufAllocator(false); + private final ByteBufAllocator pooledDirectAllocator = new PooledByteBufAllocator(true); - private static final ByteBufAllocator UNPOOLED_ALLOCATOR_HEAP = new UnpooledByteBufAllocator(false); - private static final ByteBufAllocator UNPOOLED_ALLOCATOR_DIRECT = new UnpooledByteBufAllocator(true); - private static final ByteBufAllocator POOLED_ALLOCATOR_HEAP = new PooledByteBufAllocator(false); - private static final ByteBufAllocator POOLED_ALLOCATOR_DIRECT = new PooledByteBufAllocator(true); - - @Param({ "0", "256", "1024", "4096", "16384", "65536" }) - private int size; - - @Param - private Allocator allocator; - - private final Deque queue = new ArrayDeque(); - private ByteBufAllocator alloc; - - @Override - protected void setUp() throws Exception { - alloc = allocator.alloc(); - for (int i = 0; i < 2560; i ++) { - queue.add(alloc.buffer(size)); - } + @GenerateMicroBenchmark + public void unpooledHeapAllocAndFree_1_0() { + ByteBuf buffer = unpooledHeapAllocator.buffer(0); + buffer.release(); } - @Override - protected void tearDown() throws Exception { - for (ByteBuf b: queue) { - b.release(); - } - queue.clear(); + @GenerateMicroBenchmark + public void unpooledHeapAllocAndFree_2_256() { + ByteBuf buffer = unpooledHeapAllocator.buffer(256); + buffer.release(); } - public void timeAllocAndFree(int reps) { - final ByteBufAllocator alloc = this.alloc; - final Deque queue = this.queue; - final int size = this.size; - - for (int i = 0; i < reps; i ++) { - queue.add(alloc.buffer(size)); - queue.removeFirst().release(); - } + @GenerateMicroBenchmark + public void unpooledHeapAllocAndFree_3_1024() { + ByteBuf buffer = unpooledHeapAllocator.buffer(1024); + buffer.release(); } - public enum Allocator { - UNPOOLED_HEAP { - @Override - ByteBufAllocator alloc() { - return UNPOOLED_ALLOCATOR_HEAP; - } - }, - UNPOOLED_DIRECT { - @Override - ByteBufAllocator alloc() { - return UNPOOLED_ALLOCATOR_DIRECT; - } - }, - POOLED_HEAP { - @Override - ByteBufAllocator alloc() { - return POOLED_ALLOCATOR_HEAP; - } - }, - POOLED_DIRECT { - @Override - ByteBufAllocator alloc() { - return POOLED_ALLOCATOR_DIRECT; - } - }; + @GenerateMicroBenchmark + public void unpooledHeapAllocAndFree_4_4096() { + ByteBuf buffer = unpooledHeapAllocator.buffer(4096); + buffer.release(); + } - abstract ByteBufAllocator alloc(); + @GenerateMicroBenchmark + public void unpooledHeapAllocAndFree_5_16384() { + ByteBuf buffer = unpooledHeapAllocator.buffer(16384); + buffer.release(); + } + + @GenerateMicroBenchmark + public void unpooledHeapAllocAndFree_6_65536() { + ByteBuf buffer = unpooledHeapAllocator.buffer(65536); + buffer.release(); + } + + @GenerateMicroBenchmark + public void unpooledDirectAllocAndFree_1_0() { + ByteBuf buffer = unpooledDirectAllocator.buffer(0); + buffer.release(); + } + + @GenerateMicroBenchmark + public void unpooledDirectAllocAndFree_2_256() { + ByteBuf buffer = unpooledDirectAllocator.buffer(256); + buffer.release(); + } + + @GenerateMicroBenchmark + public void unpooledDirectAllocAndFree_3_1024() { + ByteBuf buffer = unpooledDirectAllocator.buffer(1024); + buffer.release(); + } + + @GenerateMicroBenchmark + public void unpooledDirectAllocAndFree_4_4096() { + ByteBuf buffer = unpooledDirectAllocator.buffer(4096); + buffer.release(); + } + + @GenerateMicroBenchmark + public void unpooledDirectAllocAndFree_5_16384() { + ByteBuf buffer = unpooledDirectAllocator.buffer(16384); + buffer.release(); + } + + @GenerateMicroBenchmark + public void unpooledDirectAllocAndFree_6_65536() { + ByteBuf buffer = unpooledDirectAllocator.buffer(65536); + buffer.release(); + } + + @GenerateMicroBenchmark + public void pooledHeapAllocAndFree_1_0() { + ByteBuf buffer = pooledHeapAllocator.buffer(0); + buffer.release(); + } + + @GenerateMicroBenchmark + public void pooledHeapAllocAndFree_2_256() { + ByteBuf buffer = pooledHeapAllocator.buffer(256); + buffer.release(); + } + + @GenerateMicroBenchmark + public void pooledHeapAllocAndFree_3_1024() { + ByteBuf buffer = pooledHeapAllocator.buffer(1024); + buffer.release(); + } + + @GenerateMicroBenchmark + public void pooledHeapAllocAndFree_4_4096() { + ByteBuf buffer = pooledHeapAllocator.buffer(4096); + buffer.release(); + } + + @GenerateMicroBenchmark + public void pooledHeapAllocAndFree_5_16384() { + ByteBuf buffer = pooledHeapAllocator.buffer(16384); + buffer.release(); + } + + @GenerateMicroBenchmark + public void pooledHeapAllocAndFree_6_65536() { + ByteBuf buffer = pooledHeapAllocator.buffer(65536); + buffer.release(); + } + + @GenerateMicroBenchmark + public void pooledDirectAllocAndFree_1_0() { + ByteBuf buffer = pooledDirectAllocator.buffer(0); + buffer.release(); + } + + @GenerateMicroBenchmark + public void pooledDirectAllocAndFree_2_256() { + ByteBuf buffer = pooledDirectAllocator.buffer(256); + buffer.release(); + } + + @GenerateMicroBenchmark + public void pooledDirectAllocAndFree_3_1024() { + ByteBuf buffer = pooledDirectAllocator.buffer(1024); + buffer.release(); + } + + @GenerateMicroBenchmark + public void pooledDirectAllocAndFree_4_4096() { + ByteBuf buffer = pooledDirectAllocator.buffer(4096); + buffer.release(); + } + + @GenerateMicroBenchmark + public void pooledDirectAllocAndFree_5_16384() { + ByteBuf buffer = pooledDirectAllocator.buffer(16384); + buffer.release(); + } + + @GenerateMicroBenchmark + public void pooledDirectAllocAndFree_6_65536() { + ByteBuf buffer = pooledDirectAllocator.buffer(65536); + buffer.release(); } } diff --git a/microbench/src/test/java/io/netty/microbench/buffer/package-info.java b/microbench/src/test/java/io/netty/microbench/buffer/package-info.java deleted file mode 100644 index c4b7939333..0000000000 --- a/microbench/src/test/java/io/netty/microbench/buffer/package-info.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2013 The Netty Project - * - * The Netty Project licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -/** - * Benchmark classes for Netty ByteBuf - */ -package io.netty.microbench.buffer; diff --git a/microbench/src/test/java/io/netty/microbench/util/AbstractMicrobenchmark.java b/microbench/src/test/java/io/netty/microbench/util/AbstractMicrobenchmark.java new file mode 100644 index 0000000000..e0d8823c75 --- /dev/null +++ b/microbench/src/test/java/io/netty/microbench/util/AbstractMicrobenchmark.java @@ -0,0 +1,90 @@ +/* + * Copyright 2012 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package io.netty.microbench.util; + +import io.netty.util.ResourceLeakDetector; +import org.junit.Test; +import org.openjdk.jmh.output.OutputFormatType; +import org.openjdk.jmh.output.results.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.io.File; + +/** + * Base class for all JMH benchmarks. + */ +public class AbstractMicrobenchmark { + + protected static final String WARMUP_ITERATIONS = "10"; + protected static final String MEASURE_ITERATIONS = "10"; + protected static final String NUM_FORKS = "2"; + + protected static final String JVM_ARGS = "-server -dsa -da -ea:io.netty... -Xms768m" + + " -Xmx768m -XX:MaxDirectMemorySize=768m -XX:+AggressiveOpts -XX:+UseBiasedLocking" + + " -XX:+UseFastAccessorMethods -XX:+UseStringCache -XX:+OptimizeStringConcat" + + " -XX:+HeapDumpOnOutOfMemoryError -Dio.netty.noResourceLeakDetection"; + + static { + ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.DISABLED); + } + + @Test + public void run() throws Exception { + String className = getClass().getSimpleName(); + + ChainedOptionsBuilder runnerOptions = new OptionsBuilder() + .include(".*" + className + ".*") + .jvmArgs(JVM_ARGS) + .warmupIterations(getWarmupIterations()) + .measurementIterations(getMeasureIterations()) + .forks(getForks()); + + if (getReportDir() != null) { + String filePath = getReportDir() + className + ".json"; + File file = new File(filePath); + if (file.exists()) { + file.delete(); + } else { + file.getParentFile().mkdirs(); + file.createNewFile(); + } + + runnerOptions.resultFormat(ResultFormatType.JSON); + runnerOptions.result(filePath); + } + + new Runner(runnerOptions.build()).run(); + } + + protected int getWarmupIterations() { + return Integer.parseInt(System.getProperty("warmupIterations", WARMUP_ITERATIONS)); + } + + protected int getMeasureIterations() { + return Integer.parseInt(System.getProperty("measureIterations", MEASURE_ITERATIONS)); + } + + protected int getForks() { + return Integer.parseInt(System.getProperty("forks", NUM_FORKS)); + } + + protected String getReportDir() { + return System.getProperty("perfReportDir", null); + } +} diff --git a/microbench/src/test/java/io/netty/microbench/util/DefaultBenchmark.java b/microbench/src/test/java/io/netty/microbench/util/DefaultBenchmark.java deleted file mode 100644 index 1ee3f5e9c9..0000000000 --- a/microbench/src/test/java/io/netty/microbench/util/DefaultBenchmark.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 2012 The Netty Project - * - * The Netty Project licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package io.netty.microbench.util; - -import com.google.caliper.CaliperRc; -import com.google.caliper.Runner; -import com.google.caliper.SimpleBenchmark; -import org.junit.Test; - -import java.io.File; - -import static org.junit.Assert.*; - -public abstract class DefaultBenchmark extends SimpleBenchmark { - - private static boolean warned; - - private final int trials; - private final int warmupMillis; - private final int runMillis; - - protected DefaultBenchmark() { - this(1); - } - - protected DefaultBenchmark(int trials) { - this(trials, 3000, 1000); - } - - protected DefaultBenchmark(int trials, int warmupMillis, int runMillis) { - this.trials = trials; - this.warmupMillis = warmupMillis; - this.runMillis = runMillis; - } - - @Test - public void runBenchmarks() throws Exception { - File me = new File(DefaultBenchmark.class.getResource( - '/' + DefaultBenchmark.class.getName().replace('.', '/') + ".class").getPath()); - - if (!me.exists()) { - fail("failed to determine the project path"); - } - - File buildDir = - me.getParentFile().getParentFile().getParentFile().getParentFile().getParentFile().getParentFile(); - - if (!buildDir.getPath().endsWith(File.separator + "target") || !buildDir.isDirectory()) { - fail("failed to locate the build directory"); - } - - File reportDir = new File(buildDir.getAbsolutePath() + File.separator + "caliper-reports"); - - if (!reportDir.exists()) { - if (!reportDir.mkdirs()) { - fail("failed to create the Caliper report directory: " + reportDir.getAbsolutePath()); - } - } - - if (!reportDir.isDirectory()) { - fail("not a directory: " + reportDir.getAbsolutePath()); - } - - boolean deleted = deleteOldReports(reportDir); - - if (!warned) { - CaliperRc caliperrc = CaliperRc.INSTANCE; - if (caliperrc.getApiKey() == null || caliperrc.getPostUrl() == null) { - warned = true; - System.out.println(); - System.out.println(" Cannot read the configuration properties from .caliperrc."); - System.out.println(" Please follow the instructions at:"); - System.out.println(); - System.out.println(" * http://code.google.com/p/caliper/wiki/OnlineResults"); - System.out.println(); - System.out.println(" to upload and browse the benchmark results."); - } - } - - if (deleted || warned) { - // Insert a pretty newline. - System.out.println(); - } - - new Runner().run( - "--trials", String.valueOf(trials), - "--warmupMillis", String.valueOf(warmupMillis), - "--runMillis", String.valueOf(runMillis), - "--saveResults", reportDir.getAbsolutePath(), - "--captureVmLog", - getClass().getName()); - } - - private boolean deleteOldReports(File reportDir) { - final String prefix = getClass().getName() + '.'; - final String suffix = ".json"; - - boolean deleted = false; - for (File f: reportDir.listFiles()) { - String name = f.getName(); - if (name.startsWith(prefix) && name.endsWith(suffix)) { - if (f.delete()) { - if (!deleted) { - deleted = true; - System.out.println(); - } - System.out.println(" Deleted old report: " + - name.substring(prefix.length(), name.length() - suffix.length())); - } - } - } - - return deleted; - } -} diff --git a/microbench/src/test/java/io/netty/microbench/util/package-info.java b/microbench/src/test/java/io/netty/microbench/util/package-info.java deleted file mode 100644 index c25b9c6fd8..0000000000 --- a/microbench/src/test/java/io/netty/microbench/util/package-info.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2013 The Netty Project - * - * The Netty Project licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -/** - * Util benchmark classes - */ -package io.netty.microbench.util; diff --git a/microbench/src/test/resources/logback.xml b/microbench/src/test/resources/logback.xml new file mode 100644 index 0000000000..bccf735a13 --- /dev/null +++ b/microbench/src/test/resources/logback.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file