microbench: move from Caliper to JMH

This commit is contained in:
Michael Nitschinger 2013-12-23 09:29:14 +01:00 committed by Trustin Lee
parent 2f9ad15d3f
commit 78790056c7
10 changed files with 259 additions and 331 deletions

0
microbench/README.md Normal file
View File

View File

@ -45,8 +45,9 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.google.caliper</groupId>
<artifactId>caliper</artifactId>
<groupId>org.openjdk.jmh</groupId>
<artifactId>jmh-core</artifactId>
<version>0.2.1</version>
</dependency>
</dependencies>
@ -58,6 +59,19 @@
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/AbstractMicrobenchmark.java</exclude>
<exclude>**/*$*.class</exclude>
</excludes>
<systemPropertyVariables>
<perfReportDir>${project.build.directory}/reports/performance/</perfReportDir>
</systemPropertyVariables>
</configuration>
</plugin>
</plugins>
</build>
</project>

View File

@ -1,73 +0,0 @@
/*
* Copyright 2012 The Netty Project
*
* The Netty Project licenses this file to you under the Apache License,
* version 2.0 (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
/*
* Copyright (C) 2010 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.caliper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import java.util.Collections;
import java.util.List;
final class StandardVm extends Vm {
@Override public List<String> getVmSpecificOptions(MeasurementType type, Arguments arguments) {
if (!arguments.getCaptureVmLog()) {
return ImmutableList.of();
}
List<String> result = Lists.newArrayList(
"-server", "-dsa", "-da", "-ea:io.netty...",
"-Xms768m", "-Xmx768m", "-XX:MaxDirectMemorySize=768m",
"-XX:+AggressiveOpts", "-XX:+UseBiasedLocking", "-XX:+UseFastAccessorMethods",
"-XX:+UseStringCache", "-XX:+OptimizeStringConcat",
"-XX:+HeapDumpOnOutOfMemoryError", "-Dio.netty.noResourceLeakDetection");
if (type == MeasurementType.TIME) {
Collections.addAll(
result,
"-XX:+PrintCompilation");
} else {
Collections.addAll(
result,
"-verbose:gc",
"-Xbatch",
"-XX:+UseSerialGC",
"-XX:+TieredCompilation");
}
return result;
}
public static String defaultVmName() {
return "java";
}
}

View File

@ -1,20 +0,0 @@
/*
* Copyright 2013 The Netty Project
*
* The Netty Project licenses this file to you under the Apache License,
* version 2.0 (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
/**
* Caliper VM arguments
*/
package com.google.caliper;

View File

@ -13,88 +13,169 @@
* License for the specific language governing permissions and limitations
* under the License.
*/
package io.netty.microbench.buffer;
import com.google.caliper.Param;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.UnpooledByteBufAllocator;
import io.netty.microbench.util.DefaultBenchmark;
import io.netty.microbench.util.AbstractMicrobenchmark;
import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.State;
import java.util.ArrayDeque;
import java.util.Deque;
/**
* This class benchmarks different allocators with different allocation sizes.
*/
@State(Scope.Thread)
public class ByteBufAllocatorBenchmark extends AbstractMicrobenchmark {
public class ByteBufAllocatorBenchmark extends DefaultBenchmark {
private final ByteBufAllocator unpooledHeapAllocator = new UnpooledByteBufAllocator(false);
private final ByteBufAllocator unpooledDirectAllocator = new UnpooledByteBufAllocator(true);
private final ByteBufAllocator pooledHeapAllocator = new PooledByteBufAllocator(false);
private final ByteBufAllocator pooledDirectAllocator = new PooledByteBufAllocator(true);
private static final ByteBufAllocator UNPOOLED_ALLOCATOR_HEAP = new UnpooledByteBufAllocator(false);
private static final ByteBufAllocator UNPOOLED_ALLOCATOR_DIRECT = new UnpooledByteBufAllocator(true);
private static final ByteBufAllocator POOLED_ALLOCATOR_HEAP = new PooledByteBufAllocator(false);
private static final ByteBufAllocator POOLED_ALLOCATOR_DIRECT = new PooledByteBufAllocator(true);
@Param({ "0", "256", "1024", "4096", "16384", "65536" })
private int size;
@Param
private Allocator allocator;
private final Deque<ByteBuf> queue = new ArrayDeque<ByteBuf>();
private ByteBufAllocator alloc;
@Override
protected void setUp() throws Exception {
alloc = allocator.alloc();
for (int i = 0; i < 2560; i ++) {
queue.add(alloc.buffer(size));
}
@GenerateMicroBenchmark
public void unpooledHeapAllocAndFree_1_0() {
ByteBuf buffer = unpooledHeapAllocator.buffer(0);
buffer.release();
}
@Override
protected void tearDown() throws Exception {
for (ByteBuf b: queue) {
b.release();
}
queue.clear();
@GenerateMicroBenchmark
public void unpooledHeapAllocAndFree_2_256() {
ByteBuf buffer = unpooledHeapAllocator.buffer(256);
buffer.release();
}
public void timeAllocAndFree(int reps) {
final ByteBufAllocator alloc = this.alloc;
final Deque<ByteBuf> queue = this.queue;
final int size = this.size;
for (int i = 0; i < reps; i ++) {
queue.add(alloc.buffer(size));
queue.removeFirst().release();
}
@GenerateMicroBenchmark
public void unpooledHeapAllocAndFree_3_1024() {
ByteBuf buffer = unpooledHeapAllocator.buffer(1024);
buffer.release();
}
public enum Allocator {
UNPOOLED_HEAP {
@Override
ByteBufAllocator alloc() {
return UNPOOLED_ALLOCATOR_HEAP;
}
},
UNPOOLED_DIRECT {
@Override
ByteBufAllocator alloc() {
return UNPOOLED_ALLOCATOR_DIRECT;
}
},
POOLED_HEAP {
@Override
ByteBufAllocator alloc() {
return POOLED_ALLOCATOR_HEAP;
}
},
POOLED_DIRECT {
@Override
ByteBufAllocator alloc() {
return POOLED_ALLOCATOR_DIRECT;
}
};
@GenerateMicroBenchmark
public void unpooledHeapAllocAndFree_4_4096() {
ByteBuf buffer = unpooledHeapAllocator.buffer(4096);
buffer.release();
}
abstract ByteBufAllocator alloc();
@GenerateMicroBenchmark
public void unpooledHeapAllocAndFree_5_16384() {
ByteBuf buffer = unpooledHeapAllocator.buffer(16384);
buffer.release();
}
@GenerateMicroBenchmark
public void unpooledHeapAllocAndFree_6_65536() {
ByteBuf buffer = unpooledHeapAllocator.buffer(65536);
buffer.release();
}
@GenerateMicroBenchmark
public void unpooledDirectAllocAndFree_1_0() {
ByteBuf buffer = unpooledDirectAllocator.buffer(0);
buffer.release();
}
@GenerateMicroBenchmark
public void unpooledDirectAllocAndFree_2_256() {
ByteBuf buffer = unpooledDirectAllocator.buffer(256);
buffer.release();
}
@GenerateMicroBenchmark
public void unpooledDirectAllocAndFree_3_1024() {
ByteBuf buffer = unpooledDirectAllocator.buffer(1024);
buffer.release();
}
@GenerateMicroBenchmark
public void unpooledDirectAllocAndFree_4_4096() {
ByteBuf buffer = unpooledDirectAllocator.buffer(4096);
buffer.release();
}
@GenerateMicroBenchmark
public void unpooledDirectAllocAndFree_5_16384() {
ByteBuf buffer = unpooledDirectAllocator.buffer(16384);
buffer.release();
}
@GenerateMicroBenchmark
public void unpooledDirectAllocAndFree_6_65536() {
ByteBuf buffer = unpooledDirectAllocator.buffer(65536);
buffer.release();
}
@GenerateMicroBenchmark
public void pooledHeapAllocAndFree_1_0() {
ByteBuf buffer = pooledHeapAllocator.buffer(0);
buffer.release();
}
@GenerateMicroBenchmark
public void pooledHeapAllocAndFree_2_256() {
ByteBuf buffer = pooledHeapAllocator.buffer(256);
buffer.release();
}
@GenerateMicroBenchmark
public void pooledHeapAllocAndFree_3_1024() {
ByteBuf buffer = pooledHeapAllocator.buffer(1024);
buffer.release();
}
@GenerateMicroBenchmark
public void pooledHeapAllocAndFree_4_4096() {
ByteBuf buffer = pooledHeapAllocator.buffer(4096);
buffer.release();
}
@GenerateMicroBenchmark
public void pooledHeapAllocAndFree_5_16384() {
ByteBuf buffer = pooledHeapAllocator.buffer(16384);
buffer.release();
}
@GenerateMicroBenchmark
public void pooledHeapAllocAndFree_6_65536() {
ByteBuf buffer = pooledHeapAllocator.buffer(65536);
buffer.release();
}
@GenerateMicroBenchmark
public void pooledDirectAllocAndFree_1_0() {
ByteBuf buffer = pooledDirectAllocator.buffer(0);
buffer.release();
}
@GenerateMicroBenchmark
public void pooledDirectAllocAndFree_2_256() {
ByteBuf buffer = pooledDirectAllocator.buffer(256);
buffer.release();
}
@GenerateMicroBenchmark
public void pooledDirectAllocAndFree_3_1024() {
ByteBuf buffer = pooledDirectAllocator.buffer(1024);
buffer.release();
}
@GenerateMicroBenchmark
public void pooledDirectAllocAndFree_4_4096() {
ByteBuf buffer = pooledDirectAllocator.buffer(4096);
buffer.release();
}
@GenerateMicroBenchmark
public void pooledDirectAllocAndFree_5_16384() {
ByteBuf buffer = pooledDirectAllocator.buffer(16384);
buffer.release();
}
@GenerateMicroBenchmark
public void pooledDirectAllocAndFree_6_65536() {
ByteBuf buffer = pooledDirectAllocator.buffer(65536);
buffer.release();
}
}

View File

@ -1,20 +0,0 @@
/*
* Copyright 2013 The Netty Project
*
* The Netty Project licenses this file to you under the Apache License,
* version 2.0 (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
/**
* Benchmark classes for Netty ByteBuf
*/
package io.netty.microbench.buffer;

View File

@ -0,0 +1,90 @@
/*
* Copyright 2012 The Netty Project
*
* The Netty Project licenses this file to you under the Apache License,
* version 2.0 (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package io.netty.microbench.util;
import io.netty.util.ResourceLeakDetector;
import org.junit.Test;
import org.openjdk.jmh.output.OutputFormatType;
import org.openjdk.jmh.output.results.ResultFormatType;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;
import java.io.File;
/**
* Base class for all JMH benchmarks.
*/
public class AbstractMicrobenchmark {
protected static final String WARMUP_ITERATIONS = "10";
protected static final String MEASURE_ITERATIONS = "10";
protected static final String NUM_FORKS = "2";
protected static final String JVM_ARGS = "-server -dsa -da -ea:io.netty... -Xms768m" +
" -Xmx768m -XX:MaxDirectMemorySize=768m -XX:+AggressiveOpts -XX:+UseBiasedLocking" +
" -XX:+UseFastAccessorMethods -XX:+UseStringCache -XX:+OptimizeStringConcat" +
" -XX:+HeapDumpOnOutOfMemoryError -Dio.netty.noResourceLeakDetection";
static {
ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.DISABLED);
}
@Test
public void run() throws Exception {
String className = getClass().getSimpleName();
ChainedOptionsBuilder runnerOptions = new OptionsBuilder()
.include(".*" + className + ".*")
.jvmArgs(JVM_ARGS)
.warmupIterations(getWarmupIterations())
.measurementIterations(getMeasureIterations())
.forks(getForks());
if (getReportDir() != null) {
String filePath = getReportDir() + className + ".json";
File file = new File(filePath);
if (file.exists()) {
file.delete();
} else {
file.getParentFile().mkdirs();
file.createNewFile();
}
runnerOptions.resultFormat(ResultFormatType.JSON);
runnerOptions.result(filePath);
}
new Runner(runnerOptions.build()).run();
}
protected int getWarmupIterations() {
return Integer.parseInt(System.getProperty("warmupIterations", WARMUP_ITERATIONS));
}
protected int getMeasureIterations() {
return Integer.parseInt(System.getProperty("measureIterations", MEASURE_ITERATIONS));
}
protected int getForks() {
return Integer.parseInt(System.getProperty("forks", NUM_FORKS));
}
protected String getReportDir() {
return System.getProperty("perfReportDir", null);
}
}

View File

@ -1,129 +0,0 @@
/*
* Copyright 2012 The Netty Project
*
* The Netty Project licenses this file to you under the Apache License,
* version 2.0 (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package io.netty.microbench.util;
import com.google.caliper.CaliperRc;
import com.google.caliper.Runner;
import com.google.caliper.SimpleBenchmark;
import org.junit.Test;
import java.io.File;
import static org.junit.Assert.*;
public abstract class DefaultBenchmark extends SimpleBenchmark {
private static boolean warned;
private final int trials;
private final int warmupMillis;
private final int runMillis;
protected DefaultBenchmark() {
this(1);
}
protected DefaultBenchmark(int trials) {
this(trials, 3000, 1000);
}
protected DefaultBenchmark(int trials, int warmupMillis, int runMillis) {
this.trials = trials;
this.warmupMillis = warmupMillis;
this.runMillis = runMillis;
}
@Test
public void runBenchmarks() throws Exception {
File me = new File(DefaultBenchmark.class.getResource(
'/' + DefaultBenchmark.class.getName().replace('.', '/') + ".class").getPath());
if (!me.exists()) {
fail("failed to determine the project path");
}
File buildDir =
me.getParentFile().getParentFile().getParentFile().getParentFile().getParentFile().getParentFile();
if (!buildDir.getPath().endsWith(File.separator + "target") || !buildDir.isDirectory()) {
fail("failed to locate the build directory");
}
File reportDir = new File(buildDir.getAbsolutePath() + File.separator + "caliper-reports");
if (!reportDir.exists()) {
if (!reportDir.mkdirs()) {
fail("failed to create the Caliper report directory: " + reportDir.getAbsolutePath());
}
}
if (!reportDir.isDirectory()) {
fail("not a directory: " + reportDir.getAbsolutePath());
}
boolean deleted = deleteOldReports(reportDir);
if (!warned) {
CaliperRc caliperrc = CaliperRc.INSTANCE;
if (caliperrc.getApiKey() == null || caliperrc.getPostUrl() == null) {
warned = true;
System.out.println();
System.out.println(" Cannot read the configuration properties from .caliperrc.");
System.out.println(" Please follow the instructions at:");
System.out.println();
System.out.println(" * http://code.google.com/p/caliper/wiki/OnlineResults");
System.out.println();
System.out.println(" to upload and browse the benchmark results.");
}
}
if (deleted || warned) {
// Insert a pretty newline.
System.out.println();
}
new Runner().run(
"--trials", String.valueOf(trials),
"--warmupMillis", String.valueOf(warmupMillis),
"--runMillis", String.valueOf(runMillis),
"--saveResults", reportDir.getAbsolutePath(),
"--captureVmLog",
getClass().getName());
}
private boolean deleteOldReports(File reportDir) {
final String prefix = getClass().getName() + '.';
final String suffix = ".json";
boolean deleted = false;
for (File f: reportDir.listFiles()) {
String name = f.getName();
if (name.startsWith(prefix) && name.endsWith(suffix)) {
if (f.delete()) {
if (!deleted) {
deleted = true;
System.out.println();
}
System.out.println(" Deleted old report: " +
name.substring(prefix.length(), name.length() - suffix.length()));
}
}
}
return deleted;
}
}

View File

@ -1,20 +0,0 @@
/*
* Copyright 2013 The Netty Project
*
* The Netty Project licenses this file to you under the Apache License,
* version 2.0 (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
/**
* Util benchmark classes
*/
package io.netty.microbench.util;

View File

@ -0,0 +1,5 @@
<configuration>
<root>
<level value="INFO" />
</root>
</configuration>