From 3cd94822f5b687700d1fa088f1638e14f6d1953b Mon Sep 17 00:00:00 2001 From: Trustin Lee Date: Fri, 12 Dec 2014 10:30:57 +0900 Subject: [PATCH] Generate heap and thread dump when some tests fail Motivation: We have a few sporadic test failures which are only easily reproduceable in our CI machine. To get more information about the failure, we need heap and full thread dump at the moment of failure. Modifications: - Add TestUtils.dump() method to dump heap and threads - Modify SocketGatheringWriteTest and SocketSslEchoTest to call TestUtils.dump() on failure Result: We get more information about the test failure. --- .../socket/SocketGatheringWriteTest.java | 27 ++-- .../transport/socket/SocketSslEchoTest.java | 20 ++- .../io/netty/testsuite/util/TestUtils.java | 118 +++++++++++++++++- 3 files changed, 147 insertions(+), 18 deletions(-) diff --git a/testsuite/src/test/java/io/netty/testsuite/transport/socket/SocketGatheringWriteTest.java b/testsuite/src/test/java/io/netty/testsuite/transport/socket/SocketGatheringWriteTest.java index c84f71635d..2553c1510d 100644 --- a/testsuite/src/test/java/io/netty/testsuite/transport/socket/SocketGatheringWriteTest.java +++ b/testsuite/src/test/java/io/netty/testsuite/transport/socket/SocketGatheringWriteTest.java @@ -21,8 +21,11 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.CompositeByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.testsuite.util.TestUtils; +import io.netty.util.internal.StringUtil; import org.junit.Test; import java.io.IOException; @@ -40,7 +43,7 @@ public class SocketGatheringWriteTest extends AbstractSocketTest { random.nextBytes(data); } - @Test(timeout = 30000) + @Test(timeout = 60000) public void testGatheringWrite() throws Throwable { run(); } @@ -49,7 +52,7 @@ public class SocketGatheringWriteTest extends AbstractSocketTest { testGatheringWrite0(sb, cb, data, false, true); } - @Test(timeout = 30000) + @Test(timeout = 60000) public void testGatheringWriteNotAutoRead() throws Throwable { run(); } @@ -58,7 +61,7 @@ public class SocketGatheringWriteTest extends AbstractSocketTest { testGatheringWrite0(sb, cb, data, false, false); } - @Test(timeout = 30000) + @Test(timeout = 60000) public void testGatheringWriteWithComposite() throws Throwable { run(); } @@ -67,7 +70,7 @@ public class SocketGatheringWriteTest extends AbstractSocketTest { testGatheringWrite0(sb, cb, data, true, false); } - @Test(timeout = 30000) + @Test(timeout = 60000) public void testGatheringWriteWithCompositeNotAutoRead() throws Throwable { run(); } @@ -77,7 +80,7 @@ public class SocketGatheringWriteTest extends AbstractSocketTest { } // Test for https://github.com/netty/netty/issues/2647 - @Test(timeout = 30000) + @Test(timeout = 60000) public void testGatheringWriteBig() throws Throwable { run(); } @@ -88,7 +91,7 @@ public class SocketGatheringWriteTest extends AbstractSocketTest { testGatheringWrite0(sb, cb, bigData, false, true); } - private static void testGatheringWrite0( + private void testGatheringWrite0( ServerBootstrap sb, Bootstrap cb, byte[] data, boolean composite, boolean autoRead) throws Throwable { final TestHandler sh = new TestHandler(autoRead); final TestHandler ch = new TestHandler(autoRead); @@ -116,7 +119,17 @@ public class SocketGatheringWriteTest extends AbstractSocketTest { } i += length; } - assertNotEquals(cc.voidPromise(), cc.writeAndFlush(Unpooled.EMPTY_BUFFER).sync()); + + ChannelFuture cf = cc.writeAndFlush(Unpooled.EMPTY_BUFFER); + assertNotEquals(cc.voidPromise(), cf); + try { + assertTrue(cf.await(30000)); + cf.sync(); + } catch (Throwable t) { + // TODO: Remove this once we fix this test. + TestUtils.dump(StringUtil.simpleClassName(this)); + throw t; + } while (sh.counter < data.length) { if (sh.exception.get() != null) { diff --git a/testsuite/src/test/java/io/netty/testsuite/transport/socket/SocketSslEchoTest.java b/testsuite/src/test/java/io/netty/testsuite/transport/socket/SocketSslEchoTest.java index 394fbe8314..6e6d16edc6 100644 --- a/testsuite/src/test/java/io/netty/testsuite/transport/socket/SocketSslEchoTest.java +++ b/testsuite/src/test/java/io/netty/testsuite/transport/socket/SocketSslEchoTest.java @@ -34,7 +34,9 @@ import io.netty.handler.ssl.SslHandler; import io.netty.handler.ssl.SslHandshakeCompletionEvent; import io.netty.handler.ssl.util.SelfSignedCertificate; import io.netty.handler.stream.ChunkedWriteHandler; +import io.netty.testsuite.util.TestUtils; import io.netty.util.concurrent.Future; +import io.netty.util.internal.StringUtil; import io.netty.util.internal.logging.InternalLogger; import io.netty.util.internal.logging.InternalLoggerFactory; import org.junit.Test; @@ -285,12 +287,18 @@ public class SocketSslEchoTest extends AbstractSocketTest { } // When renegotiation is done, both the client and server side should be notified. - if (renegotiationType != RenegotiationType.NONE) { - assertThat(sh.negoCounter, is(2)); - assertThat(ch.negoCounter, is(2)); - } else { - assertThat(sh.negoCounter, is(1)); - assertThat(ch.negoCounter, is(1)); + try { + if (renegotiationType != RenegotiationType.NONE) { + assertThat(sh.negoCounter, is(2)); + assertThat(ch.negoCounter, is(2)); + } else { + assertThat(sh.negoCounter, is(1)); + assertThat(ch.negoCounter, is(1)); + } + } catch (Throwable t) { + // TODO: Remove this once we fix this test. + TestUtils.dump(StringUtil.simpleClassName(this)); + throw t; } } diff --git a/testsuite/src/test/java/io/netty/testsuite/util/TestUtils.java b/testsuite/src/test/java/io/netty/testsuite/util/TestUtils.java index f00e7ee4fb..9f63b54cb7 100644 --- a/testsuite/src/test/java/io/netty/testsuite/util/TestUtils.java +++ b/testsuite/src/test/java/io/netty/testsuite/util/TestUtils.java @@ -15,22 +15,36 @@ */ package io.netty.testsuite.util; +import io.netty.util.CharsetUtil; import io.netty.util.NetUtil; +import io.netty.util.internal.logging.InternalLogger; +import io.netty.util.internal.logging.InternalLoggerFactory; import org.junit.rules.TestName; +import javax.management.MBeanServer; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.reflect.Method; import java.net.DatagramSocket; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.nio.channels.Channel; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Locale; public final class TestUtils { + private static final InternalLogger logger = InternalLoggerFactory.getInstance(TestUtils.class); + private static final int START_PORT = 32768; private static final int END_PORT = 65536; private static final int NUM_CANDIDATES = END_PORT - START_PORT; @@ -38,18 +52,32 @@ public final class TestUtils { private static final List PORTS = new ArrayList(); private static Iterator portIterator; + private static final Method hotspotMXBeanDumpHeap; + private static final Object hotspotMXBean; + static { + // Populate the list of random ports. for (int i = START_PORT; i < END_PORT; i ++) { PORTS.add(i); } Collections.shuffle(PORTS); - } - private static int nextCandidatePort() { - if (portIterator == null || !portIterator.hasNext()) { - portIterator = PORTS.iterator(); + // Retrieve the hotspot MXBean and its class if available. + Object mxBean; + Method mxBeanDumpHeap; + try { + Class clazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean"); + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + mxBean = ManagementFactory.newPlatformMXBeanProxy( + server, "com.sun.management:type=HotSpotDiagnostic", clazz); + mxBeanDumpHeap = clazz.getMethod("dumpHeap", String.class, boolean.class); + } catch (Exception ignored) { + mxBean = null; + mxBeanDumpHeap = null; } - return portIterator.next(); + + hotspotMXBean = mxBean; + hotspotMXBeanDumpHeap = mxBeanDumpHeap; } /** @@ -75,6 +103,13 @@ public final class TestUtils { throw new RuntimeException("unable to find a free port"); } + private static int nextCandidatePort() { + if (portIterator == null || !portIterator.hasNext()) { + portIterator = PORTS.iterator(); + } + return portIterator.next(); + } + private static boolean isTcpPortAvailable(InetSocketAddress localAddress) { ServerSocket ss = null; try { @@ -162,5 +197,78 @@ public final class TestUtils { return testMethodName; } + public static void dump(String filenamePrefix) throws IOException { + + if (filenamePrefix == null) { + throw new NullPointerException("filenamePrefix"); + } + + final String timestamp = timestamp(); + final File heapDumpFile = new File(filenamePrefix + '.' + timestamp + ".hprof"); + if (heapDumpFile.exists()) { + if (!heapDumpFile.delete()) { + throw new IOException("Failed to remove the old heap dump: " + heapDumpFile); + } + } + + final File threadDumpFile = new File(filenamePrefix + '.' + timestamp + ".threads"); + if (threadDumpFile.exists()) { + if (!threadDumpFile.delete()) { + throw new IOException("Failed to remove the old thread dump: " + threadDumpFile); + } + } + + dumpHeap(heapDumpFile); + dumpStack(threadDumpFile); + } + + private static String timestamp() { + return new SimpleDateFormat("HHmmss.SSS").format(new Date()); + } + + private static void dumpHeap(File file) { + if (hotspotMXBean == null) { + logger.warn("Can't dump heap: HotSpotDiagnosticMXBean unavailable"); + return; + } + + final String filename = file.toString(); + try { + hotspotMXBeanDumpHeap.invoke(hotspotMXBean, filename, true); + } catch (Exception e) { + logger.warn("Failed to dump heap: {}", filename, e); + } + } + + private static void dumpStack(File file) { + final String filename = file.toString(); + OutputStream out = null; + try { + final StringBuilder buf = new StringBuilder(8192); + try { + for (ThreadInfo info : ManagementFactory.getThreadMXBean().dumpAllThreads(true, true)) { + buf.append(info); + } + buf.append('\n'); + } catch (UnsupportedOperationException ignored) { + logger.warn("Can't dump threads: ThreadMXBean.dumpAllThreads() unsupported"); + return; + } + + out = new FileOutputStream(file); + out.write(buf.toString().getBytes(CharsetUtil.UTF_8)); + } catch (Exception e) { + logger.warn("Failed to dump threads: {}", filename, e); + } finally { + if (out != null) { + try { + out.close(); + } catch (IOException ignored) { + // Ignore. + } + } + } + } + private TestUtils() { } }