Fix handling of non direct backed PemEncoded.

Motivation:

The private key and certificate that are passed into #serKeyMaterial() could be PemEncoded in which case the #toPEM() methods return the identity of the value.

That in turn will fail in the #toBIO() step because the underlying ByteBuf is not necessarily direct.

Modifications:

- Use toBIO(...) which also works with non direct PemEncoded values
- Add unit test.

Result:

Correct handling of PemEncoded.

handler/src/main/java/io/netty/handler/ssl/OpenSslKeyMaterialManager.java

@@ -29,7 +29,6 @@ import java.util.Map;
 import java.util.Set;
 
 import static io.netty.handler.ssl.ReferenceCountedOpenSslContext.freeBio;
-import static io.netty.handler.ssl.ReferenceCountedOpenSslContext.newBIO;
 import static io.netty.handler.ssl.ReferenceCountedOpenSslContext.toBIO;
 
 /**
@@ -104,8 +103,8 @@ class OpenSslKeyMaterialManager {
                 // Only encode one time
                 PemEncoded encoded = PemX509Certificate.toPEM(ByteBufAllocator.DEFAULT, true, certificates);
                 try {
-                    keyCertChainBio = newBIO(encoded.content().retainedSlice());
+                    keyCertChainBio = toBIO(ByteBufAllocator.DEFAULT, encoded.retain());
-                    keyCertChainBio2 = newBIO(encoded.content().retainedSlice());
+                    keyCertChainBio2 = toBIO(ByteBufAllocator.DEFAULT, encoded.retain());
 
                     if (key != null) {
                         keyBio = toBIO(key);

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslContext.java

@@ -642,8 +642,8 @@ public abstract class ReferenceCountedOpenSslContext extends SslContext implemen
         try {
             // Only encode one time
             encoded = PemX509Certificate.toPEM(ByteBufAllocator.DEFAULT, true, keyCertChain);
-            keyCertChainBio = newBIO(encoded.content().retainedSlice());
+            keyCertChainBio = toBIO(ByteBufAllocator.DEFAULT, encoded.retain());
-            keyCertChainBio2 = newBIO(encoded.content().retainedSlice());
+            keyCertChainBio2 = toBIO(ByteBufAllocator.DEFAULT, encoded.retain());
 
             if (key != null) {
                 keyBio = toBIO(key);
@@ -714,7 +714,7 @@ public abstract class ReferenceCountedOpenSslContext extends SslContext implemen
         }
     }
 
-    private static long toBIO(ByteBufAllocator allocator, PemEncoded pem) throws Exception {
+    static long toBIO(ByteBufAllocator allocator, PemEncoded pem) throws Exception {
         try {
             // We can turn direct buffers straight into BIOs. No need to
             // make a yet another copy.
@@ -744,7 +744,7 @@ public abstract class ReferenceCountedOpenSslContext extends SslContext implemen
         }
     }
 
-    static long newBIO(ByteBuf buffer) throws Exception {
+    private static long newBIO(ByteBuf buffer) throws Exception {
         try {
             long bio = SSL.newMemBIO();
             int readable = buffer.readableBytes();

handler/src/test/java/io/netty/handler/ssl/PemEncodedTest.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2016 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.netty.handler.ssl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+
+import org.junit.Test;
+
+import io.netty.handler.ssl.util.SelfSignedCertificate;
+import io.netty.util.ReferenceCountUtil;
+
+public class PemEncodedTest {
+
+    @Test
+    public void testPemEncodedOpenSsl() throws Exception {
+        testPemEncoded(SslProvider.OPENSSL);
+    }
+
+    @Test
+    public void testPemEncodedOpenSslRef() throws Exception {
+        testPemEncoded(SslProvider.OPENSSL_REFCNT);
+    }
+
+    private static void testPemEncoded(SslProvider provider) throws Exception {
+        assumeTrue(OpenSsl.isAvailable());
+        PemPrivateKey pemKey;
+        PemX509Certificate pemCert;
+        SelfSignedCertificate ssc = new SelfSignedCertificate();
+        try {
+            pemKey = PemPrivateKey.valueOf(toByteArray(ssc.privateKey()));
+            pemCert = PemX509Certificate.valueOf(toByteArray(ssc.certificate()));
+        } finally {
+            ssc.delete();
+        }
+
+        SslContext context = SslContextBuilder.forServer(pemKey, pemCert)
+                .sslProvider(provider)
+                .build();
+        assertEquals(1, pemKey.refCnt());
+        assertEquals(1, pemCert.refCnt());
+        try {
+            assertTrue(context instanceof ReferenceCountedOpenSslContext);
+        } finally {
+            ReferenceCountUtil.release(context);
+            assertRelease(pemKey);
+            assertRelease(pemCert);
+        }
+    }
+
+    private static void assertRelease(PemEncoded encoded) {
+        assertTrue(encoded.release());
+    }
+    private static byte[] toByteArray(File file) throws Exception {
+        FileInputStream in = new FileInputStream(file);
+        try {
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();
+            try {
+                byte[] buf = new byte[1024];
+                int len;
+                while ((len = in.read(buf)) != -1) {
+                    baos.write(buf, 0, len);
+                }
+            } finally {
+                baos.close();
+            }
+
+            return baos.toByteArray();
+        } finally {
+            in.close();
+        }
+    }
+}