Don't cause ClassCastException if registration fails during constructing DnsNameResolver. (#8280)

Motivation:

We should not try to cast the Channel to a DatagramChannel as this will cause a ClassCastException.

Modifications:

- Do not cast
- rethrow from constructor if we detect the registration failed.
- Add unit test.

Result:

Propagate correct exception.

resolver-dns/src/main/java/io/netty/resolver/dns/DnsNameResolver.java

@@ -152,7 +152,7 @@ public class DnsNameResolver extends InetNameResolver {
     private static final DatagramDnsQueryEncoder ENCODER = new DatagramDnsQueryEncoder();
 
     final Future<Channel> channelFuture;
-    final DatagramChannel ch;
+    final Channel ch;
 
     // Comparator that ensures we will try first to use the nameservers that use our preferred address type.
     private final Comparator<InetSocketAddress> nameServerComparator;
@@ -356,7 +356,18 @@ public class DnsNameResolver extends InetNameResolver {
         });
 
         channelFuture = responseHandler.channelActivePromise;
-        ch = (DatagramChannel) b.register().channel();
+        ChannelFuture future = b.register();
+        Throwable cause = future.cause();
+        if (cause != null) {
+            if (cause instanceof RuntimeException) {
+                throw (RuntimeException) cause;
+            }
+            if (cause instanceof Error) {
+                throw (Error) cause;
+            }
+            throw new IllegalStateException("Unable to create / register Channel", cause);
+        }
+        ch = future.channel();
         ch.config().setRecvByteBufAllocator(new FixedRecvByteBufAllocator(maxPayloadSize));
 
         ch.closeFuture().addListener(new ChannelFutureListener() {

resolver-dns/src/test/java/io/netty/resolver/dns/DnsNameResolverTest.java

@@ -2270,4 +2270,20 @@ public class DnsNameResolverTest {
             server.stop();
         }
     }
+
+    @Test
+    public void testChannelFactoryException() {
+        final IllegalStateException exception = new IllegalStateException();
+        try {
+            newResolver().channelFactory(new ChannelFactory<DatagramChannel>() {
+                @Override
+                public DatagramChannel newChannel() {
+                    throw exception;
+                }
+            }).build();
+            fail();
+        } catch (Exception e) {
+            assertSame(exception, e);
+        }
+    }
 }