Allow to parse hosts file which is stored in a different encoding then the default system encoding. (#8211)

Motivation:

We should support to parse and read a hosts file which is stored in a different encoding then the system default. Beside this when we are on windows we should just try to parse it with multiple different charset before giving up as there is no real standard what charset to use.

Modifications:

- Add more method overloads to HostsFileParser that take a Charset.
- Try to parse with multiple Charsets in DefaultHostsFileEntriesResolver when windows is used.
- Add unit test

Result:

Fixes https://github.com/netty/netty/issues/8208.

resolver/src/main/java/io/netty/resolver/DefaultHostsFileEntriesResolver.java

@@ -15,11 +15,14 @@
  */
 package io.netty.resolver;
 
+import io.netty.util.CharsetUtil;
+import io.netty.util.internal.PlatformDependent;
 import io.netty.util.internal.UnstableApi;
 
 import java.net.Inet4Address;
 import java.net.Inet6Address;
 import java.net.InetAddress;
+import java.nio.charset.Charset;
 import java.util.Locale;
 import java.util.Map;
 
@@ -33,7 +36,7 @@ public final class DefaultHostsFileEntriesResolver implements HostsFileEntriesRe
     private final Map<String, Inet6Address> inet6Entries;
 
     public DefaultHostsFileEntriesResolver() {
-        this(HostsFileParser.parseSilently());
+        this(parseEntries());
     }
 
     // for testing purpose only
@@ -65,4 +68,14 @@ public final class DefaultHostsFileEntriesResolver implements HostsFileEntriesRe
     String normalize(String inetHost) {
         return inetHost.toLowerCase(Locale.ENGLISH);
     }
+
+    private static HostsFileEntries parseEntries() {
+        if (PlatformDependent.isWindows()) {
+            // Ony windows there seems to be no standard for the encoding used for the hosts file, so let us
+            // try multiple until we either were able to parse it or there is none left and so we return an
+            // empty intstance.
+            return HostsFileParser.parseSilently(Charset.defaultCharset(), CharsetUtil.UTF_16, CharsetUtil.UTF_8);
+        }
+        return HostsFileParser.parseSilently();
+    }
 }

resolver/src/main/java/io/netty/resolver/HostsFileParser.java

@@ -23,12 +23,14 @@ import io.netty.util.internal.logging.InternalLoggerFactory;
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileReader;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.Reader;
 import java.net.Inet4Address;
 import java.net.Inet6Address;
 import java.net.InetAddress;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
@@ -66,14 +68,25 @@ public final class HostsFileParser {
     }
 
     /**
-     * Parse hosts file at standard OS location.
+     * Parse hosts file at standard OS location using the systems default {@link Charset} for decoding.
      *
      * @return a {@link HostsFileEntries}
      */
     public static HostsFileEntries parseSilently() {
+        return parseSilently(Charset.defaultCharset());
+    }
+
+    /**
+     * Parse hosts file at standard OS location using the given {@link Charset}s one after each other until
+     * we were able to parse something or none is left.
+     *
+     * @param charsets the {@link Charset}s to try as file encodings when parsing.
+     * @return a {@link HostsFileEntries}
+     */
+    public static HostsFileEntries parseSilently(Charset... charsets) {
         File hostsFile = locateHostsFile();
         try {
-            return parse(hostsFile);
+            return parse(hostsFile, charsets);
         } catch (IOException e) {
             if (logger.isWarnEnabled()) {
                 logger.warn("Failed to load and parse hosts file at " + hostsFile.getPath(), e);
@@ -83,7 +96,7 @@ public final class HostsFileParser {
     }
 
     /**
-     * Parse hosts file at standard OS location.
+     * Parse hosts file at standard OS location using the system default {@link Charset} for decoding.
      *
      * @return a {@link HostsFileEntries}
      * @throws IOException file could not be read
@@ -93,19 +106,37 @@ public final class HostsFileParser {
     }
 
     /**
-     * Parse a hosts file.
+     * Parse a hosts file using the system default {@link Charset} for decoding.
      *
      * @param file the file to be parsed
      * @return a {@link HostsFileEntries}
      * @throws IOException file could not be read
      */
     public static HostsFileEntries parse(File file) throws IOException {
-        checkNotNull(file, "file");
-        if (file.exists() && file.isFile()) {
-            return parse(new BufferedReader(new FileReader(file)));
-        } else {
-            return HostsFileEntries.EMPTY;
+        return parse(file, Charset.defaultCharset());
     }
+
+    /**
+     * Parse a hosts file.
+     *
+     * @param file the file to be parsed
+     * @param charsets the {@link Charset}s to try as file encodings when parsing.
+     * @return a {@link HostsFileEntries}
+     * @throws IOException file could not be read
+     */
+    public static HostsFileEntries parse(File file, Charset... charsets) throws IOException {
+        checkNotNull(file, "file");
+        checkNotNull(charsets, "charsets");
+        if (file.exists() && file.isFile()) {
+            for (Charset charset: charsets) {
+                HostsFileEntries entries = parse(new BufferedReader(new InputStreamReader(
+                        new FileInputStream(file), charset)));
+                if (entries != HostsFileEntries.EMPTY) {
+                    return entries;
+                }
+            }
+        }
+        return HostsFileEntries.EMPTY;
     }
 
     /**

resolver/src/test/java/io/netty/resolver/HostsFileParserTest.java

@@ -15,13 +15,18 @@
  */
 package io.netty.resolver;
 
+import io.netty.util.CharsetUtil;
+import org.junit.Assume;
 import org.junit.Test;
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
 import java.net.Inet4Address;
 import java.net.Inet6Address;
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
 import java.util.Map;
 
 import static org.junit.Assert.*;
@@ -60,4 +65,41 @@ public class HostsFileParserTest {
         assertEquals("192.168.0.5", inet4Entries.get("host7").getHostAddress());
         assertEquals("0:0:0:0:0:0:0:1", inet6Entries.get("host1").getHostAddress());
     }
+
+    @Test
+    public void testParseUnicode() throws IOException {
+        final Charset unicodeCharset;
+        try {
+            unicodeCharset = Charset.forName("unicode");
+        } catch (UnsupportedCharsetException e) {
+            Assume.assumeNoException(e);
+            return;
+        }
+        testParseFile(HostsFileParser.parse(
+                new File(getClass().getResource("hosts-unicode").getFile()), unicodeCharset));
+    }
+
+    @Test
+    public void testParseMultipleCharsets() throws IOException {
+        final Charset unicodeCharset;
+        try {
+            unicodeCharset = Charset.forName("unicode");
+        } catch (UnsupportedCharsetException e) {
+            Assume.assumeNoException(e);
+            return;
+        }
+        testParseFile(HostsFileParser.parse(new File(getClass().getResource("hosts-unicode").getFile()),
+                                            CharsetUtil.UTF_8, CharsetUtil.ISO_8859_1, unicodeCharset));
+    }
+
+    private static void testParseFile(HostsFileEntries entries) throws IOException {
+        Map<String, Inet4Address> inet4Entries = entries.inet4Entries();
+        Map<String, Inet6Address> inet6Entries = entries.inet6Entries();
+
+        assertEquals("Expected 2 IPv4 entries", 2, inet4Entries.size());
+        assertEquals("Expected 1 IPv6 entries", 1, inet6Entries.size());
+        assertEquals("127.0.0.1", inet4Entries.get("localhost").getHostAddress());
+        assertEquals("255.255.255.255", inet4Entries.get("broadcasthost").getHostAddress());
+        assertEquals("0:0:0:0:0:0:0:1", inet6Entries.get("localhost").getHostAddress());
+    }
 }