Fix a problem with IP protocol version confusion on MacOS when TCP FastOpen is enabled (#11588)

Motivation:
This fixes a bug that would result in an `io.netty.channel.unix.Errors$NativeIoException: connectx(..) failed: Address family not supported by protocol family` error.
This happens when the connecting socket is configured to use IPv6 but the address being connected to is IPv4.
This can occur because, for instance, Netty and `InetAddress.getLoopbackAddress()` have different preferences for IPv6 vs. IPv4.

Modification:
Pass the correct ipv6 or ipv4 flags to connectx, depending on whether the socket was created for AF_INET or AF_INET6, rather than relying on the IP version of the destination address.

Result:
No more issue with TCP FastOpen on MacOS when using addresses of the "wrong" IP version.

testsuite/src/main/java/io/netty/testsuite/transport/socket/AbstractSocketTest.java

@@ -34,11 +34,11 @@ public abstract class AbstractSocketTest extends AbstractComboTestsuiteTest<Serv
     }
 
     @Override
-    protected void configure(ServerBootstrap bootstrap, Bootstrap bootstrap2, ByteBufAllocator allocator) {
-        bootstrap.localAddress(newSocketAddress());
-        bootstrap.option(ChannelOption.ALLOCATOR, allocator);
-        bootstrap.childOption(ChannelOption.ALLOCATOR, allocator);
-        bootstrap2.option(ChannelOption.ALLOCATOR, allocator);
+    protected void configure(ServerBootstrap sb, Bootstrap cb, ByteBufAllocator allocator) {
+        sb.localAddress(newSocketAddress());
+        sb.option(ChannelOption.ALLOCATOR, allocator);
+        sb.childOption(ChannelOption.ALLOCATOR, allocator);
+        cb.option(ChannelOption.ALLOCATOR, allocator);
     }
 
     protected SocketAddress newSocketAddress() {

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainDatagramPathTest.java

@@ -41,7 +41,7 @@ class EpollDomainDatagramPathTest extends AbstractClientSocketTest {
             public void run(Bootstrap bootstrap) {
                 try {
                     bootstrap.handler(new ChannelInboundHandlerAdapter())
-                             .connect(EpollSocketTestPermutation.newSocketAddress()).sync().channel();
+                             .connect(EpollSocketTestPermutation.newDomainSocketAddress()).sync().channel();
                     fail("Expected FileNotFoundException");
                 } catch (Exception e) {
                     assertTrue(e instanceof FileNotFoundException);
@@ -57,10 +57,10 @@ class EpollDomainDatagramPathTest extends AbstractClientSocketTest {
             public void run(Bootstrap bootstrap) {
                 try {
                     Channel ch = bootstrap.handler(new ChannelInboundHandlerAdapter())
-                                          .bind(EpollSocketTestPermutation.newSocketAddress()).sync().channel();
+                                          .bind(EpollSocketTestPermutation.newDomainSocketAddress()).sync().channel();
                     ch.writeAndFlush(new DomainDatagramPacket(
                             Unpooled.copiedBuffer("test", CharsetUtil.US_ASCII),
-                            EpollSocketTestPermutation.newSocketAddress())).sync();
+                            EpollSocketTestPermutation.newDomainSocketAddress())).sync();
                     fail("Expected FileNotFoundException");
                 } catch (Exception e) {
                     assertTrue(e instanceof FileNotFoundException);

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainDatagramUnicastTest.java

@@ -80,7 +80,7 @@ class EpollDomainDatagramUnicastTest extends DatagramUnicastTest {
 
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketDataReadInitialStateTest.java

@@ -26,7 +26,7 @@ import java.util.List;
 public class EpollDomainSocketDataReadInitialStateTest extends SocketDataReadInitialStateTest {
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketEchoTest.java

@@ -25,7 +25,7 @@ import java.util.List;
 public class EpollDomainSocketEchoTest extends EpollSocketEchoTest {
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketFdTest.java

@@ -43,7 +43,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 public class EpollDomainSocketFdTest extends AbstractSocketTest {
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketFileRegionTest.java

@@ -25,7 +25,7 @@ import java.util.List;
 public class EpollDomainSocketFileRegionTest extends EpollSocketFileRegionTest {
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketFixedLengthEchoTest.java

@@ -27,7 +27,7 @@ public class EpollDomainSocketFixedLengthEchoTest extends SocketFixedLengthEchoT
 
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketGatheringWriteTest.java

@@ -27,7 +27,7 @@ public class EpollDomainSocketGatheringWriteTest extends SocketGatheringWriteTes
 
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketObjectEchoTest.java

@@ -26,7 +26,7 @@ import java.util.List;
 public class EpollDomainSocketObjectEchoTest extends SocketObjectEchoTest {
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketReuseFdTest.java

@@ -26,7 +26,7 @@ import java.util.List;
 public class EpollDomainSocketReuseFdTest extends AbstractSocketReuseFdTest {
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketShutdownOutputByPeerTest.java

@@ -35,7 +35,7 @@ public class EpollDomainSocketShutdownOutputByPeerTest extends AbstractSocketShu
 
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketSslClientRenegotiateTest.java

@@ -31,6 +31,6 @@ public class EpollDomainSocketSslClientRenegotiateTest extends SocketSslClientRe
 
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 }

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketSslEchoTest.java

@@ -26,7 +26,7 @@ import java.util.List;
 public class EpollDomainSocketSslEchoTest extends SocketSslEchoTest {
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketSslGreetingTest.java

@@ -26,7 +26,7 @@ import java.util.List;
 public class EpollDomainSocketSslGreetingTest extends SocketSslGreetingTest {
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketStartTlsTest.java

@@ -26,7 +26,7 @@ import java.util.List;
 public class EpollDomainSocketStartTlsTest extends SocketStartTlsTest {
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollDomainSocketStringEchoTest.java

@@ -26,7 +26,7 @@ import java.util.List;
 public class EpollDomainSocketStringEchoTest extends SocketStringEchoTest {
     @Override
     protected SocketAddress newSocketAddress() {
-        return EpollSocketTestPermutation.newSocketAddress();
+        return EpollSocketTestPermutation.newDomainSocketAddress();
     }
 
     @Override

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollJdkLoopbackSocketSslEchoTest.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2021 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.channel.epoll;
+
+import io.netty.channel.unix.tests.UnixTestUtils;
+
+import java.net.SocketAddress;
+
+public class EpollJdkLoopbackSocketSslEchoTest extends EpollSocketSslEchoTest {
+    @Override
+    protected SocketAddress newSocketAddress() {
+        return UnixTestUtils.newInetLoopbackSocketAddress();
+    }
+}

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollSocketTest.java

@@ -47,7 +47,7 @@ public class EpollSocketTest extends SocketTest<LinuxSocket> {
         LinuxSocket s2 = LinuxSocket.newSocketDomain();
 
         try {
-            DomainSocketAddress dsa = UnixTestUtils.newSocketAddress();
+            DomainSocketAddress dsa = UnixTestUtils.newDomainSocketAddress();
             s1.bind(dsa);
             s1.listen(1);
 

transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollSocketTestPermutation.java

@@ -259,7 +259,7 @@ class EpollSocketTestPermutation extends SocketTestPermutation {
         );
     }
 
-    public static DomainSocketAddress newSocketAddress() {
-        return UnixTestUtils.newSocketAddress();
+    public static DomainSocketAddress newDomainSocketAddress() {
+        return UnixTestUtils.newDomainSocketAddress();
     }
 }

transport-native-kqueue/src/main/java/io/netty/channel/kqueue/BsdSocket.java

@@ -125,8 +125,8 @@ final class BsdSocket extends Socket {
             sourcePort = 0;
         } else {
             InetAddress sourceInetAddress = source.getAddress();
-            sourceIPv6 = sourceInetAddress instanceof Inet6Address;
-            if (sourceIPv6) {
+            sourceIPv6 = useIpv6(this, sourceInetAddress);
+            if (sourceInetAddress instanceof Inet6Address) {
                 sourceAddress = sourceInetAddress.getAddress();
                 sourceScopeId = ((Inet6Address) sourceInetAddress).getScopeId();
             } else {
@@ -138,10 +138,10 @@ final class BsdSocket extends Socket {
         }
 
         InetAddress destinationInetAddress = destination.getAddress();
-        boolean destinationIPv6 = destinationInetAddress instanceof Inet6Address;
+        boolean destinationIPv6 = useIpv6(this, destinationInetAddress);
         byte[] destinationAddress;
         int destinationScopeId;
-        if (destinationIPv6) {
+        if (destinationInetAddress instanceof Inet6Address) {
             destinationAddress = destinationInetAddress.getAddress();
             destinationScopeId = ((Inet6Address) destinationInetAddress).getScopeId();
         } else {

transport-native-kqueue/src/test/java/io/netty/channel/kqueue/KQueueJdkLoopbackSocketSslEchoTest.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2021 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.channel.kqueue;
+
+import io.netty.channel.unix.tests.UnixTestUtils;
+
+import java.net.SocketAddress;
+
+public class KQueueJdkLoopbackSocketSslEchoTest extends KQueueSocketSslEchoTest {
+    @Override
+    protected SocketAddress newSocketAddress() {
+        return UnixTestUtils.newInetLoopbackSocketAddress();
+    }
+}

transport-native-kqueue/src/test/java/io/netty/channel/kqueue/KQueueSocketTest.java

@@ -40,7 +40,7 @@ public class KQueueSocketTest extends SocketTest<BsdSocket> {
         BsdSocket s2 = BsdSocket.newSocketDomain();
 
         try {
-            DomainSocketAddress dsa = UnixTestUtils.newSocketAddress();
+            DomainSocketAddress dsa = UnixTestUtils.newDomainSocketAddress();
             s1.bind(dsa);
             s1.listen(1);
 
@@ -61,7 +61,7 @@ public class KQueueSocketTest extends SocketTest<BsdSocket> {
         BsdSocket s2 = BsdSocket.newSocketDomain();
 
         try {
-            DomainSocketAddress dsa = UnixTestUtils.newSocketAddress();
+            DomainSocketAddress dsa = UnixTestUtils.newDomainSocketAddress();
             s1.bind(dsa);
             s1.listen(1);
 

transport-native-kqueue/src/test/java/io/netty/channel/kqueue/KQueueSocketTestPermutation.java

@@ -50,7 +50,7 @@ class KQueueSocketTestPermutation extends SocketTestPermutation {
     public List<TestsuitePermutation.BootstrapComboFactory<ServerBootstrap, Bootstrap>> socket() {
 
         List<TestsuitePermutation.BootstrapComboFactory<ServerBootstrap, Bootstrap>> list =
-                combo(serverSocket(), clientSocket());
+                combo(serverSocket(), clientSocketWithFastOpen());
 
         list.remove(list.size() - 1); // Exclude NIO x NIO test
 
@@ -202,6 +202,6 @@ class KQueueSocketTestPermutation extends SocketTestPermutation {
     }
 
     public static DomainSocketAddress newSocketAddress() {
-        return UnixTestUtils.newSocketAddress();
+        return UnixTestUtils.newDomainSocketAddress();
     }
 }

transport-native-unix-common-tests/src/main/java/io/netty/channel/unix/tests/UnixTestUtils.java

@@ -20,9 +20,26 @@ import io.netty.util.internal.PlatformDependent;
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 public final class UnixTestUtils {
+    private static final Object INET_LOOPBACK_UNAVAILABLE = new Object();
+    private static volatile Object inetLoopbackCache;
+
+    /**
+     * @deprecated Use {@link #newDomainSocketAddress()} instead.
+     */
+    @Deprecated
     public static DomainSocketAddress newSocketAddress() {
+        return newDomainSocketAddress();
+    }
+
+    public static DomainSocketAddress newDomainSocketAddress() {
         try {
             File file;
             do {
@@ -37,5 +54,30 @@ public final class UnixTestUtils {
         }
     }
 
+    /**
+     * The JDK method may produce IPv4 loopback addresses where {@link io.netty.util.NetUtil#LOCALHOST} might be an
+     * IPv6 addresses.
+     * This difference can stress the system in different ways that are important to test.
+     */
+    public static SocketAddress newInetLoopbackSocketAddress() {
+        Object loopback = inetLoopbackCache;
+
+        if (loopback == null) {
+            inetLoopbackCache = loopback = getLoopbackAddress();
+        }
+
+        assumeTrue(loopback != INET_LOOPBACK_UNAVAILABLE, "InetAddress.getLoopbackAddress() is not available");
+        return new InetSocketAddress((InetAddress) loopback, 0);
+    }
+
+    private static Object getLoopbackAddress() {
+        try {
+            Method method = InetAddress.class.getMethod("getLoopbackAddress");
+            return method.invoke(null);
+        } catch (Exception ignore) {
+            return INET_LOOPBACK_UNAVAILABLE;
+        }
+    }
+
     private UnixTestUtils() { }
 }

transport-native-unix-common/src/main/java/io/netty/channel/unix/Socket.java

@@ -59,7 +59,15 @@ public class Socket extends FileDescriptor {
      * Returns {@code true} if we should use IPv6 internally, {@code false} otherwise.
      */
     private boolean useIpv6(InetAddress address) {
-        return ipv6 || address instanceof Inet6Address;
+        return useIpv6(this, address);
+    }
+
+    /**
+     * Returns {@code true} if the given socket and address combination should use IPv6 internally,
+     * {@code false} otherwise.
+     */
+    protected static boolean useIpv6(Socket socket, InetAddress address) {
+        return socket.ipv6 || address instanceof Inet6Address;
     }
 
     public final void shutdown() throws IOException {