HTTP2: Add protection against remote control frames that are triggered by a remote peer (#9460)

Motivation:

Due how http2 spec is defined it is possible by a remote peer to flood us with frames that will trigger control frames as response, the problem here is that the remote peer can also just stop reading these (while still produce more of these) and so may drive us to the pointer where we either run out of memory or burn all CPU. To protect against this we need to implement some kind of limit that will tear down connections that cause the above mentioned situation.

See CVE-2019-9512 / CVE-2019-9514 / CVE-2019-9515

Modifications:

- Add Http2ControlFrameLimitEncoder which limits the number of queued control frames that were caused because of the remote peer.
- Allow to insert ths Http2ControlFrameLimitEncoder by setting AbstractHttp2ConnectionBuilder.encoderEnforceMaxQueuedControlFrames(...) to a number higher then 0. The default is 10000 which provides some protection by default but will hopefully not cause too many false-positives.
- Add unit tests

Result:

Protect against DDOS due control frames. Fixes CVE-2019-9512 / CVE-2019-9514 / CVE-2019-9515 .
This commit is contained in:
Norman Maurer 2019-08-13 19:02:20 +02:00
parent bc22bfa320
commit c6c679597f
6 changed files with 430 additions and 1 deletions

View File

@ -18,12 +18,12 @@ package io.netty.handler.codec.http2;
import io.netty.channel.Channel;
import io.netty.handler.codec.http2.Http2HeadersEncoder.SensitivityDetector;
import io.netty.util.internal.ObjectUtil;
import io.netty.util.internal.UnstableApi;
import static io.netty.handler.codec.http2.Http2CodecUtil.DEFAULT_HEADER_LIST_SIZE;
import static io.netty.handler.codec.http2.Http2CodecUtil.DEFAULT_MAX_RESERVED_STREAMS;
import static io.netty.handler.codec.http2.Http2PromisedRequestVerifier.ALWAYS_VERIFY;
import static io.netty.util.internal.ObjectUtil.checkPositive;
import static io.netty.util.internal.ObjectUtil.checkPositiveOrZero;
import static java.util.Objects.requireNonNull;
@ -107,6 +107,7 @@ public abstract class AbstractHttp2ConnectionHandlerBuilder<T extends Http2Conne
private Http2PromisedRequestVerifier promisedRequestVerifier = ALWAYS_VERIFY;
private boolean autoAckSettingsFrame = true;
private boolean autoAckPingFrame = true;
private int maxQueuedControlFrames = Http2CodecUtil.DEFAULT_MAX_QUEUED_CONTROL_FRAMES;
/**
* Sets the {@link Http2Settings} to use for the initial connection settings exchange.
@ -325,6 +326,30 @@ public abstract class AbstractHttp2ConnectionHandlerBuilder<T extends Http2Conne
return self();
}
/**
* Returns the maximum number of queued control frames that are allowed before the connection is closed.
* This allows to protected against various attacks that can lead to high CPU / memory usage if the remote-peer
* floods us with frames that would have us produce control frames, but stops to read from the underlying socket.
*
* {@code 0} means no protection is in place.
*/
protected int encoderEnforceMaxQueuedControlFrames() {
return maxQueuedControlFrames;
}
/**
* Sets the maximum number of queued control frames that are allowed before the connection is closed.
* This allows to protected against various attacks that can lead to high CPU / memory usage if the remote-peer
* floods us with frames that would have us produce control frames, but stops to read from the underlying socket.
*
* {@code 0} means no protection should be applied.
*/
protected B encoderEnforceMaxQueuedControlFrames(int maxQueuedControlFrames) {
enforceNonCodecConstraints("encoderEnforceMaxQueuedControlFrames");
this.maxQueuedControlFrames = ObjectUtil.checkPositiveOrZero(maxQueuedControlFrames, "maxQueuedControlFrames");
return self();
}
/**
* Returns the {@link SensitivityDetector} to use.
*/
@ -470,6 +495,9 @@ public abstract class AbstractHttp2ConnectionHandlerBuilder<T extends Http2Conne
Http2ConnectionEncoder encoder = new DefaultHttp2ConnectionEncoder(connection, writer);
boolean encoderEnforceMaxConcurrentStreams = encoderEnforceMaxConcurrentStreams();
if (maxQueuedControlFrames != 0) {
encoder = new Http2ControlFrameLimitEncoder(encoder, maxQueuedControlFrames);
}
if (encoderEnforceMaxConcurrentStreams) {
if (connection.isServer()) {
encoder.close();

View File

@ -132,6 +132,8 @@ public final class Http2CodecUtil {
public static final long DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS = MILLISECONDS.convert(30, SECONDS);
public static final int DEFAULT_MAX_QUEUED_CONTROL_FRAMES = 10000;
/**
* Returns {@code true} if the stream is an outbound stream.
*

View File

@ -0,0 +1,113 @@
/*
* Copyright 2019 The Netty Project
*
* The Netty Project licenses this file to you under the Apache License, version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package io.netty.handler.codec.http2;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelPromise;
import io.netty.util.internal.ObjectUtil;
import io.netty.util.internal.logging.InternalLogger;
import io.netty.util.internal.logging.InternalLoggerFactory;
/**
* {@link DecoratingHttp2ConnectionEncoder} which guards against a remote peer that will trigger a massive amount
* of control frames but will not consume our responses to these.
* This encoder will tear-down the connection once we reached the configured limit to reduce the risk of DDOS.
*/
final class Http2ControlFrameLimitEncoder extends DecoratingHttp2ConnectionEncoder {
private static final InternalLogger logger = InternalLoggerFactory.getInstance(Http2ControlFrameLimitEncoder.class);
private final int maxOutstandingControlFrames;
private final ChannelFutureListener outstandingControlFramesListener = new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) {
outstandingControlFrames--;
}
};
private Http2LifecycleManager lifecycleManager;
private int outstandingControlFrames;
private boolean limitReached;
Http2ControlFrameLimitEncoder(Http2ConnectionEncoder delegate, int maxOutstandingControlFrames) {
super(delegate);
this.maxOutstandingControlFrames = ObjectUtil.checkPositive(maxOutstandingControlFrames,
"maxOutstandingControlFrames");
}
@Override
public void lifecycleManager(Http2LifecycleManager lifecycleManager) {
this.lifecycleManager = lifecycleManager;
super.lifecycleManager(lifecycleManager);
}
@Override
public ChannelFuture writeSettingsAck(ChannelHandlerContext ctx, ChannelPromise promise) {
ChannelPromise newPromise = handleOutstandingControlFrames(ctx, promise);
if (newPromise == null) {
return promise;
}
return super.writeSettingsAck(ctx, newPromise);
}
@Override
public ChannelFuture writePing(ChannelHandlerContext ctx, boolean ack, long data, ChannelPromise promise) {
// Only apply the limit to ping acks.
if (ack) {
ChannelPromise newPromise = handleOutstandingControlFrames(ctx, promise);
if (newPromise == null) {
return promise;
}
return super.writePing(ctx, ack, data, newPromise);
}
return super.writePing(ctx, ack, data, promise);
}
@Override
public ChannelFuture writeRstStream(
ChannelHandlerContext ctx, int streamId, long errorCode, ChannelPromise promise) {
ChannelPromise newPromise = handleOutstandingControlFrames(ctx, promise);
if (newPromise == null) {
return promise;
}
return super.writeRstStream(ctx, streamId, errorCode, newPromise);
}
private ChannelPromise handleOutstandingControlFrames(ChannelHandlerContext ctx, ChannelPromise promise) {
if (!limitReached) {
if (outstandingControlFrames == maxOutstandingControlFrames) {
// Let's try to flush once as we may be able to flush some of the control frames.
ctx.flush();
}
if (outstandingControlFrames == maxOutstandingControlFrames) {
limitReached = true;
Http2Exception exception = Http2Exception.connectionError(Http2Error.ENHANCE_YOUR_CALM,
"Maximum number %d of outstanding control frames reached", maxOutstandingControlFrames);
logger.info("Maximum number {} of outstanding control frames reached. Closing channel {}",
maxOutstandingControlFrames, ctx.channel(), exception);
// First notify the Http2LifecycleManager and then close the connection.
lifecycleManager.onError(ctx, true, exception);
ctx.close();
}
outstandingControlFrames++;
// We did not reach the limit yet, add the listener to decrement the number of outstanding control frames
// once the promise was completed
return promise.unvoid().addListener(outstandingControlFramesListener);
}
return promise;
}
}

View File

@ -120,6 +120,16 @@ public class Http2FrameCodecBuilder extends
return super.encoderEnforceMaxConcurrentStreams(encoderEnforceMaxConcurrentStreams);
}
@Override
public int encoderEnforceMaxQueuedControlFrames() {
return super.encoderEnforceMaxQueuedControlFrames();
}
@Override
public Http2FrameCodecBuilder encoderEnforceMaxQueuedControlFrames(int maxQueuedControlFrames) {
return super.encoderEnforceMaxQueuedControlFrames(maxQueuedControlFrames);
}
@Override
public Http2HeadersEncoder.SensitivityDetector headerSensitivityDetector() {
return super.headerSensitivityDetector();

View File

@ -149,6 +149,16 @@ public class Http2MultiplexCodecBuilder
return super.encoderEnforceMaxConcurrentStreams(encoderEnforceMaxConcurrentStreams);
}
@Override
public int encoderEnforceMaxQueuedControlFrames() {
return super.encoderEnforceMaxQueuedControlFrames();
}
@Override
public Http2MultiplexCodecBuilder encoderEnforceMaxQueuedControlFrames(int maxQueuedControlFrames) {
return super.encoderEnforceMaxQueuedControlFrames(maxQueuedControlFrames);
}
@Override
public Http2HeadersEncoder.SensitivityDetector headerSensitivityDetector() {
return super.headerSensitivityDetector();

View File

@ -0,0 +1,266 @@
/*
* Copyright 2019 The Netty Project
*
* The Netty Project licenses this file to you under the Apache License, version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package io.netty.handler.codec.http2;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.UnpooledByteBufAllocator;
import io.netty.channel.Channel;
import io.netty.channel.ChannelConfig;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelMetadata;
import io.netty.channel.ChannelPromise;
import io.netty.channel.DefaultChannelPromise;
import io.netty.channel.DefaultMessageSizeEstimator;
import io.netty.handler.codec.http2.Http2Exception.ShutdownHint;
import io.netty.util.ReferenceCountUtil;
import io.netty.util.concurrent.EventExecutor;
import io.netty.util.concurrent.ImmediateEventExecutor;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import static io.netty.handler.codec.http2.Http2CodecUtil.*;
import static io.netty.handler.codec.http2.Http2Error.CANCEL;
import static io.netty.handler.codec.http2.Http2Error.ENHANCE_YOUR_CALM;
import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
/**
* Tests for {@link Http2ControlFrameLimitEncoder}.
*/
public class Http2ControlFrameLimitEncoderTest {
private Http2ControlFrameLimitEncoder encoder;
@Mock
private Http2FrameWriter writer;
@Mock
private ChannelHandlerContext ctx;
@Mock
private Channel channel;
@Mock
private Channel.Unsafe unsafe;
@Mock
private ChannelConfig config;
@Mock
private EventExecutor executor;
private int numWrites;
/**
* Init fields and do mocking.
*/
@Before
public void setup() throws Exception {
MockitoAnnotations.initMocks(this);
numWrites = 0;
Http2FrameWriter.Configuration configuration = mock(Http2FrameWriter.Configuration.class);
Http2FrameSizePolicy frameSizePolicy = mock(Http2FrameSizePolicy.class);
when(writer.configuration()).thenReturn(configuration);
when(configuration.frameSizePolicy()).thenReturn(frameSizePolicy);
when(frameSizePolicy.maxFrameSize()).thenReturn(DEFAULT_MAX_FRAME_SIZE);
when(writer.writeRstStream(eq(ctx), anyInt(), anyLong(), any(ChannelPromise.class)))
.thenAnswer(new Answer<ChannelFuture>() {
@Override
public ChannelFuture answer(InvocationOnMock invocationOnMock) {
return handlePromise(invocationOnMock, 3);
}
});
when(writer.writeSettingsAck(any(ChannelHandlerContext.class), any(ChannelPromise.class)))
.thenAnswer(new Answer<ChannelFuture>() {
@Override
public ChannelFuture answer(InvocationOnMock invocationOnMock) {
return handlePromise(invocationOnMock, 1);
}
});
when(writer.writePing(any(ChannelHandlerContext.class), anyBoolean(), anyLong(), any(ChannelPromise.class)))
.thenAnswer(new Answer<ChannelFuture>() {
@Override
public ChannelFuture answer(InvocationOnMock invocationOnMock) {
ChannelPromise promise = handlePromise(invocationOnMock, 3);
if (invocationOnMock.getArgument(1) == Boolean.FALSE) {
promise.trySuccess();
}
return promise;
}
});
when(writer.writeGoAway(any(ChannelHandlerContext.class), anyInt(), anyLong(), any(ByteBuf.class),
any(ChannelPromise.class))).thenAnswer(new Answer<ChannelFuture>() {
@Override
public ChannelFuture answer(InvocationOnMock invocationOnMock) {
ReferenceCountUtil.release(invocationOnMock.getArgument(3));
return handlePromise(invocationOnMock, 4);
}
});
Http2Connection connection = new DefaultHttp2Connection(false);
connection.remote().flowController(new DefaultHttp2RemoteFlowController(connection));
connection.local().flowController(new DefaultHttp2LocalFlowController(connection).frameWriter(writer));
DefaultHttp2ConnectionEncoder defaultEncoder =
new DefaultHttp2ConnectionEncoder(connection, writer);
encoder = new Http2ControlFrameLimitEncoder(defaultEncoder, 2);
DefaultHttp2ConnectionDecoder decoder =
new DefaultHttp2ConnectionDecoder(connection, encoder, mock(Http2FrameReader.class));
Http2ConnectionHandler handler = new Http2ConnectionHandlerBuilder()
.frameListener(mock(Http2FrameListener.class))
.codec(decoder, encoder).build();
// Set LifeCycleManager on encoder and decoder
when(ctx.channel()).thenReturn(channel);
when(ctx.alloc()).thenReturn(UnpooledByteBufAllocator.DEFAULT);
when(channel.alloc()).thenReturn(UnpooledByteBufAllocator.DEFAULT);
when(executor.inEventLoop()).thenReturn(true);
doAnswer(new Answer<ChannelPromise>() {
@Override
public ChannelPromise answer(InvocationOnMock invocation) throws Throwable {
return newPromise();
}
}).when(ctx).newPromise();
when(ctx.executor()).thenReturn(executor);
when(channel.isActive()).thenReturn(false);
when(channel.config()).thenReturn(config);
when(channel.isWritable()).thenReturn(true);
when(channel.bytesBeforeUnwritable()).thenReturn(Long.MAX_VALUE);
when(config.getWriteBufferHighWaterMark()).thenReturn(Integer.MAX_VALUE);
when(config.getMessageSizeEstimator()).thenReturn(DefaultMessageSizeEstimator.DEFAULT);
ChannelMetadata metadata = new ChannelMetadata(false, 16);
when(channel.metadata()).thenReturn(metadata);
when(channel.unsafe()).thenReturn(unsafe);
handler.handlerAdded(ctx);
}
private ChannelPromise handlePromise(InvocationOnMock invocationOnMock, int promiseIdx) {
ChannelPromise promise = invocationOnMock.getArgument(promiseIdx);
if (++numWrites == 2) {
promise.setSuccess();
}
return promise;
}
@After
public void teardown() {
// Close and release any buffered frames.
encoder.close();
}
@Test
public void testLimitSettingsAck() {
assertFalse(encoder.writeSettingsAck(ctx, newPromise()).isDone());
// The second write is always marked as success by our mock, which means it will also not be queued and so
// not count to the number of queued frames.
assertTrue(encoder.writeSettingsAck(ctx, newPromise()).isSuccess());
assertFalse(encoder.writeSettingsAck(ctx, newPromise()).isDone());
verifyFlushAndClose(0, false);
assertFalse(encoder.writeSettingsAck(ctx, newPromise()).isDone());
assertFalse(encoder.writeSettingsAck(ctx, newPromise()).isDone());
verifyFlushAndClose(1, true);
}
@Test
public void testLimitPingAck() {
assertFalse(encoder.writePing(ctx, true, 8, newPromise()).isDone());
// The second write is always marked as success by our mock, which means it will also not be queued and so
// not count to the number of queued frames.
assertTrue(encoder.writePing(ctx, true, 8, newPromise()).isSuccess());
assertFalse(encoder.writePing(ctx, true, 8, newPromise()).isDone());
verifyFlushAndClose(0, false);
assertFalse(encoder.writePing(ctx, true, 8, newPromise()).isDone());
assertFalse(encoder.writePing(ctx, true, 8, newPromise()).isDone());
verifyFlushAndClose(1, true);
}
@Test
public void testNotLimitPing() {
assertTrue(encoder.writePing(ctx, false, 8, newPromise()).isSuccess());
assertTrue(encoder.writePing(ctx, false, 8, newPromise()).isSuccess());
assertTrue(encoder.writePing(ctx, false, 8, newPromise()).isSuccess());
assertTrue(encoder.writePing(ctx, false, 8, newPromise()).isSuccess());
verifyFlushAndClose(0, false);
}
@Test
public void testLimitRst() {
assertFalse(encoder.writeRstStream(ctx, 1, CANCEL.code(), newPromise()).isDone());
// The second write is always marked as success by our mock, which means it will also not be queued and so
// not count to the number of queued frames.
assertTrue(encoder.writeRstStream(ctx, 1, CANCEL.code(), newPromise()).isSuccess());
assertFalse(encoder.writeRstStream(ctx, 1, CANCEL.code(), newPromise()).isDone());
verifyFlushAndClose(0, false);
assertFalse(encoder.writeRstStream(ctx, 1, CANCEL.code(), newPromise()).isDone());
assertFalse(encoder.writeRstStream(ctx, 1, CANCEL.code(), newPromise()).isDone());
verifyFlushAndClose(1, true);
}
@Test
public void testLimit() {
assertFalse(encoder.writeRstStream(ctx, 1, CANCEL.code(), newPromise()).isDone());
// The second write is always marked as success by our mock, which means it will also not be queued and so
// not count to the number of queued frames.
assertTrue(encoder.writePing(ctx, false, 8, newPromise()).isSuccess());
assertFalse(encoder.writePing(ctx, true, 8, newPromise()).isSuccess());
verifyFlushAndClose(0, false);
assertFalse(encoder.writeSettingsAck(ctx, newPromise()).isDone());
assertFalse(encoder.writeRstStream(ctx, 1, CANCEL.code(), newPromise()).isDone());
assertFalse(encoder.writePing(ctx, true, 8, newPromise()).isSuccess());
verifyFlushAndClose(1, true);
}
private void verifyFlushAndClose(int invocations, boolean failed) {
verify(ctx, atLeast(invocations)).flush();
verify(ctx, times(invocations)).close();
if (failed) {
verify(writer, times(1)).writeGoAway(eq(ctx), eq(0), eq(ENHANCE_YOUR_CALM.code()),
any(ByteBuf.class), any(ChannelPromise.class));
}
}
private static void assertWriteFailure(ChannelFuture future) {
Http2Exception exception = (Http2Exception) future.cause();
assertEquals(ShutdownHint.HARD_SHUTDOWN, exception.shutdownHint());
assertEquals(Http2Error.ENHANCE_YOUR_CALM, exception.error());
}
private ChannelPromise newPromise() {
return new DefaultChannelPromise(channel, ImmediateEventExecutor.INSTANCE);
}
}