andreacavalli/netty5
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[#4235] Ensure OpenSslEngine.unwrap(...) / wrap(...) correctly return HandshakeStatus.FINISHED

Browse Source 
Motivation:

OpenSslEngine.unwrap(...) / wrap(...) must return HandhsakeStatus.FINISHED if an unwrap or wrap finishes a handshake to behave like descripted in the SSLEngine docs.

Modifications:

- Ensure we return HandshakeStatus.FINISHED

Result:

Behave correctly.
This commit is contained in:
Norman Maurer 2015-09-18 16:09:43 +02:00
parent 7852057b5d
commit 83c2d7c43b
1 changed files with 22 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
handler/src/main/java/io/netty/handler/ssl/OpenSslEngine.java
View File

@ -401,8 +401,8 @@ public final class OpenSslEngine extends SSLEngine {
return bioRead; return bioRead;
} }
private SSLEngineResult readPendingBytesFromBIO(ByteBuffer dst, int bytesConsumed, int bytesProduced) private SSLEngineResult readPendingBytesFromBIO(
throws SSLException { ByteBuffer dst, int bytesConsumed, int bytesProduced, HandshakeStatus status) throws SSLException {
// Check to see if the engine wrote data into the network BIO // Check to see if the engine wrote data into the network BIO
int pendingNet = SSL.pendingWrittenBytesInBIO(networkBIO); int pendingNet = SSL.pendingWrittenBytesInBIO(networkBIO);
if (pendingNet > 0) { if (pendingNet > 0) {
@ -410,7 +410,8 @@ public final class OpenSslEngine extends SSLEngine {
// Do we have enough room in dst to write encrypted data? // Do we have enough room in dst to write encrypted data?
int capacity = dst.remaining(); int capacity = dst.remaining();
if (capacity < pendingNet) { if (capacity < pendingNet) {
return new SSLEngineResult(BUFFER_OVERFLOW, mayFinishHandshake(getHandshakeStatus(pendingNet)), return new SSLEngineResult(BUFFER_OVERFLOW,
mayFinishHandshake(status != FINISHED ? getHandshakeStatus(pendingNet) : status),
bytesConsumed, bytesProduced); bytesConsumed, bytesProduced);
} }
@ -432,7 +433,8 @@ public final class OpenSslEngine extends SSLEngine {
shutdown(); shutdown();
} }
return new SSLEngineResult(getEngineStatus(), mayFinishHandshake(getHandshakeStatus(pendingNet)), return new SSLEngineResult(getEngineStatus(),
mayFinishHandshake(status != FINISHED ? getHandshakeStatus(pendingNet) : status),
bytesConsumed, bytesProduced); bytesConsumed, bytesProduced);
} }
return null; return null;
@ -465,6 +467,7 @@ public final class OpenSslEngine extends SSLEngine {
throw new ReadOnlyBufferException(); throw new ReadOnlyBufferException();
} }
HandshakeStatus status = NOT_HANDSHAKING;
// Prepare OpenSSL to work in server mode and receive handshake // Prepare OpenSSL to work in server mode and receive handshake
if (handshakeState != HandshakeState.FINISHED) { if (handshakeState != HandshakeState.FINISHED) {
if (handshakeState != HandshakeState.STARTED_EXPLICITLY) { if (handshakeState != HandshakeState.STARTED_EXPLICITLY) {
@ -472,7 +475,7 @@ public final class OpenSslEngine extends SSLEngine {
handshakeState = HandshakeState.STARTED_IMPLICITLY; handshakeState = HandshakeState.STARTED_IMPLICITLY;
} }
HandshakeStatus status = handshake(); status = handshake();
if (status == NEED_UNWRAP) { if (status == NEED_UNWRAP) {
return NEED_UNWRAP_OK; return NEED_UNWRAP_OK;
} }
@ -517,7 +520,7 @@ public final class OpenSslEngine extends SSLEngine {
} }
} }
SSLEngineResult pendingNetResult = readPendingBytesFromBIO(dst, bytesConsumed, bytesProduced); SSLEngineResult pendingNetResult = readPendingBytesFromBIO(dst, bytesConsumed, bytesProduced, status);
if (pendingNetResult != null) { if (pendingNetResult != null) {
return pendingNetResult; return pendingNetResult;
} }
@ -526,13 +529,13 @@ public final class OpenSslEngine extends SSLEngine {
// We need to check if pendingWrittenBytesInBIO was checked yet, as we may not checked if the srcs was empty, // We need to check if pendingWrittenBytesInBIO was checked yet, as we may not checked if the srcs was empty,
// or only contained empty buffers. // or only contained empty buffers.
if (bytesConsumed == 0) { if (bytesConsumed == 0) {
SSLEngineResult pendingNetResult = readPendingBytesFromBIO(dst, 0, bytesProduced); SSLEngineResult pendingNetResult = readPendingBytesFromBIO(dst, 0, bytesProduced, status);
if (pendingNetResult != null) { if (pendingNetResult != null) {
return pendingNetResult; return pendingNetResult;
} }
} }
return newResult(bytesConsumed, bytesProduced); return newResult(bytesConsumed, bytesProduced, status);
} }
private void checkPendingHandshakeException() throws SSLHandshakeException { private void checkPendingHandshakeException() throws SSLHandshakeException {
@ -605,6 +608,7 @@ public final class OpenSslEngine extends SSLEngine {
capacity += dst.remaining(); capacity += dst.remaining();
} }
HandshakeStatus status = NOT_HANDSHAKING;
// Prepare OpenSSL to work in server mode and receive handshake // Prepare OpenSSL to work in server mode and receive handshake
if (handshakeState != HandshakeState.FINISHED) { if (handshakeState != HandshakeState.FINISHED) {
if (handshakeState != HandshakeState.STARTED_EXPLICITLY) { if (handshakeState != HandshakeState.STARTED_EXPLICITLY) {
@ -612,7 +616,7 @@ public final class OpenSslEngine extends SSLEngine {
handshakeState = HandshakeState.STARTED_IMPLICITLY; handshakeState = HandshakeState.STARTED_IMPLICITLY;
} }
HandshakeStatus status = handshake(); status = handshake();
if (status == NEED_WRAP) { if (status == NEED_WRAP) {
return NEED_WRAP_OK; return NEED_WRAP_OK;
} }
@ -703,7 +707,7 @@ public final class OpenSslEngine extends SSLEngine {
idx ++; idx ++;
} else { } else {
// We read everything return now. // We read everything return now.
return newResult(bytesConsumed, bytesProduced); return newResult(bytesConsumed, bytesProduced, status);
} }
} else { } else {
int sslError = SSL.getError(ssl, bytesRead); int sslError = SSL.getError(ssl, bytesRead);
@ -717,7 +721,7 @@ public final class OpenSslEngine extends SSLEngine {
case SSL.SSL_ERROR_WANT_READ: case SSL.SSL_ERROR_WANT_READ:
case SSL.SSL_ERROR_WANT_WRITE: case SSL.SSL_ERROR_WANT_WRITE:
// break to the outer loop // break to the outer loop
return newResult(bytesConsumed, bytesProduced); return newResult(bytesConsumed, bytesProduced, status);
default: default:
// Everything else is considered as error so shutdown and throw an exceptions // Everything else is considered as error so shutdown and throw an exceptions
shutdownWithError("SSL_read"); shutdownWithError("SSL_read");
@ -738,7 +742,8 @@ public final class OpenSslEngine extends SSLEngine {
if (pendingAppData() > 0) { if (pendingAppData() > 0) {
// We filled all buffers but there is still some data pending in the BIO buffer, return BUFFER_OVERFLOW. // We filled all buffers but there is still some data pending in the BIO buffer, return BUFFER_OVERFLOW.
return new SSLEngineResult( return new SSLEngineResult(
BUFFER_OVERFLOW, mayFinishHandshake(getHandshakeStatus()), bytesConsumed, bytesProduced); BUFFER_OVERFLOW, mayFinishHandshake(status != FINISHED ? getHandshakeStatus(): status),
bytesConsumed, bytesProduced);
} }
// Check to see if we received a close_notify message from the peer. // Check to see if we received a close_notify message from the peer.
@ -746,7 +751,7 @@ public final class OpenSslEngine extends SSLEngine {
closeAll(); closeAll();
} }
return newResult(bytesConsumed, bytesProduced); return newResult(bytesConsumed, bytesProduced, status);
} }
private int pendingAppData() { private int pendingAppData() {
@ -755,9 +760,11 @@ public final class OpenSslEngine extends SSLEngine {
return handshakeState == HandshakeState.FINISHED ? SSL.pendingReadableBytesInSSL(ssl) : 0; return handshakeState == HandshakeState.FINISHED ? SSL.pendingReadableBytesInSSL(ssl) : 0;
} }
private SSLEngineResult newResult(int bytesConsumed, int bytesProduced) throws SSLException { private SSLEngineResult newResult(
int bytesConsumed, int bytesProduced, HandshakeStatus status) throws SSLException {
return new SSLEngineResult( return new SSLEngineResult(
getEngineStatus(), mayFinishHandshake(getHandshakeStatus()), bytesConsumed, bytesProduced); getEngineStatus(), mayFinishHandshake(status != FINISHED ? getHandshakeStatus() : status)
, bytesConsumed, bytesProduced);
} }
private void closeAll() throws SSLException { private void closeAll() throws SSLException {