OpenSslEngine return NEED_WRAP if the destination buffered filled
Motivation: If the destination buffer is completely filled during a call to OpenSslEngine#wrap(..) we may return NEED_UNWRAP because there is no data pending in the SSL buffers. However during a handshake if the SSL buffers were just drained, and filled up the destination buffer it is possible OpenSSL may produce more data on the next call to SSL_write. This means we should keep trying to call SSL_write as long as the destination buffer is filled and only return NEED_UNWRAP when the destination buffer is not full and there is no data pending in OpenSSL's buffers. Modifications: - If the handshake produces data in OpenSslEngine#wrap(..) we should return NEED_WRAP if the destination buffer is completely filled Result: OpenSslEngine returns the correct handshake status from wrap(). Fixes https://github.com/netty/netty/issues/6796.
This commit is contained in:
parent
270e9d66c5
commit
24f801c7d1
|
@ -571,8 +571,11 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
|
||||||
bytesProduced = bioLengthBefore - SSL.bioLengthByteBuffer(networkBIO);
|
bytesProduced = bioLengthBefore - SSL.bioLengthByteBuffer(networkBIO);
|
||||||
|
|
||||||
if (bytesProduced > 0) {
|
if (bytesProduced > 0) {
|
||||||
// It's important we call this before wrapStatus() as wrapStatus() may shutdown the engine.
|
// If we have filled up the dst buffer and we have not finished the handshake we should try to
|
||||||
|
// wrap again. Otherwise we should only try to wrap again if there is still data pending in
|
||||||
|
// SSL buffers.
|
||||||
return newResult(mayFinishHandshake(status != FINISHED ?
|
return newResult(mayFinishHandshake(status != FINISHED ?
|
||||||
|
bytesProduced == bioLengthBefore ? NEED_WRAP :
|
||||||
getHandshakeStatus(SSL.bioLengthNonApplication(networkBIO)) : FINISHED),
|
getHandshakeStatus(SSL.bioLengthNonApplication(networkBIO)) : FINISHED),
|
||||||
0, bytesProduced);
|
0, bytesProduced);
|
||||||
}
|
}
|
||||||
|
@ -648,8 +651,12 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
|
||||||
|
|
||||||
bytesProduced += bioLengthBefore - SSL.bioLengthByteBuffer(networkBIO);
|
bytesProduced += bioLengthBefore - SSL.bioLengthByteBuffer(networkBIO);
|
||||||
|
|
||||||
|
// If we have filled up the dst buffer and we have not finished the handshake we should
|
||||||
|
// try to wrap again. Otherwise we should only try to wrap again if there is still data
|
||||||
|
// pending in SSL buffers.
|
||||||
SSLEngineResult.HandshakeStatus hs = mayFinishHandshake(
|
SSLEngineResult.HandshakeStatus hs = mayFinishHandshake(
|
||||||
status != FINISHED ? getHandshakeStatus(SSL.bioLengthNonApplication(networkBIO))
|
status != FINISHED ? bytesProduced == dst.remaining() ? NEED_WRAP
|
||||||
|
: getHandshakeStatus(SSL.bioLengthNonApplication(networkBIO))
|
||||||
: FINISHED);
|
: FINISHED);
|
||||||
return newResult(hs, bytesConsumed, bytesProduced);
|
return newResult(hs, bytesConsumed, bytesProduced);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user