record: avoid crash when calling RecordFlushReplyBuffer recursively

RecordFlushReplyBuffer can call itself recursively through
WriteClient->CallCallbacks->_CallCallbacks->RecordFlushAllContexts
when the recording client's buffer cannot be completely emptied in one
WriteClient. When a such a recursion occurs, it will not be broken out
of which results in segmentation fault when the stack is exhausted.

This patch adds a counter (a flag, really) that guards against this
situation, to break out of the recursion.

One alternative to this change would be to change _CallCallbacks to
check the corresponding counter before the callback loop, but that
might affect existing behavior, which may be relied upon.

Reviewed-by: Rami Ylimäki <rami.ylimaki@vincit.fi>
Signed-off-by: Erkki Seppälä <erkki.seppala@vincit.fi>
Signed-off-by: Keith Packard <keithp@keithp.com>
This commit is contained in:
Erkki Seppälä 2011-02-10 15:35:14 +02:00 committed by Keith Packard
parent b833f9b924
commit 0801afbd7c

View File

@ -77,6 +77,7 @@ typedef struct {
char bufCategory; /* category of protocol in replyBuffer */
int numBufBytes; /* number of bytes in replyBuffer */
char replyBuffer[REPLY_BUF_SIZE]; /* buffered recorded protocol */
int inFlush; /* are we inside RecordFlushReplyBuffer */
} RecordContextRec, *RecordContextPtr;
/* RecordMinorOpRec - to hold minor opcode selections for extension requests
@ -245,8 +246,9 @@ RecordFlushReplyBuffer(
int len2
)
{
if (!pContext->pRecordingClient || pContext->pRecordingClient->clientGone)
if (!pContext->pRecordingClient || pContext->pRecordingClient->clientGone || pContext->inFlush)
return;
++pContext->inFlush;
if (pContext->numBufBytes)
WriteToClient(pContext->pRecordingClient, pContext->numBufBytes,
(char *)pContext->replyBuffer);
@ -255,6 +257,7 @@ RecordFlushReplyBuffer(
WriteToClient(pContext->pRecordingClient, len1, (char *)data1);
if (len2)
WriteToClient(pContext->pRecordingClient, len2, (char *)data2);
--pContext->inFlush;
} /* RecordFlushReplyBuffer */
@ -1938,6 +1941,7 @@ ProcRecordCreateContext(ClientPtr client)
pContext->numBufBytes = 0;
pContext->pBufClient = NULL;
pContext->continuedReply = 0;
pContext->inFlush = 0;
err = RecordRegisterClients(pContext, client,
(xRecordRegisterClientsReq *)stuff);