db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Make ioReadLine() read less aggressively.

Browse Source 
ioReadLine() calls ioRead(), which aggressively tries to fill the output buffer, but this doesn't play well with blocking reads.

Give ioReadLine() an option that tells it to read only what is available.  That doesn't mean the function will never block but at least it won't do so by reading too far.
This commit is contained in:
David Steele 2018-11-12 21:18:53 -05:00
parent bc810e5a87
commit 086bc35ddc
3 changed files with 60 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/xml/release.xml
View File

@ -66,6 +66,10 @@
<p>Correct current history item in <code>InfoPg</code> to always be in position 0.</p>
</release-item>
<release-item>
<p>Make <code>ioReadLine()</code> read less aggressively.</p>
</release-item>
<release-item>
<p>Add <id>base</id> variants to all integer to string conversion functions.</p>
</release-item>

61
src/common/io/read.c
View File

@ -109,11 +109,12 @@ ioReadEofDriver(const IoRead *this)
Read data from IO and process filters
***********************************************************************************************************************************/
static void
ioReadInternal(IoRead *this, Buffer *buffer)
ioReadInternal(IoRead *this, Buffer *buffer, bool relaxed)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_READ, this);
FUNCTION_DEBUG_PARAM(BUFFER, buffer);
FUNCTION_DEBUG_PARAM(BOOL, relaxed);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(buffer != NULL);
@ -121,6 +122,8 @@ ioReadInternal(IoRead *this, Buffer *buffer)
FUNCTION_DEBUG_END();
// Loop until EOF or the output buffer is full
size_t bufferUsedBegin = bufUsed(buffer);
while (!this->eofAll && bufRemains(buffer) > 0)
{
// Process input buffer again to get more output
@ -143,6 +146,10 @@ ioReadInternal(IoRead *this, Buffer *buffer)
// Process the input buffer (or flush if NULL)
ioFilterGroupProcess(this->filterGroup, this->input, buffer);
// Stop if relaxed read -- we don't need to fill the buffer as long as we got some data
if (relaxed && bufUsed(buffer) > bufferUsedBegin)
break;
}
// Eof when no more input and the filter group is done
@ -186,13 +193,15 @@ ioRead(IoRead *this, Buffer *buffer)
}
// Read data
ioReadInternal(this, buffer);
ioReadInternal(this, buffer, false);
FUNCTION_DEBUG_RESULT(SIZE, outputRemains - bufRemains(buffer));
}
/***********************************************************************************************************************************
Read linefeed-terminated string
The entire string to search for must fit within a single buffer.
***********************************************************************************************************************************/
String *
ioReadLine(IoRead *this)
@ -216,32 +225,44 @@ ioReadLine(IoRead *this)
MEM_CONTEXT_END();
}
// Read more data if there is any. The entire string we are searching for must fit within the buffer so we'll make sure that
// the buffer is full
ioReadInternal(this, this->output);
// If some data was read search for a linefeed
// Search for a linefeed
String *result = NULL;
if (bufUsed(this->output) > 0)
do
{
// Search for a linefeed in the buffer
char *linefeed = memchr(bufPtr(this->output), '\n', bufUsed(this->output));
// A linefeed was found so get the string
if (linefeed != NULL)
if (bufUsed(this->output) > 0)
{
// Get the string size
size_t size = (size_t)(linefeed - (char *)bufPtr(this->output) + 1);
// Search for a linefeed in the buffer
char *linefeed = memchr(bufPtr(this->output), '\n', bufUsed(this->output));
// Create the string
result = strNewN((char *)bufPtr(this->output), size - 1);
// A linefeed was found so get the string
if (linefeed != NULL)
{
// Get the string size
size_t size = (size_t)(linefeed - (char *)bufPtr(this->output) + 1);
// Remove string from the output buffer
memmove(bufPtr(this->output), bufPtr(this->output) + size, bufUsed(this->output) - size);
bufUsedSet(this->output, bufUsed(this->output) - size);
// Create the string
result = strNewN((char *)bufPtr(this->output), size - 1);
// Remove string from the output buffer
memmove(bufPtr(this->output), bufPtr(this->output) + size, bufUsed(this->output) - size);
bufUsedSet(this->output, bufUsed(this->output) - size);
}
}
// Read data if no linefeed was found in the existing buffer
if (result == NULL)
{
if (bufFull(this->output))
THROW_FMT(FileReadError, "unable to find line in %zu byte buffer", bufSize(this->output));
if (ioReadEof(this))
THROW(FileReadError, "unexpected eof while reading line");
ioReadInternal(this, this->output, 1);
}
}
while (result == NULL);
FUNCTION_DEBUG_RESULT(STRING, result);
}

20
test/src/module/common/ioTest.c
View File

@ -345,16 +345,26 @@ testRun(void)
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "DDD", " check buffer");
// Read line doesn't work without a linefeed
TEST_RESULT_STR(strPtr(ioReadLine(read)), NULL, "read line");
TEST_ERROR(ioReadLine(read), FileReadError, "unexpected eof while reading line");
// But those bytes can be picked up by a buffer read
bufUsedSet(buffer, 0);
TEST_RESULT_INT(ioRead(read, buffer), 3, "read buffer");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "EFF", " check buffer");
buffer = bufNew(2);
TEST_RESULT_INT(ioRead(read, buffer), 2, "read buffer");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "EF", " check buffer");
buffer = bufNew(1);
TEST_RESULT_INT(ioRead(read, buffer), 1, "read buffer");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "F", " check buffer");
// Nothing left to read
TEST_RESULT_STR(strPtr(ioReadLine(read)), NULL, "read line");
TEST_ERROR(ioReadLine(read), FileReadError, "unexpected eof while reading line");
TEST_RESULT_INT(ioRead(read, buffer), 0, "read buffer");
// Error if buffer is full and there is no linefeed
ioBufferSizeSet(10);
read = ioBufferReadIo(ioBufferReadNew(bufNewZ("0123456789")));
ioReadOpen(read);
TEST_ERROR(ioReadLine(read), FileReadError, "unable to find line in 10 byte buffer");
}
// *****************************************************************************************************************************