db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Add io-timeout option.

Browse Source 
Timeout used for connections and read/write operations.

Note that the entire read/write operation does not need to complete within this timeout but some progress must be made, even if it is only a single byte.
This commit is contained in:
David Steele 2020-04-17 09:18:52 -04:00
parent c88684e2bf
commit 5d25e508ae
17 changed files with 160 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
build/lib/pgBackRestBuild/Config/Data.pm
View File

@ -146,6 +146,7 @@ use constant CFGOPT_COMPRESS => 'compress
use constant CFGOPT_COMPRESS_TYPE => 'compress-type';
use constant CFGOPT_COMPRESS_LEVEL => 'compress-level';
use constant CFGOPT_COMPRESS_LEVEL_NETWORK => 'compress-level-network';
use constant CFGOPT_IO_TIMEOUT => 'io-timeout';
use constant CFGOPT_NEUTRAL_UMASK => 'neutral-umask';
use constant CFGOPT_PROTOCOL_TIMEOUT => 'protocol-timeout';
use constant CFGOPT_PROCESS_MAX => 'process-max';
@ -1310,6 +1311,31 @@ my %hConfigDefine =
},
},
&CFGOPT_IO_TIMEOUT =>
{
&CFGDEF_SECTION => CFGDEF_SECTION_GLOBAL,
&CFGDEF_TYPE => CFGDEF_TYPE_FLOAT,
&CFGDEF_DEFAULT => 60,
&CFGDEF_ALLOW_RANGE => [.1, 3600],
&CFGDEF_COMMAND =>
{
&CFGCMD_ARCHIVE_GET => {},
&CFGCMD_ARCHIVE_PUSH => {},
&CFGCMD_BACKUP => {},
&CFGCMD_CHECK => {},
&CFGCMD_INFO => {},
&CFGCMD_REPO_CREATE => {},
&CFGCMD_REPO_GET => {},
&CFGCMD_REPO_LS => {},
&CFGCMD_REPO_PUT => {},
&CFGCMD_REPO_RM => {},
&CFGCMD_RESTORE => {},
&CFGCMD_STANZA_CREATE => {},
&CFGCMD_STANZA_DELETE => {},
&CFGCMD_STANZA_UPGRADE => {},
}
},
&CFGOPT_LOCK_PATH =>
{
&CFGDEF_SECTION => CFGDEF_SECTION_GLOBAL,

11
doc/xml/reference.xml
View File

@ -188,6 +188,17 @@
<example>y</example>
</config-key>
<!-- CONFIG - GENERAL SECTION - IO-TIMEOUT KEY -->
<config-key id="io-timeout" name="I/O Timeout">
<summary>I/O timeout.</summary>
<text>Timeout used for connections and read/write operations.
Note that the entire read/write operation does not need to complete within this timeout but <i>some</i> progress must be made, even if it is only a single byte.</text>
<example>120</example>
</config-key>
<!-- CONFIG - GENERAL SECTION - LOCK-PATH KEY -->
<config-key id="lock-path" name="Lock Path">
<summary>Path where lock files are stored.</summary>

4
doc/xml/release.xml
View File

@ -57,6 +57,10 @@
<p><proper>TCP</proper> keep-alive options are configurable.</p>
</release-item>
<release-item>
<p>Add <br-option>io-timeout</br-option> option.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-reviewer id="cynthia.shang"/>

2
src/command/repo/put.c
View File

@ -88,7 +88,7 @@ cmdStoragePut(void)
MEM_CONTEXT_TEMP_BEGIN()
{
storagePutProcess(ioHandleReadNew(STRDEF("stdin"), STDIN_FILENO, IO_TIMEOUT_DEFAULT));
storagePutProcess(ioHandleReadNew(STRDEF("stdin"), STDIN_FILENO, ioTimeoutMs()));
}
MEM_CONTEXT_TEMP_END();

23
src/common/io/io.c
View File

@ -19,6 +19,9 @@ should set their buffer size using ioBufferSize() but there may be cases where a
static size_t bufferSize = (8 * IO_BUFFER_BLOCK_SIZE);
// I/O timeout in milliseconds
static TimeMSec timeoutMs = 60000;
/**********************************************************************************************************************************/
size_t
ioBufferSize(void)
@ -39,6 +42,26 @@ ioBufferSizeSet(size_t bufferSizeParam)
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
TimeMSec
ioTimeoutMs(void)
{
FUNCTION_TEST_VOID();
FUNCTION_TEST_RETURN(timeoutMs);
}
void
ioTimeoutMsSet(TimeMSec timeoutMsParam)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(TIME_MSEC, timeoutMsParam);
FUNCTION_TEST_END();
timeoutMs = timeoutMsParam;
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
Buffer *
ioReadBuf(IoRead *read)

11
src/common/io/io.h
View File

@ -9,11 +9,7 @@ Common IO functions.
#include <stddef.h>
#include <common/io/read.h>
/***********************************************************************************************************************************
Default IO timeout to use when nothing else is configured
***********************************************************************************************************************************/
#define IO_TIMEOUT_DEFAULT 60000
#include <common/time.h>
/***********************************************************************************************************************************
Functions
@ -31,4 +27,9 @@ Getters/Setters
size_t ioBufferSize(void);
void ioBufferSizeSet(size_t bufferSize);
// I/O timeout in milliseconds. Used to timeout on connections and read/write operations. Note that an *entire* read/write operation
// does not need to take place within this timeout but at least some progress needs to be made, even if it is only a byte.
TimeMSec ioTimeoutMs(void);
void ioTimeoutMsSet(TimeMSec timeout);
#endif

9
src/config/config.auto.c
View File

@ -310,6 +310,7 @@ STRING_EXTERN(CFGOPT_FILTER_STR, CFGOPT_FILTE
STRING_EXTERN(CFGOPT_FORCE_STR, CFGOPT_FORCE);
STRING_EXTERN(CFGOPT_HOST_ID_STR, CFGOPT_HOST_ID);
STRING_EXTERN(CFGOPT_IGNORE_MISSING_STR, CFGOPT_IGNORE_MISSING);
STRING_EXTERN(CFGOPT_IO_TIMEOUT_STR, CFGOPT_IO_TIMEOUT);
STRING_EXTERN(CFGOPT_LINK_ALL_STR, CFGOPT_LINK_ALL);
STRING_EXTERN(CFGOPT_LINK_MAP_STR, CFGOPT_LINK_MAP);
STRING_EXTERN(CFGOPT_LOCK_PATH_STR, CFGOPT_LOCK_PATH);
@ -686,6 +687,14 @@ static ConfigOptionData configOptionData[CFG_OPTION_TOTAL] = CONFIG_OPTION_LIST
CONFIG_OPTION_DEFINE_ID(cfgDefOptIgnoreMissing)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(
CONFIG_OPTION_NAME(CFGOPT_IO_TIMEOUT)
CONFIG_OPTION_INDEX(0)
CONFIG_OPTION_DEFINE_ID(cfgDefOptIoTimeout)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(

5
src/config/config.auto.h
View File

@ -107,6 +107,8 @@ Option constants
STRING_DECLARE(CFGOPT_HOST_ID_STR);
#define CFGOPT_IGNORE_MISSING "ignore-missing"
STRING_DECLARE(CFGOPT_IGNORE_MISSING_STR);
#define CFGOPT_IO_TIMEOUT "io-timeout"
STRING_DECLARE(CFGOPT_IO_TIMEOUT_STR);
#define CFGOPT_LINK_ALL "link-all"
STRING_DECLARE(CFGOPT_LINK_ALL_STR);
#define CFGOPT_LINK_MAP "link-map"
@ -416,7 +418,7 @@ Option constants
#define CFGOPT_TYPE "type"
STRING_DECLARE(CFGOPT_TYPE_STR);
#define CFG_OPTION_TOTAL 181
#define CFG_OPTION_TOTAL 182
/***********************************************************************************************************************************
Command enum
@ -477,6 +479,7 @@ typedef enum
cfgOptForce,
cfgOptHostId,
cfgOptIgnoreMissing,
cfgOptIoTimeout,
cfgOptLinkAll,
cfgOptLinkMap,
cfgOptLockPath,

47
src/config/define.auto.c
View File

@ -1377,6 +1377,53 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
)
)
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(
CFGDEFDATA_OPTION_NAME("io-timeout")
CFGDEFDATA_OPTION_REQUIRED(true)
CFGDEFDATA_OPTION_SECTION(cfgDefSectionGlobal)
CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeFloat)
CFGDEFDATA_OPTION_INTERNAL(false)
CFGDEFDATA_OPTION_INDEX_TOTAL(1)
CFGDEFDATA_OPTION_SECURE(false)
CFGDEFDATA_OPTION_HELP_SECTION("general")
CFGDEFDATA_OPTION_HELP_SUMMARY("I/O timeout.")
CFGDEFDATA_OPTION_HELP_DESCRIPTION
(
"Timeout used for connections and read/write operations.\n"
"\n"
"Note that the entire read/write operation does not need to complete within this timeout but some progress must be "
"made, even if it is only a single byte."
)
CFGDEFDATA_OPTION_COMMAND_LIST
(
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoCreate)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoGet)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoLs)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoPut)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoRm)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRestore)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaCreate)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaDelete)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaUpgrade)
)
CFGDEFDATA_OPTION_OPTIONAL_LIST
(
CFGDEFDATA_OPTION_OPTIONAL_ALLOW_RANGE(0.1, 3600)
CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("60")
)
)
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(

1
src/config/define.auto.h
View File

@ -79,6 +79,7 @@ typedef enum
cfgDefOptForce,
cfgDefOptHostId,
cfgDefOptIgnoreMissing,
cfgDefOptIoTimeout,
cfgDefOptLinkAll,
cfgDefOptLinkMap,
cfgDefOptLockPath,

4
src/config/load.c
View File

@ -350,6 +350,10 @@ cfgLoad(unsigned int argListSize, const char *argList[])
if (cfgOptionValid(cfgOptBufferSize))
ioBufferSizeSet(cfgOptionUInt(cfgOptBufferSize));
// Set IO timeout
if (cfgOptionValid(cfgOptIoTimeout))
ioTimeoutMsSet((TimeMSec)(cfgOptionDbl(cfgOptIoTimeout) * MSEC_PER_SEC));
// Open the log file if this command logs to a file
cfgLoadLogFile();

13
src/config/parse.auto.c
View File

@ -328,6 +328,18 @@ static const struct option optionList[] =
.val = PARSE_OPTION_FLAG | cfgOptIgnoreMissing,
},
// io-timeout option
// -----------------------------------------------------------------------------------------------------------------------------
{
.name = CFGOPT_IO_TIMEOUT,
.has_arg = required_argument,
.val = PARSE_OPTION_FLAG | cfgOptIoTimeout,
},
{
.name = "reset-" CFGOPT_IO_TIMEOUT,
.val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptIoTimeout,
},
// link-all option
// -----------------------------------------------------------------------------------------------------------------------------
{
@ -2443,6 +2455,7 @@ static const ConfigOption optionResolveOrder[] =
cfgOptFilter,
cfgOptHostId,
cfgOptIgnoreMissing,
cfgOptIoTimeout,
cfgOptLinkAll,
cfgOptLinkMap,
cfgOptLockPath,

3
src/storage/helper.c
View File

@ -6,6 +6,7 @@ Storage Helper
#include <string.h>
#include "common/debug.h"
#include "common/io/io.h"
#include "common/memContext.h"
#include "common/regExp.h"
#include "config/define.h"
@ -383,7 +384,7 @@ storageRepoGet(const String *type, bool write)
strEqZ(cfgOptionStr(cfgOptRepoS3UriStyle), STORAGE_S3_URI_STYLE_HOST) ? storageS3UriStyleHost : storageS3UriStylePath,
cfgOptionStr(cfgOptRepoS3Region), cfgOptionStr(cfgOptRepoS3Key), cfgOptionStr(cfgOptRepoS3KeySecret),
cfgOptionTest(cfgOptRepoS3Token) ? cfgOptionStr(cfgOptRepoS3Token) : NULL, STORAGE_S3_PARTSIZE_MIN,
STORAGE_S3_DELETE_MAX, host, port, STORAGE_S3_TIMEOUT_DEFAULT, cfgOptionBool(cfgOptRepoS3VerifyTls),
STORAGE_S3_DELETE_MAX, host, port, ioTimeoutMs(), cfgOptionBool(cfgOptRepoS3VerifyTls),
cfgOptionTest(cfgOptRepoS3CaFile) ? cfgOptionStr(cfgOptRepoS3CaFile) : NULL,
cfgOptionTest(cfgOptRepoS3CaPath) ? cfgOptionStr(cfgOptRepoS3CaPath) : NULL);
}

1
src/storage/s3/storage.h
View File

@ -27,7 +27,6 @@ typedef enum
/***********************************************************************************************************************************
Defaults
***********************************************************************************************************************************/
#define STORAGE_S3_TIMEOUT_DEFAULT 60000
#define STORAGE_S3_PARTSIZE_MIN ((size_t)5 * 1024 * 1024)
#define STORAGE_S3_DELETE_MAX 1000

1
test/src/module/command/helpTest.c
View File

@ -157,6 +157,7 @@ testRun(void)
" --config-path base path of pgBackRest configuration files\n"
" [default=/etc/pgbackrest]\n"
" --delta restore or backup using checksums [default=n]\n"
" --io-timeout i/O timeout [default=60]\n"
" --lock-path path where lock files are stored\n"
" [default=/tmp/pgbackrest]\n"
" --neutral-umask use a neutral umask [default=y]\n"

6
test/src/module/common/ioTest.c
View File

@ -249,11 +249,15 @@ testRun(void)
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("ioBufferSize() and ioBufferSizeSet()"))
if (testBegin("ioBufferSize()/ioBufferSizeSet() and ioTimeoutMs()/ioTimeoutMsSet()"))
{
TEST_RESULT_UINT(ioBufferSize(), 65536, "check initial buffer size");
TEST_RESULT_VOID(ioBufferSizeSet(16384), "set buffer size");
TEST_RESULT_UINT(ioBufferSize(), 16384, "check buffer size");
TEST_RESULT_UINT(ioTimeoutMs(), 60000, "check initial timeout ms");
TEST_RESULT_VOID(ioTimeoutMsSet(77777), "set timeout ms");
TEST_RESULT_UINT(ioTimeoutMs(), 77777, "check timeout ms");
}
// *****************************************************************************************************************************

3
test/src/module/config/loadTest.c
View File

@ -360,6 +360,7 @@ testRun(void)
TEST_RESULT_VOID(cfgLoad(strLstSize(argList), strLstPtr(argList)), "load config and don't set umask");
TEST_RESULT_BOOL(socketLocal.init, true, " check socketLocal.init");
TEST_RESULT_BOOL(socketLocal.keepAlive, false, " check socketLocal.keepAlive");
TEST_RESULT_UINT(ioTimeoutMs(), 60000, " check io timeout");
// Set a distinct umask value and test that the umask is reset by configLoad since default for neutral-umask=y
// -------------------------------------------------------------------------------------------------------------------------
@ -369,11 +370,13 @@ testRun(void)
strLstAdd(argList, strNew("--log-level-console=off"));
strLstAdd(argList, strNew("--log-level-stderr=off"));
strLstAdd(argList, strNew("--log-level-file=off"));
strLstAdd(argList, strNew("--io-timeout=95.5"));
strLstAdd(argList, strNew("archive-get"));
umask(0111);
TEST_RESULT_VOID(cfgLoad(strLstSize(argList), strLstPtr(argList)), "load config for neutral-umask");
TEST_RESULT_INT(umask(0111), 0000, " umask was reset");
TEST_RESULT_UINT(ioTimeoutMs(), 95500, " check io timeout");
// Set a distinct umask value and test that the umask is not reset by configLoad with option --no-neutral-umask
// -------------------------------------------------------------------------------------------------------------------------