db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-15 01:04:37 +02:00
Code Issues Releases Activity

Add retries to PostgreSQL sleep when starting a backup.

Browse Source 
Inaccuracies in sleep time or clock skew might make a single sleep insufficient to reach the next second.

Add a few retries to make the process more reliable but still avoid an infinite loop if something is seriously wrong.
This commit is contained in:
David Steele
2020-12-02 22:41:14 -05:00
parent ec9f23d31f
commit d4211d3aaf
3 changed files with 39 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
doc/xml/release.xml
View File

@ -89,6 +89,15 @@
<p>Improve performance of large file lists in <cmd>backup</cmd>/<cmd>restore</cmd> commands.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-ideator id="vitaliy.kukharik"/>
<release-item-reviewer id="cynthia.shang"/>
</release-item-contributor-list>
<p>Add retries to <postgres/> sleep when starting a backup.</p>
</release-item>
</release-improvement-list>
<release-development-list>

22
src/command/backup/backup.c
View File

@ -324,11 +324,25 @@ backupTime(BackupData *backupData, bool waitRemainder)
// Sleep the remainder of the second when requested (this is so copyStart is not subject to one second resolution issues)
if (waitRemainder)
{
sleepMSec(MSEC_PER_SEC - (timeMSec % MSEC_PER_SEC));
unsigned int retry = 0;
// Check time again to be sure we slept long enough
if (result >= (time_t)(dbTimeMSec(backupData->dbPrimary) / MSEC_PER_SEC))
THROW(AssertError, "invalid sleep for online backup time with wait remainder");
// Just to be safe we'll loop until PostgreSQL reports that we have slept long enough
do
{
// Error if the clock has not advanced after several attempts
if (retry == 3)
THROW_FMT(KernelError, PG_NAME " clock has not advanced to the next second after %u tries", retry);
// Sleep remainder of current second
sleepMSec(((TimeMSec)(result + 1) * MSEC_PER_SEC) - timeMSec);
// Check time again to be sure we slept long enough
timeMSec = dbTimeMSec(backupData->dbPrimary);
// Increment retry to prevent an infinite loop
retry++;
}
while ((time_t)(timeMSec / MSEC_PER_SEC) <= result);
}
}

15
test/src/module/command/backupTest.c
View File

@ -1206,7 +1206,7 @@ testRun(void)
const String *repoPath = strNewFmt("%s/repo", testPath());
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error when second does not advance after sleep");
TEST_TITLE("sleep retries and stall error");
StringList *argList = strLstNew();
strLstAddZ(argList, "--" CFGOPT_STANZA "=test1");
@ -1225,9 +1225,16 @@ testRun(void)
// Connect to primary
HRNPQ_MACRO_OPEN_GE_92(1, "dbname='postgres' port=5432", PG_VERSION_93, strZ(pg1Path), false, NULL, NULL),
// Don't advance time after wait
// Advance the time slowly to force retries
HRNPQ_MACRO_TIME_QUERY(1, 1575392588998),
HRNPQ_MACRO_TIME_QUERY(1, 1575392588999),
HRNPQ_MACRO_TIME_QUERY(1, 1575392589001),
// Stall time to force an error
HRNPQ_MACRO_TIME_QUERY(1, 1575392589998),
HRNPQ_MACRO_TIME_QUERY(1, 1575392589997),
HRNPQ_MACRO_TIME_QUERY(1, 1575392589998),
HRNPQ_MACRO_TIME_QUERY(1, 1575392589999),
HRNPQ_MACRO_DONE()
});
@ -1235,7 +1242,9 @@ testRun(void)
BackupData *backupData = backupInit(
infoBackupNew(PG_VERSION_93, PG_VERSION_93, pgCatalogTestVersion(PG_VERSION_93), NULL));
TEST_ERROR(backupTime(backupData, true), AssertError, "invalid sleep for online backup time with wait remainder");
TEST_RESULT_INT(backupTime(backupData, true), 1575392588, "multiple tries for sleep");
TEST_ERROR(backupTime(backupData, true), KernelError, "PostgreSQL clock has not advanced to the next second after 3 tries");
dbFree(backupData->dbPrimary);
}