From c41fb575fbef7b4ce9714986b08f48d32b719862 Mon Sep 17 00:00:00 2001 From: David Steele Date: Thu, 26 Sep 2019 17:39:45 -0400 Subject: [PATCH] Add standby restore type. This restore type automatically adds standby_mode=on to recovery.conf. This could be accomplished previously by setting --recovery-option=standby_mode=on but PostgreSQL 12 requires standby mode to be enabled by a special file named standby.signal. The new restore type allows us to maintain a common interface between PostgreSQL versions. --- build/lib/pgBackRestBuild/Config/Data.pm | 5 ++++ doc/xml/reference.xml | 1 + doc/xml/release.xml | 6 ++++ doc/xml/user-guide.xml | 18 ++++++------ lib/pgBackRest/LibCAuto.pm | 2 ++ src/command/restore/restore.c | 9 ++++++ src/config/define.auto.c | 6 +++- src/perl/embed.auto.c | 2 ++ test/expect/real-all-001.log | 7 ++--- test/expect/real-all-002.log | 6 ++-- test/expect/real-all-003.log | 6 ++-- test/expect/real-all-005.log | 6 ++-- .../pgBackRestTest/Module/Real/RealAllTest.pm | 10 ++++--- test/src/module/command/restoreTest.c | 28 +++++++++++++++++++ 14 files changed, 84 insertions(+), 28 deletions(-) diff --git a/build/lib/pgBackRestBuild/Config/Data.pm b/build/lib/pgBackRestBuild/Config/Data.pm index 68cd9cebb..4c67bf030 100644 --- a/build/lib/pgBackRestBuild/Config/Data.pm +++ b/build/lib/pgBackRestBuild/Config/Data.pm @@ -438,6 +438,8 @@ use constant CFGOPTVAL_RESTORE_TYPE_IMMEDIATE => 'immediat push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_IMMEDIATE); use constant CFGOPTVAL_RESTORE_TYPE_DEFAULT => 'default'; push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_DEFAULT); +use constant CFGOPTVAL_RESTORE_TYPE_STANDBY => 'standby'; + push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_STANDBY); # Restore target action #----------------------------------------------------------------------------------------------------------------------------------- @@ -949,6 +951,7 @@ my %hConfigDefine = [ &CFGOPTVAL_RESTORE_TYPE_DEFAULT, &CFGOPTVAL_RESTORE_TYPE_NAME, + &CFGOPTVAL_RESTORE_TYPE_STANDBY, &CFGOPTVAL_RESTORE_TYPE_TIME, &CFGOPTVAL_RESTORE_TYPE_XID, ], @@ -1003,6 +1006,7 @@ my %hConfigDefine = &CFGOPTVAL_RESTORE_TYPE_NONE, &CFGOPTVAL_RESTORE_TYPE_IMMEDIATE, &CFGOPTVAL_RESTORE_TYPE_DEFAULT, + &CFGOPTVAL_RESTORE_TYPE_STANDBY, ] } } @@ -2389,6 +2393,7 @@ my %hConfigDefine = &CFGOPTVAL_RESTORE_TYPE_IMMEDIATE, &CFGOPTVAL_RESTORE_TYPE_NAME, &CFGOPTVAL_RESTORE_TYPE_TIME, + &CFGOPTVAL_RESTORE_TYPE_STANDBY, &CFGOPTVAL_RESTORE_TYPE_XID, ], }, diff --git a/doc/xml/reference.xml b/doc/xml/reference.xml index 1a6399299..fea304a53 100644 --- a/doc/xml/reference.xml +++ b/doc/xml/reference.xml @@ -1055,6 +1055,7 @@
  • xid - recover to the transaction id specified in --target.
  • time - recover to the time specified in --target.
  • preserve - preserve the existing recovery.conf file.
    • +
  • standby - add standby_mode=on to recovery.conf file so cluster will start in standby mode.
  • none - no recovery.conf file is written so will attempt to achieve consistency using WAL segments present in pg_xlog/pg_wal. Provide the required WAL segments or use the archive-copy setting to include them with the backup.
    • xid diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 0b2cebd6a..83bf6db3a 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -14,6 +14,12 @@ + + +

    Add standby restore type which automatically adds standby_mode to recovery.conf.

    +     +     + diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index d44ebda5d..9f3fe02e4 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -2611,7 +2611,7 @@

    A hot standby performs replication using the WAL archive and allows read-only queries.

    -

    configuration is very similar to {[host-pg1]} except that the standby_mode setting will be enabled to keep the cluster in recovery mode when the end of the WAL stream has been reached.

    +

    configuration is very similar to {[host-pg1]} except that the standby recovery type will be used to keep the cluster in recovery mode when the end of the WAL stream has been reached.

    Configure <backrest/> on the standby @@ -2620,8 +2620,6 @@ {[host-repo1]} - standby_mode=on - detail off @@ -2656,11 +2654,11 @@ Restore the {[postgres-cluster-demo]} standby cluster - {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta restore + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta --type=standby restore - {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} restore + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=standby restore @@ -2668,8 +2666,6 @@ - The standby_mode setting has been written into the recovery.conf file. Configuring recovery settings in means that the recovery.conf file does not need to be stored elsewhere since it will be properly recreated with each restore. The --type=preserve option can be used with the restore to leave the existing recovery.conf file in place if that behavior is preferred. -

    The hot_standby setting must be enabled before starting to allow read-only connections on {[host-pg2]}. Otherwise, connection attempts will be refused. The rest of the configuration is in case the standby is promoted to a primary.

    @@ -2851,7 +2847,7 @@ - {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta restore + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta --type=standby restore @@ -2859,6 +2855,8 @@ + The primary_conninfo setting has been written into the recovery.conf file. Configuring recovery settings in means that the recovery.conf file does not need to be stored elsewhere since it will be properly recreated with each restore. The {[dash]}-type=preserve option can be used with the restore to leave the existing recovery.conf file in place if that behavior is preferred. +

    By default {[user-guide-os]} stores the postgresql.conf file in the data directory. That means the change made to postgresql.conf was overwritten by the last restore and the hot_standby setting must be enabled again. Other solutions to this problem are to store the postgresql.conf file elsewhere or to enable the hot_standby setting on the {[host-pg1]} host where it will be ignored.

    @@ -3321,11 +3319,11 @@ Restore the {[postgres-cluster-demo]} standby cluster - {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta restore + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta --type=standby restore - {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} restore + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=standby restore diff --git a/lib/pgBackRest/LibCAuto.pm b/lib/pgBackRest/LibCAuto.pm index d31fe4a8e..d67937092 100644 --- a/lib/pgBackRest/LibCAuto.pm +++ b/lib/pgBackRest/LibCAuto.pm @@ -53,6 +53,7 @@ sub libcAutoConstant CFGOPTVAL_RESTORE_TYPE_NONE => 'none', CFGOPTVAL_RESTORE_TYPE_IMMEDIATE => 'immediate', CFGOPTVAL_RESTORE_TYPE_DEFAULT => 'default', + CFGOPTVAL_RESTORE_TYPE_STANDBY => 'standby', CFGDEF_TYPE_BOOLEAN => 0, CFGDEF_TYPE_FLOAT => 1, @@ -114,6 +115,7 @@ sub libcAutoExportTag 'CFGOPTVAL_RESTORE_TYPE_NONE', 'CFGOPTVAL_RESTORE_TYPE_IMMEDIATE', 'CFGOPTVAL_RESTORE_TYPE_DEFAULT', + 'CFGOPTVAL_RESTORE_TYPE_STANDBY', 'CFGCMD_ARCHIVE_GET', 'CFGCMD_ARCHIVE_GET_ASYNC', 'CFGCMD_ARCHIVE_PUSH', diff --git a/src/command/restore/restore.c b/src/command/restore/restore.c index 3a1b594d8..adc38a5dd 100644 --- a/src/command/restore/restore.c +++ b/src/command/restore/restore.c @@ -40,6 +40,7 @@ Recovery constants #define RECOVERY_TARGET_INCLUSIVE "recovery_target_inclusive" #define RECOVERY_TARGET_TIMELINE "recovery_target_timeline" #define PAUSE_AT_RECOVERY_TARGET "pause_at_recovery_target" +#define STANDBY_MODE "standby_mode" #define RECOVERY_TYPE_DEFAULT "default" STRING_STATIC(RECOVERY_TYPE_DEFAULT_STR, RECOVERY_TYPE_DEFAULT); @@ -49,6 +50,8 @@ Recovery constants STRING_STATIC(RECOVERY_TYPE_NONE_STR, RECOVERY_TYPE_NONE); #define RECOVERY_TYPE_PRESERVE "preserve" STRING_STATIC(RECOVERY_TYPE_PRESERVE_STR, RECOVERY_TYPE_PRESERVE); +#define RECOVERY_TYPE_STANDBY "standby" + STRING_STATIC(RECOVERY_TYPE_STANDBY_STR, RECOVERY_TYPE_STANDBY); /*********************************************************************************************************************************** Validate restore path @@ -1179,6 +1182,12 @@ restoreRecoveryConf(unsigned int pgVersion) { strCat(result, RECOVERY_TARGET " = '" RECOVERY_TYPE_IMMEDIATE "'\n"); } + // Else recovery type is standby + else if (strEq(cfgOptionStr(cfgOptType), RECOVERY_TYPE_STANDBY_STR)) + { + // Write standby_mode + strCatFmt(result, STANDBY_MODE " = 'on'\n"); + } // Else recovery type is not default so write target options else if (!strEq(cfgOptionStr(cfgOptType), RECOVERY_TYPE_DEFAULT_STR)) { diff --git a/src/config/define.auto.c b/src/config/define.auto.c index 12ec0fb60..9424a6419 100644 --- a/src/config/define.auto.c +++ b/src/config/define.auto.c @@ -2645,6 +2645,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST "immediate", "name", "time", + "standby", "xid" ) ) @@ -4610,6 +4611,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST cfgDefOptType, "default", "name", + "standby", "time", "xid" ) @@ -4788,7 +4790,8 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST "preserve", "none", "immediate", - "default" + "default", + "standby" ) CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("default") @@ -4805,6 +4808,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST "* xid - recover to the transaction id specified in --target.\n" "* time - recover to the time specified in --target.\n" "* preserve - preserve the existing recovery.conf file.\n" + "* standby - add standby_mode=on to recovery.conf file so cluster will start in standby mode.\n" "* none - no recovery.conf file is written so PostgreSQL will attempt to achieve consistency using WAL " "segments present in pg_xlog/pg_wal. Provide the required WAL segments or use the archive-copy setting to " "include them with the backup." diff --git a/src/perl/embed.auto.c b/src/perl/embed.auto.c index b96d4f453..adb3f9611 100644 --- a/src/perl/embed.auto.c +++ b/src/perl/embed.auto.c @@ -7861,6 +7861,7 @@ static const EmbeddedModule embeddedModule[] = "CFGOPTVAL_RESTORE_TYPE_NONE => 'none',\n" "CFGOPTVAL_RESTORE_TYPE_IMMEDIATE => 'immediate',\n" "CFGOPTVAL_RESTORE_TYPE_DEFAULT => 'default',\n" + "CFGOPTVAL_RESTORE_TYPE_STANDBY => 'standby',\n" "\n" "CFGDEF_TYPE_BOOLEAN => 0,\n" "CFGDEF_TYPE_FLOAT => 1,\n" @@ -7921,6 +7922,7 @@ static const EmbeddedModule embeddedModule[] = "'CFGOPTVAL_RESTORE_TYPE_NONE',\n" "'CFGOPTVAL_RESTORE_TYPE_IMMEDIATE',\n" "'CFGOPTVAL_RESTORE_TYPE_DEFAULT',\n" + "'CFGOPTVAL_RESTORE_TYPE_STANDBY',\n" "'CFGCMD_ARCHIVE_GET',\n" "'CFGCMD_ARCHIVE_GET_ASYNC',\n" "'CFGCMD_ARCHIVE_PUSH',\n" diff --git a/test/expect/real-all-001.log b/test/expect/real-all-001.log index abda21c9c..f96657fc9 100644 --- a/test/expect/real-all-001.log +++ b/test/expect/real-all-001.log @@ -227,14 +227,14 @@ restore delta, force, type 'name', target 'backrest' (db-master host) restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"' recovery_target_name = 'backrest' -restore delta, backup '[BACKUP-INCR-1]', type 'default', timeline '4' (db-master host) -> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-INCR-1] --target-timeline="4" --link-all --stanza=db restore +restore delta, backup '[BACKUP-INCR-1]', type 'standby', timeline '4' (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-INCR-1] --type=standby --target-timeline="4" --link-all --stanza=db restore ------------------------------------------------------------------------------------------------------------------------------------ + supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf ---------------------------------------------------------------- -standby_mode = 'on' restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"' +standby_mode = 'on' recovery_target_timeline = '4' incr backup - fail on --no-online (db-master host) @@ -251,7 +251,6 @@ incr backup - succeed on --no-online with --force (db-master host) pg1-path=[TEST_PATH]/db-master/db/base pg1-port=6543 pg1-socket-path=[TEST_PATH]/db-master/db -recovery-option=standby-mode=on [global] archive-async=y diff --git a/test/expect/real-all-002.log b/test/expect/real-all-002.log index 122df49c1..e20172e9d 100644 --- a/test/expect/real-all-002.log +++ b/test/expect/real-all-002.log @@ -70,15 +70,15 @@ repo1-host-config=[TEST_PATH]/db-master/pgbackrest.conf repo1-host-user=[USER-1] spool-path=[TEST_PATH]/db-standby/spool -restore, type 'default', remap - restore backup on replica (db-standby host) -> [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --recovery-option=standby_mode=on --recovery-option="primary_conninfo=host=db-master port=6543 user=replicator" --link-map="pg_xlog=[TEST_PATH]/db-standby/db/pg_xlog" --link-all --stanza=db restore +restore, type 'standby', remap - restore backup on replica (db-standby host) +> [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --recovery-option="primary_conninfo=host=db-master port=6543 user=replicator" --type=standby --link-map="pg_xlog=[TEST_PATH]/db-standby/db/pg_xlog" --link-all --stanza=db restore ------------------------------------------------------------------------------------------------------------------------------------ + supplemental file: [TEST_PATH]/db-standby/db/base/recovery.conf ----------------------------------------------------------------- primary_conninfo = 'host=db-master port=6543 user=replicator' -standby_mode = 'on' restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --stanza=db archive-get %f "%p"' +standby_mode = 'on' full backup - backup from standby, failure to access at least one standby (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --pg8-host=bogus --backup-standby --type=full --stanza=db backup diff --git a/test/expect/real-all-003.log b/test/expect/real-all-003.log index 305437547..a51e3a6e1 100644 --- a/test/expect/real-all-003.log +++ b/test/expect/real-all-003.log @@ -72,15 +72,15 @@ spool-path=[TEST_PATH]/db-standby/spool archive-copy=y start-fast=y -restore, type 'default', remap - restore backup on replica (db-standby host) -> [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --recovery-option=standby_mode=on --recovery-option="primary_conninfo=host=db-master port=6543 user=replicator" --link-map="pg_xlog=[TEST_PATH]/db-standby/db/pg_xlog" --link-all --stanza=db restore +restore, type 'standby', remap - restore backup on replica (db-standby host) +> [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --recovery-option="primary_conninfo=host=db-master port=6543 user=replicator" --type=standby --link-map="pg_xlog=[TEST_PATH]/db-standby/db/pg_xlog" --link-all --stanza=db restore ------------------------------------------------------------------------------------------------------------------------------------ + supplemental file: [TEST_PATH]/db-standby/db/base/recovery.conf ----------------------------------------------------------------- primary_conninfo = 'host=db-master port=6543 user=replicator' -standby_mode = 'on' restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --stanza=db archive-get %f "%p"' +standby_mode = 'on' full backup - backup from standby, failure to reach master (db-standby host) > [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --pg8-host=bogus --backup-standby --type=full --stanza=db backup diff --git a/test/expect/real-all-005.log b/test/expect/real-all-005.log index 4d9c6d2f4..95c183a77 100644 --- a/test/expect/real-all-005.log +++ b/test/expect/real-all-005.log @@ -101,15 +101,15 @@ repo1-path=[TEST_PATH]/backup/repo archive-copy=y start-fast=y -restore, type 'default', remap - restore backup on replica (db-standby host) -> [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --recovery-option=standby_mode=on --recovery-option="primary_conninfo=host=db-master port=6543 user=replicator" --link-map="pg_xlog=[TEST_PATH]/db-standby/db/pg_xlog" --link-all --stanza=db restore +restore, type 'standby', remap - restore backup on replica (db-standby host) +> [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --recovery-option="primary_conninfo=host=db-master port=6543 user=replicator" --type=standby --link-map="pg_xlog=[TEST_PATH]/db-standby/db/pg_xlog" --link-all --stanza=db restore ------------------------------------------------------------------------------------------------------------------------------------ + supplemental file: [TEST_PATH]/db-standby/db/base/recovery.conf ----------------------------------------------------------------- primary_conninfo = 'host=db-master port=6543 user=replicator' -standby_mode = 'on' restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --stanza=db archive-get %f "%p"' +standby_mode = 'on' full backup - backup from standby, failure to access at least one standby (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --pg8-host=bogus --backup-standby --type=full --stanza=db backup diff --git a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm index 7891e522b..a601ee60a 100644 --- a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm +++ b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm @@ -439,9 +439,9 @@ sub run $oHostDbStandby->restore( 'restore backup on replica', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), - {rhRemapHash => \%oRemapHash, + {rhRemapHash => \%oRemapHash, strType => CFGOPTVAL_RESTORE_TYPE_STANDBY, strOptionalParam => - ' --recovery-option=standby_mode=on --recovery-option="primary_conninfo=host=' . HOST_DB_MASTER . + ' --recovery-option="primary_conninfo=host=' . HOST_DB_MASTER . ' port=' . $oHostDbMaster->pgPort() . ' user=replicator"'}); $oHostDbStandby->clusterStart({bHotStandby => true}); @@ -960,8 +960,10 @@ sub run $oHostDbMaster->restore( undef, $strIncrBackup, - {bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_DEFAULT, strTargetTimeline => 4, - rhRecoveryHash => $oHostDbMaster->pgVersion() >= PG_VERSION_90 ? {'standby-mode' => 'on'} : undef}); + {bDelta => true, + strType => $oHostDbMaster->pgVersion() >= PG_VERSION_90 ? + CFGOPTVAL_RESTORE_TYPE_STANDBY : CFGOPTVAL_RESTORE_TYPE_DEFAULT, + strTargetTimeline => 4}); $oHostDbMaster->clusterStart({bHotStandby => true}); $oHostDbMaster->sqlSelectOneTest('select message from test', $strTimelineMessage, {iTimeout => 120}); diff --git a/test/src/module/command/restoreTest.c b/test/src/module/command/restoreTest.c index 9a35dd55e..c7696cd2c 100644 --- a/test/src/module/command/restoreTest.c +++ b/test/src/module/command/restoreTest.c @@ -1323,6 +1323,34 @@ testRun(void) TEST_ERROR( restoreRecoveryConf(PG_VERSION_90), OptionInvalidError, "target-action option is only available in PostgreSQL >= 9.1"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("recovery type = standby"); + + argList = strLstDup(argBaseList); + strLstAddZ(argList, "--type=standby"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_STR_Z( + restoreRecoveryConf(PG_VERSION_94), + "restore_command = 'my_restore_command'\n" + "standby_mode = 'on'\n", + "check recovery options"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("recovery type = standby with timeline"); + + argList = strLstDup(argBaseList); + strLstAddZ(argList, "--type=standby"); + strLstAddZ(argList, "--target-timeline=current"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_STR_Z( + restoreRecoveryConf(PG_VERSION_94), + "restore_command = 'my_restore_command'\n" + "standby_mode = 'on'\n" + "recovery_target_timeline = 'current'\n", + "check recovery options"); } // *****************************************************************************************************************************