diff --git a/README.md b/README.md index 9a05bc500..ff0a7df3e 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ PgBackRest aims to be a simple backup and restore system that can seamlessly sca * Removed dependency on Storable and replaced with a custom ini file implementation. +* Added much needed documentation (see INSTALL.md). + * Numerous other changes that can only be identified with a diff. ### v0.19: improved error reporting/handling diff --git a/bin/pg_backrest.pl b/bin/pg_backrest.pl index d58e1346a..d57de64e8 100755 --- a/bin/pg_backrest.pl +++ b/bin/pg_backrest.pl @@ -47,6 +47,9 @@ pg_backrest.pl [options] [operation] Backup Options: --type type of backup to perform (full, diff, incr) + --no-start-stop do not call pg_start/stop_backup(). Postmaster should not be running. + --force force backup when --no-start-stop passed and postmaster.pid exists. + Use with extreme caution as this will produce an inconsistent backup! =cut #################################################################################################################################### @@ -102,27 +105,31 @@ use constant #################################################################################################################################### # Command line parameters #################################################################################################################################### -my $strConfigFile; # Configuration file -my $strStanza; # Stanza in the configuration file to load -my $strType; # Type of backup: full, differential (diff), incremental (incr) -my $bVersion = false; # Display version and exit -my $bHelp = false; # Display help and exit +my $strConfigFile; # Configuration file +my $strStanza; # Stanza in the configuration file to load +my $strType; # Type of backup: full, differential (diff), incremental (incr) +my $bNoStartStop = false; # Do not perform start/stop backup (and archive-required gets set to false) +my $bForce = false; # Force an action that would not normally be allowed (varies by action) +my $bVersion = false; # Display version and exit +my $bHelp = false; # Display help and exit # Test parameters - not for general use my $bNoFork = false; # Prevents the archive process from forking when local archiving is enabled my $bTest = false; # Enters test mode - not harmful in anyway, but adds special logging and pauses for unit testing my $iTestDelay = 5; # Amount of time to delay after hitting a test point (the default would not be enough for manual tests) -GetOptions ('config=s' => \$strConfigFile, - 'stanza=s' => \$strStanza, - 'type=s' => \$strType, - 'version' => \$bVersion, - 'help' => \$bHelp, +GetOptions ('config=s' => \$strConfigFile, + 'stanza=s' => \$strStanza, + 'type=s' => \$strType, + 'no-start-stop' => \$bNoStartStop, + 'force' => \$bForce, + 'version' => \$bVersion, + 'help' => \$bHelp, # Test parameters - not for general use (and subject to change without notice) - 'no-fork' => \$bNoFork, - 'test' => \$bTest, - 'test-delay=s' => \$iTestDelay) + 'no-fork' => \$bNoFork, + 'test' => \$bTest, + 'test-delay=s' => \$iTestDelay) or pod2usage(2); # Display version and exit if requested @@ -637,7 +644,7 @@ if (!lock_file_create($strLockPath)) remote_exit(0); } -# Run file_init_archive - the rest of the file config required for backup and restore +# Initialize the default file object my $oFile = BackRest::File->new ( strStanza => $strStanza, @@ -646,13 +653,19 @@ my $oFile = BackRest::File->new strBackupPath => config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true) ); -my $oDb = BackRest::Db->new -( - strDbUser => config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER), - strDbHost => config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST), - strCommandPsql => config_key_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_PSQL), - oDbSSH => $oFile->{oDbSSH} -); +# Initialize the db object +my $oDb; + +if (!$bNoStartStop) +{ + $oDb = BackRest::Db->new + ( + strDbUser => config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER), + strDbHost => config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST), + strCommandPsql => config_key_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_PSQL), + oDbSSH => $oFile->{oDbSSH} + ); +} # Run backup_init - parameters required for backup and restore operations backup_init @@ -666,8 +679,8 @@ backup_init config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX), config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_ARCHIVE_REQUIRED, true, 'y') eq 'y' ? true : false, config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT), - $bTest, - $iTestDelay + $bNoStartStop, + $bForce ); #################################################################################################################################### diff --git a/lib/BackRest/Backup.pm b/lib/BackRest/Backup.pm index ee718fa89..219d81e2f 100644 --- a/lib/BackRest/Backup.pm +++ b/lib/BackRest/Backup.pm @@ -34,6 +34,8 @@ my $iThreadLocalMax; my $iThreadThreshold = 10; my $iSmallFileThreshold = 65536; my $bArchiveRequired; +my $bNoStartStop; +my $bForce; my $iThreadTimeout; # Thread variables @@ -56,6 +58,8 @@ sub backup_init my $iThreadMaxParam = shift; my $bArchiveRequiredParam = shift; my $iThreadTimeoutParam = shift; + my $bNoStartStopParam = shift; + my $bForceParam = shift; $oDb = $oDbParam; $oFile = $oFileParam; @@ -64,8 +68,15 @@ sub backup_init $bHardLink = $bHardLinkParam; $bNoChecksum = $bNoChecksumParam; $iThreadMax = $iThreadMaxParam; - $bArchiveRequired = $bArchiveRequiredParam; $iThreadTimeout = $iThreadTimeoutParam; + $bNoStartStop = $bNoStartStopParam; + $bForce = $bForceParam; + + # If no-start-stop is specified then archive-required must be false + if ($bNoStartStop) + { + $bArchiveRequired = false; + } if (!defined($iThreadMax)) { @@ -802,7 +813,7 @@ sub backup_manifest_build foreach my $strName (sort(keys $oManifestHash{name})) { # Skip certain files during backup - if ($strName =~ /^pg\_xlog\/.*/ || # pg_xlog/ - this will be reconstructed + if (($strName =~ /^pg\_xlog\/.*/ && !$bNoStartStop) || # pg_xlog/ - this will be reconstructed $strName =~ /^postmaster\.pid$/) # postmaster.pid - to avoid confusing postgres when restoring { next; @@ -1312,18 +1323,65 @@ sub backup my $strBackupTmpPath = $oFile->path_get(PATH_BACKUP_TMP); my $strBackupConfFile = $oFile->path_get(PATH_BACKUP_TMP, 'backup.manifest'); - # Start backup + # Start backup (unless no-start-stop is set) ${oBackupManifest}{backup}{'timestamp-start'} = $strTimestampStart; + my $strArchiveStart; + + if ($bNoStartStop) + { + if ($oFile->exists(PATH_DB_ABSOLUTE, $strDbClusterPath . '/postmaster.pid')) + { + if ($bForce) + { + &log(WARN, '--no-start-stop passed and postmaster.pid exists but --force was passed so backup will continue, ' . + 'though it looks like the postmaster is running and the backup will probably not be consistent'); + } + else + { + &log(ERROR, '--no-start-stop passed but postmaster.pid exists - looks like the postmaster is running. ' . + 'Shutdown the postmaster and try again, or use --force.'); + exit 1; + } + } + } + else + { + $strArchiveStart = $oDb->backup_start('pg_backrest backup started ' . $strTimestampStart, $bStartFast); + ${oBackupManifest}{backup}{'archive-start'} = $strArchiveStart; + &log(INFO, 'archive start: ' . ${oBackupManifest}{backup}{'archive-start'}); + } - my $strArchiveStart = $oDb->backup_start('pg_backrest backup started ' . $strTimestampStart, $bStartFast); - ${oBackupManifest}{backup}{'archive-start'} = $strArchiveStart; ${oBackupManifest}{backup}{version} = version_get(); - &log(INFO, 'archive start: ' . ${oBackupManifest}{backup}{'archive-start'}); - # Build the backup manifest my %oTablespaceMap; - $oDb->tablespace_map_get(\%oTablespaceMap); + + if ($bNoStartStop) + { + my %oTablespaceManifestHash; + $oFile->manifest(PATH_DB_ABSOLUTE, $strDbClusterPath . '/pg_tblspc', \%oTablespaceManifestHash); + + foreach my $strName (sort(keys $oTablespaceManifestHash{name})) + { + if ($strName eq '.' or $strName eq '..') + { + next; + } + + if ($oTablespaceManifestHash{name}{"${strName}"}{type} ne 'l') + { + confess &log(ERROR, "pg_tblspc/${strName} is not a link"); + } + + &log(DEBUG, "Found tablespace ${strName}"); + + $oTablespaceMap{oid}{"${strName}"}{name} = $strName; + } + } + else + { + $oDb->tablespace_map_get(\%oTablespaceMap); + } backup_manifest_build($strDbClusterPath, \%oBackupManifest, \%oLastManifest, \%oTablespaceMap); &log(TEST, TEST_MANIFEST_BUILD); @@ -1355,11 +1413,15 @@ sub backup # Perform the backup backup_file($strDbClusterPath, \%oBackupManifest); - # Stop backup - my $strArchiveStop = $oDb->backup_stop(); + # Stop backup (unless no-start-stop is set) + my $strArchiveStop; - ${oBackupManifest}{backup}{'archive-stop'} = $strArchiveStop; - &log(INFO, 'archive stop: ' . ${oBackupManifest}{backup}{'archive-stop'}); + if (!$bNoStartStop) + { + $strArchiveStop = $oDb->backup_stop(); + ${oBackupManifest}{backup}{'archive-stop'} = $strArchiveStop; + &log(INFO, 'archive stop: ' . ${oBackupManifest}{backup}{'archive-stop'}); + } # If archive logs are required to complete the backup, then fetch them. This is the default, but can be overridden if the # archive logs are going to a different server. Be careful here because there is no way to verify that the backup will be diff --git a/test/lib/BackRestTest/BackupTest.pm b/test/lib/BackRestTest/BackupTest.pm index e42e2132c..5d97acf84 100755 --- a/test/lib/BackRestTest/BackupTest.pm +++ b/test/lib/BackRestTest/BackupTest.pm @@ -187,6 +187,18 @@ sub BackRestTestBackup_Create mkdir(BackRestTestCommon_DbCommonPathGet()) or confess 'Unable to create ' . BackRestTestCommon_DbCommonPathGet() . ' path'; + # Create the db/tablespace directory + mkdir(BackRestTestCommon_DbTablespacePathGet()) + or confess 'Unable to create ' . BackRestTestCommon_DbTablespacePathGet() . ' path'; + + # Create the db/tablespace/ts1 directory + mkdir(BackRestTestCommon_DbTablespacePathGet() . '/ts1') + or confess 'Unable to create ' . BackRestTestCommon_DbTablespacePathGet() . '/ts1 path'; + + # Create the db/tablespace/ts2 directory + mkdir(BackRestTestCommon_DbTablespacePathGet() . '/ts2') + or confess 'Unable to create ' . BackRestTestCommon_DbTablespacePathGet() . '/ts2 path'; + # Create the archive directory mkdir(BackRestTestCommon_ArchivePathGet(), oct('0700')) or confess 'Unable to create ' . BackRestTestCommon_ArchivePathGet() . ' path'; @@ -577,6 +589,13 @@ sub BackRestTestBackup_Test &log(INFO, ' ' . ($iIncr == 0 ? ('full ' . sprintf('%02d', $iFull)) : (' incr ' . sprintf('%02d', $iIncr)))); + # Create tablespace + if ($iIncr == 0) + { + BackRestTestBackup_PgExecute("create tablespace ts1 location '" . + BackRestTestCommon_DbTablespacePathGet() . "/ts1'", true); + } + # Create a table in each backup to check references BackRestTestBackup_PgExecute("create table test_backup_${iIncr} (id int)", true); diff --git a/test/lib/BackRestTest/CommonTest.pm b/test/lib/BackRestTest/CommonTest.pm index 9d8fa1bd1..16846534f 100755 --- a/test/lib/BackRestTest/CommonTest.pm +++ b/test/lib/BackRestTest/CommonTest.pm @@ -29,7 +29,8 @@ our @EXPORT = qw(BackRestTestCommon_Setup BackRestTestCommon_ExecuteBegin BackRe BackRestTestCommon_CommandRemoteGet BackRestTestCommon_HostGet BackRestTestCommon_UserGet BackRestTestCommon_GroupGet BackRestTestCommon_UserBackRestGet BackRestTestCommon_TestPathGet BackRestTestCommon_DataPathGet BackRestTestCommon_BackupPathGet BackRestTestCommon_ArchivePathGet - BackRestTestCommon_DbPathGet BackRestTestCommon_DbCommonPathGet BackRestTestCommon_DbPortGet); + BackRestTestCommon_DbPathGet BackRestTestCommon_DbCommonPathGet BackRestTestCommon_DbTablespacePathGet + BackRestTestCommon_DbPortGet); my $strPgSqlBin; my $strCommonStanza; @@ -46,6 +47,7 @@ my $strCommonBackupPath; my $strCommonArchivePath; my $strCommonDbPath; my $strCommonDbCommonPath; +my $strCommonDbTablespacePath; my $iCommonDbPort; my $iModuleTestRun; my $bDryRun; @@ -228,6 +230,7 @@ sub BackRestTestCommon_Setup $strCommonArchivePath = "${strCommonTestPath}/archive"; $strCommonDbPath = "${strCommonTestPath}/db"; $strCommonDbCommonPath = "${strCommonTestPath}/db/common"; + $strCommonDbTablespacePath = "${strCommonTestPath}/db/tablespace"; $strCommonCommandMain = "${strBasePath}/bin/pg_backrest.pl"; $strCommonCommandRemote = "${strBasePath}/bin/pg_backrest_remote.pl"; @@ -422,6 +425,11 @@ sub BackRestTestCommon_DbCommonPathGet return $strCommonDbCommonPath; } +sub BackRestTestCommon_DbTablespacePathGet +{ + return $strCommonDbTablespacePath; +} + sub BackRestTestCommon_DbPortGet { return $iCommonDbPort;